diff options
154 files changed, 13910 insertions, 4622 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index f1ff7a2e..4f140e72 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -16,7 +16,7 @@ task: folder: $HOME/.ccache pkginstall_script: - - pkg install -y autoconf automake libtool pkgconf sqlite3 wget unzip ccache gmake + - pkg install -y autoconf automake libtool pkgconf sqlite3 tiff wget unzip ccache gmake download_grid_script: - wget https://download.osgeo.org/proj/proj-datumgrid-1.8.zip - (cd data && unzip -o ../proj-datumgrid-1.8.zip) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d108135..e2264b25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,7 @@ elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel") set(PROJ_CXX_WARN_FLAGS -Wall) endif() endif() + set(PROJ_C_WARN_FLAGS "${PROJ_C_WARN_FLAGS}" CACHE STRING "C flags used to compile PROJ targets") set(PROJ_CXX_WARN_FLAGS "${PROJ_CXX_WARN_FLAGS}" @@ -134,6 +135,39 @@ IF("${SQLITE3_VERSION}" VERSION_LESS "3.11") ENDIF() ################################################################################ +# Check for libtiff +################################################################################ + +option(DISABLE_TIFF "Disable TIFF support" OFF) +mark_as_advanced(DISABLE_TIFF) +if(DISABLE_TIFF) + message(WARNING "TIFF support has been disabled and will result in the inability to read some grids") +else() + find_package(TIFF REQUIRED) + if(TIFF_FOUND) + boost_report_value(TIFF_FOUND) + else() + message(SEND_ERROR "libtiff dependency not found!") + endif() + add_definitions(-DTIFF_ENABLED) +endif() + +################################################################################ +# Check for curl +################################################################################ + +option(ENABLE_CURL "Enable Curl support" ON) +if(ENABLE_CURL) + find_package(CURL REQUIRED) + if(CURL_FOUND) + boost_report_value(CURL_FOUND) + else() + message(SEND_ERROR "curl dependency not found!") + endif() + add_definitions(-DCURL_ENABLED) +endif() + +################################################################################ # threading configuration ################################################################################ set(CMAKE_THREAD_PREFER_PTHREAD TRUE) @@ -772,7 +772,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = src/iso19111 include/proj src/proj.h src/proj_experimental.h src/general_doc.dox +INPUT = src/iso19111 include/proj src/proj.h src/proj_experimental.h src/general_doc.dox src/filemanager.cpp src/networkfilemanager.cpp # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/appveyor.yml b/appveyor.yml index 79d15917..4c388284 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,9 +27,10 @@ build_script: - set PATH=%CD%;%PATH% - cd .. - vcpkg install sqlite3:"%platform%"-windows + - vcpkg install tiff:"%platform%"-windows + - vcpkg install curl:"%platform%"-windows - set SQLITE3_BIN=%APPVEYOR_BUILD_FOLDER%\sqlite3\bin - mkdir %SQLITE3_BIN% - - copy c:\projects\proj\vcpkg\installed\"%platform%"-windows\bin\sqlite3.dll %SQLITE3_BIN% - ps: | appveyor DownloadFile https://sqlite.org/2018/sqlite-tools-win32-x86-3250100.zip 7z x sqlite-tools-win32-x86-3250100.zip @@ -50,6 +51,8 @@ build_script: - set PROJ_DIR=%APPVEYOR_BUILD_FOLDER%\proj_dir - cmake -G "%VS_FULL%" .. -DCMAKE_BUILD_TYPE=Release -DBUILD_LIBPROJ_SHARED="%BUILD_LIBPROJ_SHARED%" -DCMAKE_C_FLAGS="/WX" -DCMAKE_CXX_FLAGS="/WX" -DCMAKE_TOOLCHAIN_FILE=C:/projects/proj/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX="%PROJ_DIR%" - cmake --build . --config Release --target install + - copy c:\projects\proj\vcpkg\installed\"%platform%"-windows\bin\*.dll %PROJ_DIR%\bin + - dir %PROJ_DIR%\bin test_script: - echo test_script diff --git a/cmake/ProjTest.cmake b/cmake/ProjTest.cmake index 81ffcd42..7797ed83 100644 --- a/cmake/ProjTest.cmake +++ b/cmake/ProjTest.cmake @@ -2,6 +2,11 @@ # add test with sh script # +function(proj_test_set_properties TESTNAME) + set_tests_properties( ${TESTNAME} + PROPERTIES ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests") +endfunction() + function(proj_add_test_script_sh SH_NAME BIN_USE) if(UNIX) get_filename_component(testname ${SH_NAME} NAME_WE) @@ -26,8 +31,8 @@ function(proj_add_test_script_sh SH_NAME BIN_USE) COMMAND ${PROJECT_SOURCE_DIR}/test/cli/${SH_NAME} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${${BIN_USE}} ) - set_tests_properties( ${testname} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests") + + proj_test_set_properties(${testname}) endif() endif() @@ -43,8 +48,6 @@ function(proj_add_gie_test TESTNAME TESTCASE) COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${GIE_BIN} ${TESTFILE} ) - set_tests_properties( ${TESTNAME} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests") - + proj_test_set_properties(${TESTNAME}) endfunction() diff --git a/configure.ac b/configure.ac index 715ce3a6..2abf5684 100644 --- a/configure.ac +++ b/configure.ac @@ -20,6 +20,8 @@ AC_PROG_LN_S AC_PROG_MAKE_SET AM_PROG_LIBTOOL +PKG_PROG_PKG_CONFIG + dnl Enable as much warnings as possible AX_CFLAGS_WARN_ALL(C_WFLAGS) AX_CXXFLAGS_WARN_ALL(CXX_WFLAGS) @@ -169,6 +171,15 @@ AC_SUBST(NO_ZERO_AS_NULL_POINTER_CONSTANT_FLAG,$NO_ZERO_AS_NULL_POINTER_CONSTANT CFLAGS="${CFLAGS} -fvisibility=hidden" CXXFLAGS="${CXXFLAGS} -fvisibility=hidden" +case "${host_os}" in + cygwin* | mingw32* | pw32* | beos* | darwin*) + CFLAGS="${CFLAGS} -DNOMINMAX" + CXXFLAGS="${CXXFLAGS} -DNOMINMAX" + ;; + *) + ;; +esac + dnl Checks for libraries. save_CFLAGS="$CFLAGS" CFLAGS=`echo "$CFLAGS" | sed "s/-Werror/ /"` @@ -241,6 +252,78 @@ if test x"$SQLITE3_CHECK" != x"yes" ; then fi dnl --------------------------------------------------------------------------- +dnl Check for libtiff +dnl --------------------------------------------------------------------------- + +AC_ARG_ENABLE([tiff-is-strongly-discouraged], + AS_HELP_STRING([--disable-tiff-is-strongly-discouraged], + [Disable TIFF support. Strongly discouraged !]), + [enable_tiff=no], + [enable_tiff=yes]) + +if test "x$enable_tiff" = "xyes" -o "x$enable_tiff" = ""; then + if test "x$TIFF_CFLAGS$TIFF_LIBS" = "x" ; then + if $PKG_CONFIG libtiff; then + PKG_CHECK_MODULES([TIFF], [libtiff]) + else + PKG_CHECK_MODULES([TIFF], [libtiff-4]) + fi + fi + TIFF_ENABLED_FLAGS=-DTIFF_ENABLED +fi +AC_SUBST(TIFF_CFLAGS,$TIFF_CFLAGS) +AC_SUBST(TIFF_LIBS,$TIFF_LIBS) +AC_SUBST(TIFF_ENABLED_FLAGS,$TIFF_ENABLED_FLAGS) + +dnl --------------------------------------------------------------------------- +dnl Check for curl +dnl --------------------------------------------------------------------------- + +FOUND_CURL=no +CURL_CFLAGS= +CURL_LIB= + +AC_ARG_WITH(curl, + [ --with-curl[=ARG] Enable curl support (ARG=path to curl-config.)],,,) + +dnl Clear some cache variables +unset ac_cv_path_LIBCURL + +if test "`basename xx/$with_curl`" = "curl-config" ; then + LIBCURL_CONFIG="$with_curl" +elif test "$with_curl" = "no" ; then + LIBCURL_CONFIG=no +else + AC_PATH_PROG(LIBCURL_CONFIG, curl-config, not-found) +fi + +if test "$LIBCURL_CONFIG" = "not-found" ; then + AC_MSG_ERROR([curl not found. If wanting to do a build without curl support (and thus without built-in networking capability), explictly disable it with --without-curl]) +elif test "$LIBCURL_CONFIG" != "no" ; then + + CURL_VERNUM=`$LIBCURL_CONFIG --vernum` + CURL_VER=`$LIBCURL_CONFIG --version | awk '{print $2}'` + + AC_MSG_RESULT([ found libcurl version $CURL_VER]) + + AC_CHECK_LIB(curl,curl_global_init,FOUND_CURL=yes,FOUND_CURL=no,`$LIBCURL_CONFIG --libs`) + + if test "$FOUND_CURL" = "no" ; then + AC_MSG_ERROR([curl not found. If wanting to do a build without curl support (and thus without built-in networking capability), explictly disable it with --without-curl]) + fi + CURL_ENABLED_FLAGS=-DCURL_ENABLED +fi + +if test "$FOUND_CURL" = "yes" ; then + CURL_CFLAGS=`$LIBCURL_CONFIG --cflags` + CURL_LIBS=`$LIBCURL_CONFIG --libs` +fi + +AC_SUBST(CURL_CFLAGS,$CURL_CFLAGS) +AC_SUBST(CURL_LIBS,$CURL_LIBS) +AC_SUBST(CURL_ENABLED_FLAGS,$CURL_ENABLED_FLAGS) + +dnl --------------------------------------------------------------------------- dnl Check for external Google Test dnl --------------------------------------------------------------------------- diff --git a/data/CMakeLists.txt b/data/CMakeLists.txt index f0e1dab6..cf20fea4 100644 --- a/data/CMakeLists.txt +++ b/data/CMakeLists.txt @@ -2,6 +2,10 @@ # files containing dictionary of useful projection # +set(CONFIG_FILES + proj.ini +) + set(PROJ_DICTIONARY null world @@ -54,7 +58,7 @@ add_custom_command( add_custom_target(generate_proj_db ALL DEPENDS ${PROJ_DB}) if(NOT "${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}") - foreach(FILE ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES}) + foreach(FILE ${CONFIG_FILES} ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES}) configure_file(${FILE} ${FILE} COPYONLY) endforeach() endif() @@ -78,18 +82,25 @@ set(DATA_FOR_TESTS nzgd2kgrid0005.gsb ITRF2000 BETA2007.gsb) +file(GLOB DATA_TESTS tests/*) execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/for_tests) -foreach(FILE ${DATA_FOR_TESTS}) +execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/for_tests/tests) +foreach(FILE ${DATA_FOR_TESTS} ${CONFIG_FILES}) if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${FILE}) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${FILE} ${CMAKE_CURRENT_BINARY_DIR}/for_tests/${FILE} COPYONLY) endif() endforeach() +foreach(FILE ${DATA_TESTS}) + get_filename_component(FILENAME ${FILE} NAME) + configure_file(${FILE} ${CMAKE_CURRENT_BINARY_DIR}/for_tests/tests/${FILENAME} COPYONLY) +endforeach() # #install # set(ALL_DATA_FILE + ${CONFIG_FILES} ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES} ${PROJ_DB} diff --git a/data/Makefile.am b/data/Makefile.am index 7e822381..0f880aef 100644 --- a/data/Makefile.am +++ b/data/Makefile.am @@ -1,6 +1,6 @@ DATAPATH = $(top_srcdir)/data -pkgdata_DATA = GL27 nad.lst nad27 nad83 world other.extra \ +pkgdata_DATA = proj.ini GL27 nad.lst nad27 nad83 world other.extra \ CH null \ ITRF2000 ITRF2008 ITRF2014 proj.db \ projjson.schema.json @@ -38,12 +38,55 @@ SQL_ORDERED_LIST = sql/begin.sql \ sql/customizations.sql \ sql/commit.sql -EXTRA_DIST = GL27 nad.lst nad27 nad83 \ +EXTRA_DIST = proj.ini GL27 nad.lst nad27 nad83 \ world other.extra \ CH \ ITRF2000 ITRF2008 ITRF2014 \ projjson.schema.json \ - CMakeLists.txt tests/test_nodata.gtx null \ + CMakeLists.txt \ + tests/test_nodata.gtx \ + tests/test_vgrid_bigendian_bigtiff.tif \ + tests/test_vgrid_bigendian.tif \ + tests/test_vgrid_bigtiff.tif \ + tests/test_vgrid_bottomup_with_matrix.tif \ + tests/test_vgrid_bottomup_with_scale.tif \ + tests/test_vgrid_deflate_floatingpointpredictor.tif \ + tests/test_vgrid_deflate.tif \ + tests/test_vgrid_float64.tif \ + tests/test_vgrid_in_second_channel.tif \ + tests/test_vgrid_int16.tif \ + tests/test_vgrid_int32.tif \ + tests/test_vgrid_uint32.tif \ + tests/test_vgrid_invalid_channel_type.tif \ + tests/test_vgrid_nodata.tif \ + tests/test_vgrid_pixelisarea.tif \ + tests/test_vgrid_pixelispoint.tif \ + tests/test_vgrid_uint16.tif \ + tests/test_vgrid_uint16_with_scale_offset.tif \ + tests/test_vgrid_unsupported_byte.tif \ + tests/test_vgrid_with_overview.tif \ + tests/test_vgrid_with_subgrid.tif \ + tests/test_hgrid.tif \ + tests/test_hgrid_separate.tif \ + tests/test_hgrid_tiled.tif \ + tests/test_hgrid_tiled_separate.tif \ + tests/test_hgrid_strip.tif \ + tests/test_hgrid_positive_west.tif \ + tests/test_hgrid_lon_shift_first.tif \ + tests/test_hgrid_radian.tif \ + tests/test_hgrid_degree.tif \ + tests/test_hgrid_with_overview.tif \ + tests/test_hgrid_extra_ifd_with_other_info.tif \ + tests/test_hgrid_with_subgrid.tif \ + tests/test_hgrid_with_subgrid_no_grid_name.tif \ + tests/subset_of_gr3df97a.tif \ + tests/egm96_15_uncompressed_truncated.tif \ + tests/test_vgrid_single_strip_truncated.tif \ + tests/nkgrf03vel_realigned_extract.tif \ + tests/nkgrf03vel_realigned_xy_extract.ct2 \ + tests/nkgrf03vel_realigned_z_extract.gtx \ + tests/test_hgrid_with_two_level_of_subgrids_no_grid_name.tif \ + null \ generate_all_sql_in.cmake sql_filelist.cmake \ $(SQL_ORDERED_LIST) @@ -127,6 +170,8 @@ check-local: echo "WARNING: grid $$x missing: some tests will be skipped"; \ fi \ done; \ + ln -sf ../$(DATAPATH)/tests for_tests; \ + ln -sf ../$(DATAPATH)/proj.ini for_tests; \ ln -sf ../proj.db for_tests clean-local: diff --git a/data/proj.ini b/data/proj.ini new file mode 100644 index 00000000..0ae33249 --- /dev/null +++ b/data/proj.ini @@ -0,0 +1,16 @@ +[general] +; Lines starting by ; are commented lines. +; + +; Network capabilities disabled by default. +; Can be overriden with the PROJ_NETWORK=ON environment variable. +; network = on + +; Can be overriden with the PROJ_NETWORK_ENDPOINT environment variable. +cdn_endpoint = https://cdn.proj.org + +cache_enabled = on + +cache_size_MB = 300 + +cache_ttl_sec = 86400 diff --git a/data/tests/egm96_15_uncompressed_truncated.tif b/data/tests/egm96_15_uncompressed_truncated.tif Binary files differnew file mode 100644 index 00000000..bd34a7e2 --- /dev/null +++ b/data/tests/egm96_15_uncompressed_truncated.tif diff --git a/data/tests/nkgrf03vel_realigned_extract.tif b/data/tests/nkgrf03vel_realigned_extract.tif Binary files differnew file mode 100644 index 00000000..6db8eae4 --- /dev/null +++ b/data/tests/nkgrf03vel_realigned_extract.tif diff --git a/data/tests/nkgrf03vel_realigned_xy_extract.ct2 b/data/tests/nkgrf03vel_realigned_xy_extract.ct2 Binary files differnew file mode 100644 index 00000000..89232b9f --- /dev/null +++ b/data/tests/nkgrf03vel_realigned_xy_extract.ct2 diff --git a/data/tests/nkgrf03vel_realigned_z_extract.gtx b/data/tests/nkgrf03vel_realigned_z_extract.gtx Binary files differnew file mode 100644 index 00000000..5ea8aac7 --- /dev/null +++ b/data/tests/nkgrf03vel_realigned_z_extract.gtx diff --git a/data/tests/subset_of_gr3df97a.tif b/data/tests/subset_of_gr3df97a.tif Binary files differnew file mode 100644 index 00000000..a98783f3 --- /dev/null +++ b/data/tests/subset_of_gr3df97a.tif diff --git a/data/tests/test_hgrid.tif b/data/tests/test_hgrid.tif Binary files differnew file mode 100644 index 00000000..94718c21 --- /dev/null +++ b/data/tests/test_hgrid.tif diff --git a/data/tests/test_hgrid_degree.tif b/data/tests/test_hgrid_degree.tif Binary files differnew file mode 100644 index 00000000..d06782ec --- /dev/null +++ b/data/tests/test_hgrid_degree.tif diff --git a/data/tests/test_hgrid_extra_ifd_with_other_info.tif b/data/tests/test_hgrid_extra_ifd_with_other_info.tif Binary files differnew file mode 100644 index 00000000..b7e67af7 --- /dev/null +++ b/data/tests/test_hgrid_extra_ifd_with_other_info.tif diff --git a/data/tests/test_hgrid_lon_shift_first.tif b/data/tests/test_hgrid_lon_shift_first.tif Binary files differnew file mode 100644 index 00000000..395f0743 --- /dev/null +++ b/data/tests/test_hgrid_lon_shift_first.tif diff --git a/data/tests/test_hgrid_positive_west.tif b/data/tests/test_hgrid_positive_west.tif Binary files differnew file mode 100644 index 00000000..4ebc17cc --- /dev/null +++ b/data/tests/test_hgrid_positive_west.tif diff --git a/data/tests/test_hgrid_radian.tif b/data/tests/test_hgrid_radian.tif Binary files differnew file mode 100644 index 00000000..30219ccd --- /dev/null +++ b/data/tests/test_hgrid_radian.tif diff --git a/data/tests/test_hgrid_separate.tif b/data/tests/test_hgrid_separate.tif Binary files differnew file mode 100644 index 00000000..ef2ca575 --- /dev/null +++ b/data/tests/test_hgrid_separate.tif diff --git a/data/tests/test_hgrid_strip.tif b/data/tests/test_hgrid_strip.tif Binary files differnew file mode 100644 index 00000000..e38fc609 --- /dev/null +++ b/data/tests/test_hgrid_strip.tif diff --git a/data/tests/test_hgrid_tiled.tif b/data/tests/test_hgrid_tiled.tif Binary files differnew file mode 100644 index 00000000..b0d5dd8b --- /dev/null +++ b/data/tests/test_hgrid_tiled.tif diff --git a/data/tests/test_hgrid_tiled_separate.tif b/data/tests/test_hgrid_tiled_separate.tif Binary files differnew file mode 100644 index 00000000..d7e0934f --- /dev/null +++ b/data/tests/test_hgrid_tiled_separate.tif diff --git a/data/tests/test_hgrid_with_overview.tif b/data/tests/test_hgrid_with_overview.tif Binary files differnew file mode 100644 index 00000000..d7453b49 --- /dev/null +++ b/data/tests/test_hgrid_with_overview.tif diff --git a/data/tests/test_hgrid_with_subgrid.tif b/data/tests/test_hgrid_with_subgrid.tif Binary files differnew file mode 100644 index 00000000..46a8f2f4 --- /dev/null +++ b/data/tests/test_hgrid_with_subgrid.tif diff --git a/data/tests/test_hgrid_with_subgrid_no_grid_name.tif b/data/tests/test_hgrid_with_subgrid_no_grid_name.tif Binary files differnew file mode 100644 index 00000000..974699b5 --- /dev/null +++ b/data/tests/test_hgrid_with_subgrid_no_grid_name.tif diff --git a/data/tests/test_hgrid_with_two_level_of_subgrids_no_grid_name.tif b/data/tests/test_hgrid_with_two_level_of_subgrids_no_grid_name.tif Binary files differnew file mode 100644 index 00000000..2abb3226 --- /dev/null +++ b/data/tests/test_hgrid_with_two_level_of_subgrids_no_grid_name.tif diff --git a/data/tests/test_vgrid_bigendian.tif b/data/tests/test_vgrid_bigendian.tif Binary files differnew file mode 100644 index 00000000..5cf4a039 --- /dev/null +++ b/data/tests/test_vgrid_bigendian.tif diff --git a/data/tests/test_vgrid_bigendian_bigtiff.tif b/data/tests/test_vgrid_bigendian_bigtiff.tif Binary files differnew file mode 100644 index 00000000..a586b85f --- /dev/null +++ b/data/tests/test_vgrid_bigendian_bigtiff.tif diff --git a/data/tests/test_vgrid_bigtiff.tif b/data/tests/test_vgrid_bigtiff.tif Binary files differnew file mode 100644 index 00000000..2a01893a --- /dev/null +++ b/data/tests/test_vgrid_bigtiff.tif diff --git a/data/tests/test_vgrid_bottomup_with_matrix.tif b/data/tests/test_vgrid_bottomup_with_matrix.tif Binary files differnew file mode 100644 index 00000000..90f637dc --- /dev/null +++ b/data/tests/test_vgrid_bottomup_with_matrix.tif diff --git a/data/tests/test_vgrid_bottomup_with_scale.tif b/data/tests/test_vgrid_bottomup_with_scale.tif Binary files differnew file mode 100644 index 00000000..636b7dc7 --- /dev/null +++ b/data/tests/test_vgrid_bottomup_with_scale.tif diff --git a/data/tests/test_vgrid_deflate.tif b/data/tests/test_vgrid_deflate.tif Binary files differnew file mode 100644 index 00000000..ee3b5f0a --- /dev/null +++ b/data/tests/test_vgrid_deflate.tif diff --git a/data/tests/test_vgrid_deflate_floatingpointpredictor.tif b/data/tests/test_vgrid_deflate_floatingpointpredictor.tif Binary files differnew file mode 100644 index 00000000..5fd7b9fa --- /dev/null +++ b/data/tests/test_vgrid_deflate_floatingpointpredictor.tif diff --git a/data/tests/test_vgrid_float64.tif b/data/tests/test_vgrid_float64.tif Binary files differnew file mode 100644 index 00000000..16b3e790 --- /dev/null +++ b/data/tests/test_vgrid_float64.tif diff --git a/data/tests/test_vgrid_in_second_channel.tif b/data/tests/test_vgrid_in_second_channel.tif Binary files differnew file mode 100644 index 00000000..d377f8b7 --- /dev/null +++ b/data/tests/test_vgrid_in_second_channel.tif diff --git a/data/tests/test_vgrid_int16.tif b/data/tests/test_vgrid_int16.tif Binary files differnew file mode 100644 index 00000000..1c69b5d6 --- /dev/null +++ b/data/tests/test_vgrid_int16.tif diff --git a/data/tests/test_vgrid_int32.tif b/data/tests/test_vgrid_int32.tif Binary files differnew file mode 100644 index 00000000..1b6dfd7b --- /dev/null +++ b/data/tests/test_vgrid_int32.tif diff --git a/data/tests/test_vgrid_invalid_channel_type.tif b/data/tests/test_vgrid_invalid_channel_type.tif Binary files differnew file mode 100644 index 00000000..ec9e641f --- /dev/null +++ b/data/tests/test_vgrid_invalid_channel_type.tif diff --git a/data/tests/test_vgrid_nodata.tif b/data/tests/test_vgrid_nodata.tif Binary files differnew file mode 100644 index 00000000..65ec5343 --- /dev/null +++ b/data/tests/test_vgrid_nodata.tif diff --git a/data/tests/test_vgrid_pixelisarea.tif b/data/tests/test_vgrid_pixelisarea.tif Binary files differnew file mode 100644 index 00000000..a5409f66 --- /dev/null +++ b/data/tests/test_vgrid_pixelisarea.tif diff --git a/data/tests/test_vgrid_pixelispoint.tif b/data/tests/test_vgrid_pixelispoint.tif Binary files differnew file mode 100644 index 00000000..cfeb598f --- /dev/null +++ b/data/tests/test_vgrid_pixelispoint.tif diff --git a/data/tests/test_vgrid_single_strip_truncated.tif b/data/tests/test_vgrid_single_strip_truncated.tif Binary files differnew file mode 100644 index 00000000..9a0030f6 --- /dev/null +++ b/data/tests/test_vgrid_single_strip_truncated.tif diff --git a/data/tests/test_vgrid_uint16.tif b/data/tests/test_vgrid_uint16.tif Binary files differnew file mode 100644 index 00000000..a03d9a73 --- /dev/null +++ b/data/tests/test_vgrid_uint16.tif diff --git a/data/tests/test_vgrid_uint16_with_scale_offset.tif b/data/tests/test_vgrid_uint16_with_scale_offset.tif Binary files differnew file mode 100644 index 00000000..b08fa4a3 --- /dev/null +++ b/data/tests/test_vgrid_uint16_with_scale_offset.tif diff --git a/data/tests/test_vgrid_uint32.tif b/data/tests/test_vgrid_uint32.tif Binary files differnew file mode 100644 index 00000000..cae7e9e7 --- /dev/null +++ b/data/tests/test_vgrid_uint32.tif diff --git a/data/tests/test_vgrid_unsupported_byte.tif b/data/tests/test_vgrid_unsupported_byte.tif Binary files differnew file mode 100644 index 00000000..ccf03fc8 --- /dev/null +++ b/data/tests/test_vgrid_unsupported_byte.tif diff --git a/data/tests/test_vgrid_with_overview.tif b/data/tests/test_vgrid_with_overview.tif Binary files differnew file mode 100644 index 00000000..aa15aa1d --- /dev/null +++ b/data/tests/test_vgrid_with_overview.tif diff --git a/data/tests/test_vgrid_with_subgrid.tif b/data/tests/test_vgrid_with_subgrid.tif Binary files differnew file mode 100644 index 00000000..5c7584c4 --- /dev/null +++ b/data/tests/test_vgrid_with_subgrid.tif diff --git a/docs/source/apps/cct.rst b/docs/source/apps/cct.rst index a4cc690c..96b5f13b 100644 --- a/docs/source/apps/cct.rst +++ b/docs/source/apps/cct.rst @@ -91,6 +91,17 @@ cartesian coordinates) and *Coordinate Transformations*, which are coordinate operations where input and output datums differ (e.g. change of reference frame). +Use of remote grids +******************* + +.. versionadded:: 7.0.0 + +If the :envvar:`PROJ_NETWORK` environment variable is set to ``ON``, +:program:`cct` will attempt to use remote grids stored on CDN (Content +Delivery Network) storage, when they are not available locally. + +More details are available in the :ref:`network` section. + Examples ******** diff --git a/docs/source/apps/cs2cs.rst b/docs/source/apps/cs2cs.rst index 12f91e64..c95bbaed 100644 --- a/docs/source/apps/cs2cs.rst +++ b/docs/source/apps/cs2cs.rst @@ -171,6 +171,16 @@ normally be in DMS format (use ``-f %.12f`` for decimal degrees with 12 decimal places), while projected (cartesian) coordinates will be in linear (meter, feet) units. +Use of remote grids +------------------- + +.. versionadded:: 7.0.0 + +If the :envvar:`PROJ_NETWORK` environment variable is set to ``ON``, +:program:`cs2cs` will attempt to use remote grids stored on CDN (Content +Delivery Network) storage, when they are not available locally. + +More details are available in the :ref:`network` section. Examples ******** diff --git a/docs/source/apps/projinfo.rst b/docs/source/apps/projinfo.rst index d1889844..47deaaeb 100644 --- a/docs/source/apps/projinfo.rst +++ b/docs/source/apps/projinfo.rst @@ -20,7 +20,7 @@ Synopsis | [[--area name_or_code] | [--bbox west_long,south_lat,east_long,north_lat]] | [--spatial-test contains|intersects] | [--crs-extent-use none|both|intersection|smallest] - | [--grid-check none|discard_missing|sort] [--show-superseded] + | [--grid-check none|discard_missing|sort|known_available] [--show-superseded] | [--pivot-crs always|if_no_direct_transformation|never|{auth:code[,auth:code]*}] | [--boundcrs-to-wgs84] | [--main-db-path path] [--aux-db-path path]* @@ -151,12 +151,13 @@ The following control parameters can appear in any order: .. note:: only used for coordinate operation computation -.. option:: --grid-check none|discard_missing|sort +.. option:: --grid-check none|discard_missing|sort|known_available Specify how the presence or absence of a horizontal or vertical shift grid required for a coordinate operation affects the results returned when researching coordinate operations between 2 CRS. - The default strategy is ``sort``: in that case, all candidate + The default strategy is ``sort`` (if :envvar:`PROJ_NETWORK` is not defined). + In that case, all candidate operations are returned, but the actual availability of the grids is used to determine the sorting order. That is, if a coordinate operation involves using a grid that is not available in the PROJ resource directories @@ -166,6 +167,9 @@ The following control parameters can appear in any order: this returns the results as if all the grids where available. The ``discard_missing`` strategy discards results that involve grids not present in the PROJ resource directories. + The ``known_available`` strategy discards results that involve grids not + present in the PROJ resource directories and that are not known of the CDN. + This is the default strategy is :envvar:`PROJ_NETWORK` is set to ``ON``. .. note:: only used for coordinate operation computation diff --git a/docs/source/community/rfc/rfc-4.rst b/docs/source/community/rfc/rfc-4.rst index 6cd9a82f..5019a5f6 100644 --- a/docs/source/community/rfc/rfc-4.rst +++ b/docs/source/community/rfc/rfc-4.rst @@ -695,613 +695,11 @@ choice. .. _description_geotiff_format: -Description of the PROJ GeoTIFF format -++++++++++++++++++++++++++++++++++++++ - -The general principles that guide the following requirements and recommendations -are such that files will be properly recognized by PROJ, and also by GDAL which -is an easy way to inspect such grid files: - -- `TIFF 6.0 <https://www.awaresystems.be/imaging/tiff/specification/TIFF6.pdf>`_ - based (could possibly be BigTIFF without code changes, if we ever - need some day to handle grids larger than 4GB) - -- `GeoTIFF 1.1 <http://docs.opengeospatial.org/is/19-008r4/19-008r4.html>`_ for the georeferencing. - GeoTIFF 1.1 is a recent standard, compared to the original GeoTIFF 1.0 version, - but its backward compatibility is excellent, so that should not cause much trouble - to readers that are not official GeoTIFF 1.1 compliant. - -- Files hosted on the CDN will use a Geographic 2D CRS for the GeoTIFF GeoKeys. - That CRS is intended to be the interpolation CRS as defined in - `OGC Abstract Specification Topic 2 <http://docs.opengeospatial.org/as/18-005r4/18-005r4.html>`_, - that is the CRS to which grid values are refered to. - - Given that they will nominally be related to the EPSG dataset, the `GeodeticCRSGeoKey - <http://docs.opengeospatial.org/is/19-008r4/19-008r4.html#_requirements_class_geodeticcrsgeokey>`_ - will be used to store the EPSG code of the CRS. If the CRS cannot be reliably - encoded through that key or other geokeys, the ``interpolation_crs_wkt`` metadata - item detailed afterwards should be used. - - This CRS will be generally the source CRS (for geographic to - geographic horizontal shift grids, or geographic to vertical shift grids), but - for vertical to vertical CRS adjustment, this will be the geographic CRS to - which the grid is referenced. In some very rare cases of geographic to vertical - shift grids, the interpolation CRS might be a geographic CRS that is not the - same as the source CRS (into which ellipsoidal height are expressed). The only - instance we have in mind is for the EPSG:7001 "ETRS89 to NAP height (1)" transformation - using the naptrans2008 VDatum-grid which is referenced to Amersfoort EPSG:4289 - instead of ETRS89... - - On the reading side, PROJ will ignore that information: - the CRS is already stored in the source_crs or interpolation_crs column of the - grid_transformation table. - - For geographic to vertical shift files (geoid models), the GeoTIFF 1.1 - convention will be used to store the value of the `VerticalGeoKey - <http://docs.opengeospatial.org/is/19-008r4/19-008r4.html#_requirements_class_verticalgeokey>`_ - So a geoid model that apply to WGS 84 EPSG:4979 will have GeodeticCRSGeoKey = 4326 - and VerticalGeoKey = 4979. - -- Files hosted on the CDN will use the GeoTIFF defined `ModelTiepointTag and ModelPixelScaleTag - <http://docs.opengeospatial.org/is/19-008r4/19-008r4.html#_raster_to_model_coordinate_transformation_requirements>`_ TIFF tags - to store the coordinates of the upper-left pixel and the resolution of the pixels. - On the reading side, they will be required and ModelTransformationTag will be ignored. - - .. note:: - - Regarding anti-meridian handling, a variety of possibilities exist. - We do not attempt to standardize this and filesh hosted on the CDN will use - a georeferencing close to the original data producer. - For example, NOAA vertical grids that apply to Conterminous USA might even have a top-left - longitude beyond 180 (for consistency with Alaska grids, whose origin is < 180) - Anti-meridian handling in PROJ has probably issues. This RFC does not attempt - to address them in particular, as they are believed to be orthogonal to the - topics it covers, and being mostly implementation issues. - -- Files hosted on the CDN will use the `GTRasterTypeGeoKey - <http://docs.opengeospatial.org/is/19-008r4/19-008r4.html#_requirements_class_gtrastertypegeokey>`_ - = PixelIsPoint convention. - This is the convention used by most existing grid formats currently. Note that GDAL - typically use a PixelIsArea convention (but can handle both conventions), so the - georeferencing it displays when opening a .gsb or .gtx file appears to have a - half-pixel shift regarding to the coordinates stored in the original grid file. On - the reading side, PROJ will accept both conventions (for equivalent georeferencing, - the value of the origin in a PixelIsArea convention is shifted by a half-pixel - towards the upper-left direction). Unspecified behaviour if this GeoKey is absent. - -- Files hosted on the CDN will be tiled, presumably with 256x256 tiles (small - grids that are smaller than 256x256 will use a single strip). On the reading - side, PROJ will accept TIFF files with any strip or tile organization. - Tiling is expressed by specifying the TileWidth, TileHeight, TileOffsets - and TileByteCounts tags. Strip organization is expressed by specifying the - RowsPerStrip, StripByteCounts and StripOffsets tags. - -- Files hosted on the CDN will use `Compression - <https://www.awaresystems.be/imaging/tiff/tifftags/compression.html>`_ = DEFLATE - or LZW (to be determined, possibly with - `Predictor <https://www.awaresystems.be/imaging/tiff/tifftags/predictor.html>`_ = 2 - or 3) - On the reading side, PROJ will accept TIFF files with any compression method - (appropriate for the data types and PhotometricInterpretation considered) - supported by the libtiff build used by PROJ. Of course uncompressed files will be supported. - -- Files hosted on the CDN will use little-endian byte ordering. On the reading - side, libtiff will transparently handle both little-endian and big-endian - ordering. - -- Files hosted on the CDN will use PlanarConfiguration=Separate. - The tools described in a later section will order blocks so that blocks needed - for a given location are close to each other. - On the reading side, PROJ will handle also PlanarConfiguration=Contig. - -- Files hosted on the CDN will generally use Float32 (BitsPerSample=32 and SampleFormat=IEEEFP) - Files may be created using Signed Int 16 ( - `BitsPerSample <https://www.awaresystems.be/imaging/tiff/tifftags/bitspersample.html>`_ =16 and - `SampleFormat <https://www.awaresystems.be/imaging/tiff/tifftags/sampleformat.html>`_ = INT), - Unsigned Int 16 (BitsPerSample=16 and SampleFormat=UINT), Signed Int 32 or Unsigned Int 32 generally with an - associate scale/offset. - On the reading side, only those three data types will be supported as well. - -- Files hosted on the CDN will have a `PhotometricInterpretation - <https://www.awaresystems.be/imaging/tiff/tifftags/photometricinterpretation.html>`_ = MinIsBlack. - It will be assumed, and ignored on the reading side. - -- Files hosted on the CDN will nominally have: - - * `SamplesPerPixel <https://www.awaresystems.be/imaging/tiff/tifftags/samplesperpixel.html>`_ = 2 - for horizontal shift grid, with the first sample being the longitude offset - and the second sample being the latitude offset. - - * SamplesPerPixel = 1 for vertical shift grids. - - In the future, different values of SamplesPerPixel may be used to accomodate - for other needs. For example for deformation models, SamplesPerPixel = 3 to combine - horizontal and vertical adjustments. - And even for the current identified needs of horizontal or vertical shifts, - more samples may be present (to indicate for example uncertainties), but - will be ignored by PROJ. - - The `ExtraSamples <https://www.awaresystems.be/imaging/tiff/tifftags/extrasamples.html>`_ - tag should be set to a value of SamplesPerPixel - 1 (given the rules that - apply for PhotometricInterpretation = MinIsBlack) - -- The `ImageDescription <https://www.awaresystems.be/imaging/tiff/tifftags/imagedescription.html>`_ - tag may be used to convey extra information about the name, provenance, version - and last updated date of the grid. - Will be set when possible fo files hosted on the CDN. - Ignored by PROJ. - -- The `Copyright <https://www.awaresystems.be/imaging/tiff/tifftags/copyright.html>`_ - tag may be used to convey extra information about the copyright and license of the grid. - Will be set when possible fo files hosted on the CDN. - Ignored by PROJ. - -- The `DateTime <https://www.awaresystems.be/imaging/tiff/tifftags/datetime.html>`_ - tag may be used to convey the date at which the file has been created or - converted. In case of a file conversion, for example from NTv2, this will be - the date at which the conversion has been performed. The ``ImageDescription`` - tag however will contain the latest of the CREATED or UPDATED fields from the NTv2 file. - Will be set when possible fo files hosted on the CDN. - Ignored by PROJ. - -- Files hosted on the CDN will use the `GDAL_NODATA - <https://www.awaresystems.be/imaging/tiff/tifftags/gdal_nodata.html>`_ tag to encode - the value of the nodata / missing value, when it applies to the grid. - - If offset and/or scaling is used, the nodata value corresponds to the raw value, - before applying offset and scaling. - The value found in this tag, if present, will be honoured (to the extent to - which current PROJ code makes use of nodata). - For floating point data, writers are strongly discouraged to use non-finite values - (+/- infinity, NaN) of nodata to maximimize interoperability. - The GDAL_NODATA value applies to all samples of a given TIFF IFD. - -- Files hosted on the CDN will use the `GDAL_METADATA - <https://www.awaresystems.be/imaging/tiff/tifftags/gdal_metadata.html>`_ tag to encode extra - metadata not supported by baseline or extended TIFF. - - * The root XML node should be ``GDALMetadata`` - - * Zero, one or several child XML nodes ``Item`` may be present. - - * A Item should have a ``name`` attribute, and a child text node with its value. - ``role`` and ``sample`` attributes may be present for attributes that have - a special semantics (recognized by GDAL). The value of `sample` should be - a integer value between 0 and number_of_samples - 1. - - * Scale and offset to convert integer raw values to floating point values - may be expressed with XML `Item` elements whose name attribute is respectively - ``SCALE`` and ``OFFSET``, and their ``role`` attribute is respectively ``scale`` - and ``offset``. The decoded value will be: {offset} + {scale} * raw_value_from_geotiff_file - - For a offset value of 1 and scaling of 2, the following payload should be - stored: - - .. code-block:: xml - - <GDALMetadata> - <Item name="OFFSET" sample="0" role="offset">1</Item> - <Item name="SCALE" sample="0" role="scale">2</Item> - </GDALMetadata> - - * The type of the grid must be specified with a `Item` whose ``name`` is set - to ``TYPE``. - - Values recognized by PROJ currently are: - - - ``HORIZONTAL_OFFSET``: implies the presence of at least two samples. - The first sample must contain the latitude offset and the second - sample must contain the longitude offset. - Corresponds to PROJ ``hgridshift`` method. - - - ``VERTICAL_OFFSET_GEOGRAPHIC_TO_VERTICAL``: implies the presence of at least one sample. - The first sample must contain the vertical adjustment. Must be used when - the source/interpolation CRS is a Geographic CRS and the target CRS a Vertical CRS. - Corresponds to PROJ ``vgridshift`` method. - - - ``VERTICAL_OFFSET_VERTICAL_TO_VERTICAL``: implies the presence of at least one sample. - The first sample must contain the vertical adjustment. Must be used when - the source and target CRS are Vertical CRS. - Corresponds to PROJ ``vgridshift`` method. - - - ``GEOCENTRIC_TRANSLATION``: implies the presence of at least 3 samples. - The first 3 samples must be respectively the geocentric adjustments along - the X, Y and Z axis. Must be used when the source and target CRS are - geocentric CRS. The interpolation CRS must be a geographic CRS. - Corresponds to PROJ ``xyzgridshift`` method. - - - ``VELOCITY``: implies the presence of at least 3 samples. - The first 3 samples must be respectively the velocities along - the E(ast), N(orth), U(p) axis in the local topocentric coordinate system. - Corresponds to PROJ ``deformation`` method. - - For example: - - .. code-block:: xml - - <Item name="TYPE">HORIZONTAL_OFFSET</Item> - - * The description of each sample must be specified with a Item whose ``name`` - attribute is set to ``DESCRIPTION`` and ``role`` attribute to ``description``. - - Values recognized by PROJ for this Item are currently: - - + ``latitude_offset``: valid for TYPE=HORIZONTAL_OFFSET. Sample values should be - the value to add a latitude expressed in the CRS encoded in the GeoKeys - to obtain a latitude value expressed in the target CRS. - - + ``longitude_offset``: valid for TYPE=HORIZONTAL_OFFSET. Sample values should be - the value to add a longitude expressed in the CRS encoded in the GeoKeys - to obtain a longitude value expressed in the target CRS. - - + ``geoid_undulation``: valid for TYPE=VERTICAL_OFFSET_GEOGRAPHIC_TO_VERTICAL. - For a source CRS being a geographic CRS and a target CRS being a vertical CRS, - sample values should be the value to add to a geoid-related height (that - is expressed in the target CRS) to - get an ellipsoidal height (that is expressed in the source CRS), also - called the geoid undulation. - Note the possible confusion related to what is the source CRS and target CRS and - the semantics of the value stored (to convert from the source to the target, - one must subtract the value contained in the grid). This is the convention - used by the `EPSG:9665 <https://www.epsg-registry.org/export.htm?gml=urn:ogc:def:method:EPSG::9665>`_ - operation method. - - + ``vertical_offset``: valid for TYPE=VERTICAL_OFFSET_VERTICAL_TO_VERTICAL. - For a source and target CRS being vertical CRS, - sample values should be the value to add to an elevation expressed in the - source CRS to obtain a longitude value expressed in the target CRS. - - + ``x_translation`` / ``y_translation`` / ``z_translation``: valid for - TYPE=GEOCENTRIC_TRANSLATION. - Sample values should be the value to add to the input geocentric coordinates - expressed in the source CRS to geocentric coordinates expressed in the target CRS. - - + ``east_velocity`` / ``north_velocity`` / ``up_velocity``: valid for - TYPE=VELOCITY. - Sample values should be the velocity in a linear/time unit in a ENU local - topocentric coordinate system. - - For example: - - .. code-block:: xml - - <Item name="DESCRIPTION" sample="0" role="description">latitude_offset</Item> - <Item name="DESCRIPTION" sample="1" role="description">longitude_offset</Item> - - Other values may be used (not used by PROJ): - - + ``latitude_offset_accuracy``: valid for TYPE=HORIZONTAL_OFFSET. Sample values should be - the accuracy of corresponding latitude_offset samples. Generally in metre (if converted from NTv2) - - + ``longitude_offset_accuracy``: valid for TYPE=HORIZONTAL_OFFSET. Sample values should be - the accuracy of corresponding longitude_offset samples. Generally in metre (if converted from NTv2) - - * The sign convention for the values of the ``longitude_offset`` channel - should be indicated with an Item named ``positive_value`` whose value - can be ``west`` or ``east``. NTv2 products originally use a ``west`` - convention, but when converting from them to GeoTIFF, the sign of those - samples will be inverted so they use a more natural ``east`` convention. - If this item is absent, the default value is ``east``. - - * The unit of the values stored in the grid must be specified for each - sample through an Item of name ``UNITTYPE`` and role ``unittype`` - Valid values should be the name of entries from the EPSG ``unitofmeasure`` - table. To maximize interoperability, writers are strongly encouraged to - limit themselves to the following values: - - For linear units: - - - ``metre`` (default value assumed if absent for vertical shift grid files, and value used for files stored on PROJ CDN) - - ``US survey foot`` - - For angular units: - - - ``degree`` - - ``arc-second`` (default value assumed if absent for longitude and latitude offset samples of horizontal shift grid files, and value used for files stored on PROJ CDN) - - For velocity units: - - - ``millimetres per year`` - - The longitude and latitude offset samples should use the same unit. - The geocentric translation samples should use the same unit. - The velocity samples should use the same unit. - - Example: - - .. code-block:: xml - - <Item name="UNITTYPE" sample="0" role="unittype">arc-second</Item> - <Item name="UNITTYPE" sample="1" role="unittype">arc-second</Item> - - * The ``target_crs_epsg_code`` metadata item should be present. - For a horizontal shift grid, this is the EPSG - code of the target geographic CRS. For a vertical shift grid, this is the - EPSG code of a the target vertical CRS. - If the target CRS has no associated EPSG code, ``target_crs_wkt`` must be - used. - Ignored by PROJ currently. - - * The ``target_crs_wkt`` metadata item must be present if the - ``target_crs_epsg_code`` cannot be used. - Its value should be a valid WKT string according to - `WKT:2015 <http://docs.opengeospatial.org/is/12-063r5/12-063r5.html>`_ - or `WKT:2019 <hhttp://docs.opengeospatial.org/is/18-010r7/18-010r7.html>`_ - Ignored by PROJ currently. - - * The ``source_crs_epsg_code`` metadata item must be present if the source - and interpolation CRS are not the same (typical use case is vertical CRS to vertical CRS - transformation), because the GeoKeys encode the interpolation CRS and not the source CRS. - If the source CRS has no associated EPSG code, ``source_crs_wkt`` must be - used. - Ignored by PROJ currently. - - * The ``source_crs_wkt`` metadata item must be present if the - ``source_crs_epsg_code`` cannot be used. - Its value should be a valid WKT string according to WKT:2015 or WKT:2019. - Ignored by PROJ currently. - - * The ``interpolation_crs_wkt`` metadata item may be present if the GeoKeys - cannot be used to express reliably the interpolation CRS. - Its value should be a valid WKT string according to WKT:2015 or WKT:2019. - Ignored by PROJ currently. - - * The ``recommended_interpolation_method`` metadata item may be present to - describe the method to use to interpolation values at locations not - coincident with nodes stored in the grid file. Potential values: ``bilinear``, - ``bicubic``. - Ignored by PROJ currently. - - * The ``area_of_use`` metadata item can be used to indicate plain text information - about the area of use of the grid (like "USA - Wisconsin"). In case of multiple - subgrids, it should be set only on the first one, but applies to the whole - set of grids, not just the first one. - - * The ``grid_name`` metadata item should be present if there are - subgrids for this grid (that is grids whose extent is contained in the extent - of this grid), or if this is a subgrid. - It is intended to be a relatively short identifier - Will be ignored by PROJ (this information can be inferred by the grids extent) - - * The ``parent_grid_name`` metadata item should be present if this is a - subgrid and its value should be equal to the paren's ``grid_name`` - Will be ignored by PROJ (this information can be inferred by the grids extent) - - * The ``number_of_nested_grids`` metadata item should be present if there are - subgrids for this grid (that is grids whose extent is contained in the extent - of this grid). - Will be ignored by PROJ (this information can be inferred by the grids extent) - -Example -+++++++ - -https://github.com/rouault/sample_proj_gtiff_grids/blob/master/ntf_r93.tif has -been converted from https://github.com/OSGeo/proj-datumgrid/blob/master/ntf_r93.gsb -with https://github.com/rouault/sample_proj_gtiff_grids/blob/master/ntv2_to_gtiff.py - -:: - - $ tiffinfo ntf_r93.tif - - TIFF Directory at offset 0x4e (78) - Image Width: 156 Image Length: 111 - Bits/Sample: 32 - Sample Format: IEEE floating point - Compression Scheme: AdobeDeflate - Photometric Interpretation: min-is-black - Extra Samples: 3<unspecified, unspecified, unspecified> - Samples/Pixel: 4 - Rows/Strip: 111 - Planar Configuration: separate image planes - ImageDescription: NTF (EPSG:4275) to RGF93 (EPSG:4171). Converted from ntf_r93.gsb (version IGN07_01, last updated on 2007-10-31) - DateTime: 2019:12:09 00:00:00 - Copyright: Derived from work by IGN France. Open License https://www.etalab.gouv.fr/wp-content/uploads/2014/05/Open_Licence.pdf - Tag 33550: 0.100000,0.100000,0.000000 - Tag 33922: 0.000000,0.000000,0.000000,-5.500000,52.000000,0.000000 - Tag 34735: 1,1,1,3,1024,0,1,2,1025,0,1,2,2048,0,1,4275 - Tag 42112: <GDALMetadata> - <Item name="grid_name">FRANCE</Item> - <Item name="target_crs_epsg_code">4171</Item> - <Item name="TYPE">HORIZONTAL_OFFSET</Item> - <Item name="UNITTYPE" sample="0" role="unittype">arc-second</Item> - <Item name="DESCRIPTION" sample="0" role="description">latitude_offset</Item> - <Item name="positive_value" sample="1">east</Item> - <Item name="UNITTYPE" sample="1" role="unittype">arc-second</Item> - <Item name="DESCRIPTION" sample="1" role="description">longitude_offset</Item> - <Item name="UNITTYPE" sample="2" role="unittype">arc-second</Item> - <Item name="DESCRIPTION" sample="2" role="description">latitude_offset_accuracy</Item> - <Item name="UNITTYPE" sample="3" role="unittype">arc-second</Item> - <Item name="DESCRIPTION" sample="3" role="description">longitude_offset_accuracy</Item> - </GDALMetadata> - - Predictor: floating point predictor 3 (0x3) - - -:: - - $ listgeo ntf_r93.tif - - Geotiff_Information: - Version: 1 - Key_Revision: 1.1 - Tagged_Information: - ModelTiepointTag (2,3): - 0 0 0 - -5.5 52 0 - ModelPixelScaleTag (1,3): - 0.1 0.1 0 - End_Of_Tags. - Keyed_Information: - GTModelTypeGeoKey (Short,1): ModelTypeGeographic - GTRasterTypeGeoKey (Short,1): RasterPixelIsPoint - GeodeticCRSGeoKey (Short,1): Code-4275 (NTF) - End_Of_Keys. - End_Of_Geotiff. - - GCS: 4275/NTF - Datum: 6275/Nouvelle Triangulation Francaise - Ellipsoid: 7011/Clarke 1880 (IGN) (6378249.20,6356515.00) - Prime Meridian: 8901/Greenwich (0.000000/ 0d 0' 0.00"E) - Projection Linear Units: User-Defined (1.000000m) - - Corner Coordinates: - Upper Left ( 5d30' 0.00"W, 52d 0' 0.00"N) - Lower Left ( 5d30' 0.00"W, 40d54' 0.00"N) - Upper Right ( 10d 6' 0.00"E, 52d 0' 0.00"N) - Lower Right ( 10d 6' 0.00"E, 40d54' 0.00"N) - Center ( 2d18' 0.00"E, 46d27' 0.00"N) - -:: - - $ gdalinfo ntf_r93.tif - - Driver: GTiff/GeoTIFF - Files: ntf_r93.tif - Size is 156, 111 - Coordinate System is: - GEOGCRS["NTF", - DATUM["Nouvelle Triangulation Francaise", - ELLIPSOID["Clarke 1880 (IGN)",6378249.2,293.466021293627, - LENGTHUNIT["metre",1]]], - PRIMEM["Greenwich",0, - ANGLEUNIT["degree",0.0174532925199433]], - CS[ellipsoidal,2], - AXIS["geodetic latitude (Lat)",north, - ORDER[1], - ANGLEUNIT["degree",0.0174532925199433]], - AXIS["geodetic longitude (Lon)",east, - ORDER[2], - ANGLEUNIT["degree",0.0174532925199433]], - ID["EPSG",4275]] - Data axis to CRS axis mapping: 2,1 - Origin = (-5.550000000000000,52.049999999999997) - Pixel Size = (0.100000000000000,-0.100000000000000) - Metadata: - AREA_OR_POINT=Point - grid_name=FRANCE - target_crs_epsg_code=4171 - TIFFTAG_DATETIME=2019:12:09 00:00:00 - TIFFTAG_IMAGEDESCRIPTION=NTF (EPSG:4275) to RGF93 (EPSG:4171). Converted from ntf_r93.gsb (version IGN07_01, last updated on 2007-10-31) - TYPE=HORIZONTAL_OFFSET - Image Structure Metadata: - COMPRESSION=DEFLATE - INTERLEAVE=BAND - Corner Coordinates: - Upper Left ( -5.5500000, 52.0500000) ( 5d33' 0.00"W, 52d 3' 0.00"N) - Lower Left ( -5.5500000, 40.9500000) ( 5d33' 0.00"W, 40d57' 0.00"N) - Upper Right ( 10.0500000, 52.0500000) ( 10d 3' 0.00"E, 52d 3' 0.00"N) - Lower Right ( 10.0500000, 40.9500000) ( 10d 3' 0.00"E, 40d57' 0.00"N) - Center ( 2.2500000, 46.5000000) ( 2d15' 0.00"E, 46d30' 0.00"N) - Band 1 Block=156x111 Type=Float32, ColorInterp=Gray - Description = latitude_offset - Unit Type: arc-second - Band 2 Block=156x111 Type=Float32, ColorInterp=Undefined - Description = longitude_offset - Unit Type: arc-second - Metadata: - positive_value=east - Band 3 Block=156x111 Type=Float32, ColorInterp=Undefined - Description = latitude_offset_accuracy - Unit Type: arc-second - Band 4 Block=156x111 Type=Float32, ColorInterp=Undefined - Description = longitude_offset_accuracy - Unit Type: arc-second - -Multi-grid storage +Format description ++++++++++++++++++ -Formats like NTv2 can contain multiple subgrids. This can be transposed to -TIFF by using several IFD chained together with the last 4 bytes (or 8 bytes -for BigTIFF) of an IFD pointing to the offset of the next one. - -The first IFD should have a full description according to the -:ref:`Description of the PROJ GeoTIFF format <description_geotiff_format>`. -Subsequent IFD might have a more compact description, omitting for example, CRS -information if it is identical to the main IFD (which should be the case for -the currently envisionned use cases), or Copyright / ImageDescription metadata -items. - -Each IFD will have its -`NewSubfileType <https://www.awaresystems.be/imaging/tiff/tifftags/newsubfiletype.html>`_ -tag set to 0. - -If a low-resolution grid is available, it should be put before subgrids of -higher-resolution in the chain of IFD linking. On reading, PROJ will use the -value from the highest-resoluted grid that contains the point of interest. - -For efficient reading from the network, files hosted on the CDN will use -a layout similar to the one described in the `low level paragraph of the Cloud Optimized GeoTIFF -GDAL driver page <https://gdal.org/drivers/raster/cog.html#low-level>`_ - -The layout for a file converted from NTv2 will for example be: - -- TIFF/BigTIFF header/signature and pointer to first IFD (Image File Directory) -- "ghost area" indicating the generated process -- IFD of the first grid, followed by TIFF tags values, excluding the TileOffsets and TileByteCounts arrays -- ... -- IFD of the last grid, followed by TIFF tags values, excluding the GDAL_METADATA tag, TileOffsets and TileByteCounts arrays -- TileOffsets and TileByteCounts arrays for first IFD -- ... -- TileOffsets and TileByteCounts arrays for last IFD -- Value of GDAL_METADATA tag for IFDs following the first IFD -- First IFD: Data corresponding to latitude offset of Block_0_0 -- First IFD: Data corresponding to longitude offset of Block_0_0 -- First IFD: Data corresponding to latitude offset of Block_0_1 -- First IFD: Data corresponding to longitude offset of Block_0_1 -- ... -- First IFD: Data corresponding to latitude offset of Block_n_m -- First IFD: Data corresponding to longitude offset of Block_n_m -- ... -- Last IFD: Data corresponding to latitude offset of Block_0_0 -- Last IFD: Data corresponding to longitude offset of Block_0_0 -- Last IFD: Data corresponding to latitude offset of Block_0_1 -- Last IFD: Data corresponding to longitude offset of Block_0_1 -- ... -- Last IFD: Data corresponding to latitude offset of Block_n_m -- Last IFD: Data corresponding to longitude offset of Block_n_m - -If longitude_offset_accuracy and latitude_offset_accuracy are present, this -will be followed by: - -- First IFD: Data corresponding to latitude offset accuracy of Block_0_0 -- First IFD: Data corresponding to longitude offset accuracy of Block_0_0 -- ... -- First IFD: Data corresponding to latitude offset accuracy of Block_n_m -- First IFD: Data corresponding to longitude offset accuracy of Block_n_m -- ... -- Last IFD: Data corresponding to latitude offset accuracy of Block_0_0 -- Last IFD: Data corresponding to longitude offset accuracy of Block_0_0 -- ... -- Last IFD: Data corresponding to latitude offset accuracy of Block_n_m -- Last IFD: Data corresponding to longitude offset accuracy of Block_n_m - -.. note:: - - TIFF has another mechanism to link IFDs, the SubIFD tag. This potentially - enables to define a hiearchy of IFDs (similar to HDF5 groups). There is no - support for that in most TIFF-using software, notably GDAL, and no compelling - need to have a nested hiearchy, so "flat" organization with the standard IFD chaining - mechanism is adopted. - -Examples of multi-grid dataset -++++++++++++++++++++++++++++++ - -https://github.com/rouault/sample_proj_gtiff_grids/blob/master/GDA94_GDA2020_conformal.tif has -been converted from https://github.com/OSGeo/proj-datumgrid/blob/master/oceania/GDA94_GDA2020_conformal.gsb -with https://github.com/rouault/sample_proj_gtiff_grids/blob/master/ntv2_to_gtiff.py - -It contains 5 subgrids. All essential metadata to list the subgrids and their -georeferencing is contained within the first 3 KB of the file. - -The file size is 4.8 MB using DEFLATE compression and floating-point predictor. -To be compared with the 83 MB of the original .gsb file. - -https://github.com/rouault/sample_proj_gtiff_grids/blob/master/ntv2_0.tif has -been converted from https://github.com/OSGeo/proj-datumgrid/blob/master/north-america/ntv2_0.gsb - -It contains 114 subgrids. All essential metadata to list the subgrids and their -georeferencing is contained within the first 40 KB of the file. - +The format description is available in a dedicated :ref:`geodetictiffgrids` +document. Tooling +++++++ diff --git a/docs/source/conf.py b/docs/source/conf.py index 95458917..f2fa3961 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -376,7 +376,7 @@ texinfo_documents = [ #texinfo_no_detailmenu = False breathe_projects = { - "cpp_stuff":"../build/xml/", + "doxygen_api":"../build/xml/", } import redirects diff --git a/docs/source/development/reference/cpp/common.rst b/docs/source/development/reference/cpp/common.rst index c1a28d37..4e92f6b2 100644 --- a/docs/source/development/reference/cpp/common.rst +++ b/docs/source/development/reference/cpp/common.rst @@ -4,5 +4,5 @@ common namespace ---------------- .. doxygennamespace:: osgeo::proj::common - :project: cpp_stuff + :project: doxygen_api :members: diff --git a/docs/source/development/reference/cpp/cpp_general.rst b/docs/source/development/reference/cpp/cpp_general.rst index 3b47d01e..d54daaa8 100644 --- a/docs/source/development/reference/cpp/cpp_general.rst +++ b/docs/source/development/reference/cpp/cpp_general.rst @@ -4,4 +4,4 @@ General documentation --------------------- .. doxygenfile:: general_doc.dox.reworked.h - :project: cpp_stuff + :project: doxygen_api diff --git a/docs/source/development/reference/cpp/crs.rst b/docs/source/development/reference/cpp/crs.rst index 2abea378..f8df118e 100644 --- a/docs/source/development/reference/cpp/crs.rst +++ b/docs/source/development/reference/cpp/crs.rst @@ -4,5 +4,5 @@ crs namespace ------------- .. doxygennamespace:: osgeo::proj::crs - :project: cpp_stuff + :project: doxygen_api :members: diff --git a/docs/source/development/reference/cpp/cs.rst b/docs/source/development/reference/cpp/cs.rst index b168213f..376ae6a6 100644 --- a/docs/source/development/reference/cpp/cs.rst +++ b/docs/source/development/reference/cpp/cs.rst @@ -4,5 +4,5 @@ cs namespace ------------ .. doxygennamespace:: osgeo::proj::cs - :project: cpp_stuff + :project: doxygen_api :members: diff --git a/docs/source/development/reference/cpp/datum.rst b/docs/source/development/reference/cpp/datum.rst index 1fee5f8a..0a545a68 100644 --- a/docs/source/development/reference/cpp/datum.rst +++ b/docs/source/development/reference/cpp/datum.rst @@ -4,5 +4,5 @@ datum namespace --------------- .. doxygennamespace:: osgeo::proj::datum - :project: cpp_stuff + :project: doxygen_api :members: diff --git a/docs/source/development/reference/cpp/io.rst b/docs/source/development/reference/cpp/io.rst index 9da0f680..9729af90 100644 --- a/docs/source/development/reference/cpp/io.rst +++ b/docs/source/development/reference/cpp/io.rst @@ -4,5 +4,5 @@ io namespace ------------ .. doxygennamespace:: osgeo::proj::io - :project: cpp_stuff + :project: doxygen_api :members: diff --git a/docs/source/development/reference/cpp/metadata.rst b/docs/source/development/reference/cpp/metadata.rst index b3960122..348627e8 100644 --- a/docs/source/development/reference/cpp/metadata.rst +++ b/docs/source/development/reference/cpp/metadata.rst @@ -4,5 +4,5 @@ metadata namespace ------------------ .. doxygennamespace:: osgeo::proj::metadata - :project: cpp_stuff + :project: doxygen_api :members: diff --git a/docs/source/development/reference/cpp/operation.rst b/docs/source/development/reference/cpp/operation.rst index bc3f201d..88096cb8 100644 --- a/docs/source/development/reference/cpp/operation.rst +++ b/docs/source/development/reference/cpp/operation.rst @@ -4,5 +4,5 @@ operation namespace ------------------- .. doxygennamespace:: osgeo::proj::operation - :project: cpp_stuff + :project: doxygen_api :members: diff --git a/docs/source/development/reference/cpp/util.rst b/docs/source/development/reference/cpp/util.rst index 1aba2954..bc9676a4 100644 --- a/docs/source/development/reference/cpp/util.rst +++ b/docs/source/development/reference/cpp/util.rst @@ -4,5 +4,5 @@ util namespace -------------- .. doxygennamespace:: osgeo::proj::util - :project: cpp_stuff + :project: doxygen_api :members: diff --git a/docs/source/development/reference/datatypes.rst b/docs/source/development/reference/datatypes.rst index 18bd7efd..7ec17eb1 100644 --- a/docs/source/development/reference/datatypes.rst +++ b/docs/source/development/reference/datatypes.rst @@ -797,11 +797,48 @@ Logging .. versionadded:: 5.1.0 +Setting custom I/O functions +------------------------------------------------------------------------------- + +.. versionadded:: 7.0.0 + +.. doxygenstruct:: PROJ_FILE_API + :project: doxygen_api + :members: + +.. doxygentypedef:: PROJ_FILE_HANDLE + :project: doxygen_api + +.. doxygenenum:: PROJ_OPEN_ACCESS + :project: doxygen_api + + +Network related functionality +------------------------------------------------------------------------------- + +.. versionadded:: 7.0.0 + +.. doxygentypedef:: PROJ_NETWORK_HANDLE + :project: doxygen_api + +.. doxygentypedef:: proj_network_open_cbk_type + :project: doxygen_api + +.. doxygentypedef:: proj_network_close_cbk_type + :project: doxygen_api + +.. doxygentypedef:: proj_network_get_header_value_cbk_type + :project: doxygen_api + +.. doxygentypedef:: proj_network_read_range_type + :project: doxygen_api + + C API for ISO-19111 functionality ------------------------------------------------------------------------------- .. doxygengroup:: iso19111_types - :project: cpp_stuff + :project: doxygen_api :content-only: diff --git a/docs/source/development/reference/functions.rst b/docs/source/development/reference/functions.rst index c010fd13..a2a0c3ae 100644 --- a/docs/source/development/reference/functions.rst +++ b/docs/source/development/reference/functions.rst @@ -697,6 +697,54 @@ Various :returns: :c:type:`int` 1 if output units is expected in radians, otherwise 0 +Setting custom I/O functions +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. versionadded:: 7.0.0 + +.. doxygenfunction:: proj_context_set_fileapi + :project: doxygen_api + +.. doxygenfunction:: proj_context_set_sqlite3_vfs_name + :project: doxygen_api + + +Network related functionality +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. versionadded:: 7.0.0 + +.. doxygenfunction:: proj_context_set_network_callbacks + :project: doxygen_api + +.. doxygenfunction:: proj_context_set_enable_network + :project: doxygen_api + +.. doxygenfunction:: proj_context_set_url_endpoint + :project: doxygen_api + +.. doxygenfunction:: proj_grid_cache_set_enable + :project: doxygen_api + +.. doxygenfunction:: proj_grid_cache_set_filename + :project: doxygen_api + +.. doxygenfunction:: proj_grid_cache_set_max_size + :project: doxygen_api + +.. doxygenfunction:: proj_grid_cache_set_ttl + :project: doxygen_api + +.. doxygenfunction:: proj_grid_cache_clear + :project: doxygen_api + +.. doxygenfunction:: proj_is_download_needed + :project: doxygen_api + +.. doxygenfunction:: proj_download_file + :project: doxygen_api + + Cleanup ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -727,6 +775,6 @@ which are not of type CRS (can be tested with :c:func:`proj_is_crs`), will return an error when used with functions of this section. .. doxygengroup:: iso19111_functions - :project: cpp_stuff + :project: doxygen_api :content-only: diff --git a/docs/source/index.rst b/docs/source/index.rst index e9590d99..3a2df4b4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -17,6 +17,7 @@ PROJ resource_files geodesic development/index + specifications/index community/index faq glossary diff --git a/docs/source/operations/transformations/deformation.rst b/docs/source/operations/transformations/deformation.rst index 3a9d025c..02924a25 100644 --- a/docs/source/operations/transformations/deformation.rst +++ b/docs/source/operations/transformations/deformation.rst @@ -35,6 +35,9 @@ GTX format. Both grids are expected to contain grid-values in units of mm/year. GDAL both reads and writes both file formats. Using GDAL for construction of new grids is recommended. +Starting with PROJ 7.0, use of a GeoTIFF format is recommended to store both +the horizontal and vertical velocities. + Example ------------------------------------------------------------------------------- @@ -89,13 +92,15 @@ Parameters .. option:: +xy_grids=<list> - Comma-separated list of grids to load. If a grid is prefixed by an `@` the + Comma-separated list of grids to load. If a grid is prefixed by an ``@`` the grid is considered optional and PROJ will the not complain if the grid is not available. - Grids for the horizontla component of a deformation model is expected to be + Grids for the horizontal component of a deformation model is expected to be in CTable2 format. + .. note:: :option:`+xy_grids` is mutually exclusive with :option:`+grids` + .. option:: +z_grids=<list> Comma-separated list of grids to load. If a grid is prefixed by an `@` the @@ -105,6 +110,36 @@ Parameters Grids for the vertical component of a deformation model is expected to be in either GTX format. + .. note:: :option:`+z_grids` is mutually exclusive with :option:`+grids` + +.. option:: +grids=<list> + + .. versionadded:: 7.0.0 + + Comma-separated list of grids to load. If a grid is prefixed by an `@` the + grid is considered optional and PROJ will the not complain if the grid is + not available. + + Grids should be in GeoTIFF format with the first 3 components being + respectively the easting, northing and up velocities in mm/year. + Setting the Description and Unit Type GDAL band metadata items is strongly + recommended, so that gdalinfo reports: + + :: + + Band 1 Block=... Type=Float32, ColorInterp=Gray + Description = east_velocity + Unit Type: mm/year + Band 2 Block=... Type=Float32, ColorInterp=Undefined + Description = north_velocity + Unit Type: mm/year + Band 3 Block=... Type=Float32, ColorInterp=Undefined + Description = up_velocity + Unit Type: mm/year + + .. note:: :option:`+grids` is mutually exclusive with :option:`+xy_grids` + and :option:`+z_grids` + .. option:: +t_epoch=<value> Central epoch of transformation given in decimalyears. Will be used in diff --git a/docs/source/operations/transformations/index.rst b/docs/source/operations/transformations/index.rst index d4bbf4e0..6bd503d4 100644 --- a/docs/source/operations/transformations/index.rst +++ b/docs/source/operations/transformations/index.rst @@ -19,3 +19,4 @@ systems are based on different datums. molobadekas hgridshift vgridshift + xyzgridshift diff --git a/docs/source/operations/transformations/xyzgridshift.rst b/docs/source/operations/transformations/xyzgridshift.rst new file mode 100644 index 00000000..30537032 --- /dev/null +++ b/docs/source/operations/transformations/xyzgridshift.rst @@ -0,0 +1,91 @@ +.. _xyzgridshift: + +================================================================================ +Geocentric grid shift +================================================================================ + +.. versionadded:: 7.0.0 + +Geocentric translation using a grid shift + ++-----------------+-------------------------------------------------------------------+ +| **Domain** | 3D | ++-----------------+-------------------------------------------------------------------+ +| **Input type** | Cartesian coordinates | ++-----------------+-------------------------------------------------------------------+ +| **Output type** | Cartesian coordinates | ++-----------------+-------------------------------------------------------------------+ + +Perform a geocentric translation by bilinear interpolation of dx, dy, dz +translation values from a grid. The grid is referenced against either the +2D geographic CRS corresponding to the input (or sometimes output) CRS. + +This method is described (in French) in :cite:`NTF_88` +and as EPSG operation method code 9655 in :cite:`IOGP2018` (§2.4.4.1.1 +France geocentric interpolation). + +The translation in the grids are added to the input coordinates in the forward direction, +and subtracted in the reverse direction. +By default (if ``grid_ref=input_crs``), in the forward direction, the input coordinates +are converted to their geographic equivalent to directly read and interpolate from +the grid. In the reverse direction, an iterative method is used to be able to find +the grid locations to read. +If ``grid_ref=output_crs`` is used, then the reverse strategy is applied: iterative +method in the forward direction, and direct read in the reverse direction. + +Example +------------------------------------------------------------------------------- + +NTF to RGF93 transformation using gr3df97a.tif grid + +:: + + +proj=pipeline + +step +proj=push +v_3 + +step +proj=cart +ellps=clrk80ign + +step +proj=xyzgridshift +grids=gr3df97a.tif +grid_ref=output_crs +ellps=GRS80 + +step +proj=cart +ellps=GRS80 +inv + +step +proj=pop +v_3 + +Parameters +################################################################################ + +The ellipsoid parameters should be the ones consistent with ``grid_ref``. +They are used to perform a geocentric to geographic conversion to find the +translation parameters. + + +Required ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. include:: ../options/ellps.rst + +.. option:: +grids=<list> + + Comma-separated list of grids to load. If a grid is prefixed by an ``@`` the + grid is considered optional and PROJ will the not complain if the grid is + not available. + + Grids are expected to be in GeoTIFF format. If no metadata is provided, + the first, second and third samples are assumed to be the geocentric + translation along X, Y and Z axis respectively, in metres. + +Optional ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. option:: +grid_ref=input_crs/output_crs + + Specify in which CRS the grid is referenced to. The default value is + input_crs. That is the grid is referenced in the geographic CRS corresponding + to the input geocentric CRS. + + If output_crs is specified, the grid is referenced in the geographic CRS corresponding + to the output geocentric CRS. This is for example the case for the French + gr3df97a.tif grid converting from NTF to RGF93, but referenced against RGF93. + Thus in the forward direction (NTF->RGF93), an iterative method is used to find + the appropriate shift. + +.. option:: +multiplier=<value> + + Specify the multiplier to apply to the grid values. Defaults to 1.0 + diff --git a/docs/source/references.bib b/docs/source/references.bib index 529c20e1..13853764 100644 --- a/docs/source/references.bib +++ b/docs/source/references.bib @@ -229,6 +229,15 @@ Doi = {10.2312/PE/PG/PG2012short/005-010} } +@TechReport{NTF_88, + Title = {Grille de parametres de transformation de coordonnees - {GR3DF97A} - Notice d'utilisation}, + Author = {IGN}, + Institution = {Service de Geodesie et Nivellement, Institut Geographique National}, + Year = {1997}, + + Url = {https://geodesie.ign.fr/contenu/fichiers/documentation/algorithmes/notice/NTG_88.pdf} +} + @TechReport{ONeilLaubscher1976, Title = {Extended Studies of a Quadrilateralized Spherical Cube Earth Data Base}, Author = {E. M. O'Neill and R. E. Laubscher}, diff --git a/docs/source/resource_files.rst b/docs/source/resource_files.rst index 7a0ede63..28628d30 100644 --- a/docs/source/resource_files.rst +++ b/docs/source/resource_files.rst @@ -14,6 +14,8 @@ In addition to the bundled init-files the PROJ project also distributes a number of packages containing transformation grids and additional init-files not included in the main PROJ package. +.. _resource_file_paths: + Where are PROJ resource files looked for ? ------------------------------------------------------------------------------- @@ -23,23 +25,43 @@ The following paths are checked in order: - For transformation grids that have an explict relative or absolute path, the directory specified in the grid filename. + - Path resolved by the callback function set with the :c:func:`proj_context_set_file_finder`. If it is set, the next tests will not be run. + - Path(s) set with the :c:func:`proj_context_set_search_paths`. If set, the next tests will not be run. + +.. _user_writable_directory: + +- The PROJ user writable directory, which is : + + * on Windows, ${LOCALAPPDATA}/proj + * on MacOSX, ${HOME}/Library/Application Support/proj + * on other platforms (Linux), ${XDG_DATA_HOME}/proj if :envvar:`XDG_DATA_HOME` + is defined. Else ${HOME}/.local/share/proj + - Path(s) set with by the environment variable :envvar:`PROJ_LIB`. On Linux/MacOSX/Unix, use ``:`` to separate paths. On Windows, ``;`` + - On Windows, the *..\\share\\proj\\* and its contents are found automatically at run-time if the installation respects the build structure. That is, the binaries and proj.dll are installed under *..\\bin\\*, and resource files are in *..\\share\\proj\\*. + - A path built into PROJ as its resource installation directory (whose value is $(pkgdatadir)), for builds using the Makefile build system. Note, however, that since this is a hard-wired path setting, it only works if the whole PROJ installation is not moved somewhere else. + - The current directory +When networking capabilities are enabled, either by API with the +:c:func:`proj_context_set_enable_network` function or when the +:envvar:`PROJ_NETWORK` environment variable is set to ``ON``, PROJ will +attempt to use remote grids stored on CDN (Content Delivery Network) storage. + .. _proj-db: proj.db @@ -49,6 +71,38 @@ A proj installation includes a SQLite database of transformation information that must be accessible for the library to work properly. The library will print an error if the database can't be found. +.. _proj-ini: + +proj.ini +------------------------------------------------------------------------------- + +.. versionadded:: 7.0 + +proj.ini is a text configuration file, mostly dedicated at setting up network +related parameters. + +Its default content is: + +:: + + [general] + ; Lines starting by ; are commented lines. + ; + + ; Network capabilities disabled by default. + ; Can be overriden with the PROJ_NETWORK=ON environment variable. + ; network = on + + ; Can be overriden with the PROJ_NETWORK_ENDPOINT environment variable. + cdn_endpoint = https://cdn.proj.org + + cache_enabled = on + + cache_size_MB = 300 + + cache_ttl_sec = 86400 + + Transformation grids ------------------------------------------------------------------------------- diff --git a/docs/source/specifications/geodetictiffgrids.rst b/docs/source/specifications/geodetictiffgrids.rst new file mode 100644 index 00000000..98dc8408 --- /dev/null +++ b/docs/source/specifications/geodetictiffgrids.rst @@ -0,0 +1,626 @@ +.. _geodetictiffgrids: + +================================================================================ +Geodetic TIFF grids (GTG) +================================================================================ + +.. versionadded:: 7.0 + +Introduction +++++++++++++ + +The Geodetic TIFF grid format has been introduced per :ref:`rfc4`. It is a +profile of the TIFF and GeoTIFF formats that addresses the specific requirements +of geodetic grids: horizontal shifts, vertical shifts, velocity grids, etc... +It also follows the `Cloud Optimized GeoTIFF <http://cogeo.org>`_ principles. + +Such grids are available on a :ref:`CDN of GeoTIFF grids <cdn_grids>`. + +.. _gtg_general_description: + +General description ++++++++++++++++++++ + +The general principles that guide the following requirements and recommendations +are such that files will be properly recognized by PROJ, and also by GDAL which +is an easy way to inspect such grid files: + +- `TIFF 6.0 <https://www.awaresystems.be/imaging/tiff/specification/TIFF6.pdf>`_ + based (could possibly be BigTIFF without code changes, if we ever + need some day to handle grids larger than 4GB) + +- `GeoTIFF 1.1 <http://docs.opengeospatial.org/is/19-008r4/19-008r4.html>`_ for the georeferencing. + GeoTIFF 1.1 is a recent standard, compared to the original GeoTIFF 1.0 version, + but its backward compatibility is excellent, so that should not cause much trouble + to readers that are not official GeoTIFF 1.1 compliant. + +- Files hosted on the CDN will use a Geographic 2D CRS for the GeoTIFF GeoKeys. + That CRS is intended to be the interpolation CRS as defined in + `OGC Abstract Specification Topic 2 <http://docs.opengeospatial.org/as/18-005r4/18-005r4.html>`_, + that is the CRS to which grid values are refered to. + + Given that they will nominally be related to the EPSG dataset, the `GeodeticCRSGeoKey + <http://docs.opengeospatial.org/is/19-008r4/19-008r4.html#_requirements_class_geodeticcrsgeokey>`_ + will be used to store the EPSG code of the CRS. If the CRS cannot be reliably + encoded through that key or other geokeys, the ``interpolation_crs_wkt`` metadata + item detailed afterwards should be used. + + This CRS will be generally the source CRS (for geographic to + geographic horizontal shift grids, or geographic to vertical shift grids), but + for vertical to vertical CRS adjustment, this will be the geographic CRS to + which the grid is referenced. In some very rare cases of geographic to vertical + shift grids, the interpolation CRS might be a geographic CRS that is not the + same as the source CRS (into which ellipsoidal height are expressed). The only + instance we have in mind is for the EPSG:7001 "ETRS89 to NAP height (1)" transformation + using the naptrans2008 VDatum-grid which is referenced to Amersfoort EPSG:4289 + instead of ETRS89... + + On the reading side, PROJ will ignore that information: + the CRS is already stored in the source_crs or interpolation_crs column of the + grid_transformation table. + + For geographic to vertical shift files (geoid models), the GeoTIFF 1.1 + convention will be used to store the value of the `VerticalGeoKey + <http://docs.opengeospatial.org/is/19-008r4/19-008r4.html#_requirements_class_verticalgeokey>`_ + So a geoid model that apply to WGS 84 EPSG:4979 will have GeodeticCRSGeoKey = 4326 + and VerticalGeoKey = 4979. + +- Files hosted on the CDN will use the GeoTIFF defined `ModelTiepointTag and ModelPixelScaleTag + <http://docs.opengeospatial.org/is/19-008r4/19-008r4.html#_raster_to_model_coordinate_transformation_requirements>`_ TIFF tags + to store the coordinates of the upper-left pixel and the resolution of the pixels. + On the reading side, they will be required and ModelTransformationTag will be ignored. + + .. note:: + + Regarding anti-meridian handling, a variety of possibilities exist. + We do not attempt to standardize this and filesh hosted on the CDN will use + a georeferencing close to the original data producer. + For example, NOAA vertical grids that apply to Conterminous USA might even have a top-left + longitude beyond 180 (for consistency with Alaska grids, whose origin is < 180) + Anti-meridian handling in PROJ has probably issues. This RFC does not attempt + to address them in particular, as they are believed to be orthogonal to the + topics it covers, and being mostly implementation issues. + +- Files hosted on the CDN will use the `GTRasterTypeGeoKey + <http://docs.opengeospatial.org/is/19-008r4/19-008r4.html#_requirements_class_gtrastertypegeokey>`_ + = PixelIsPoint convention. + This is the convention used by most existing grid formats currently. Note that GDAL + typically use a PixelIsArea convention (but can handle both conventions), so the + georeferencing it displays when opening a .gsb or .gtx file appears to have a + half-pixel shift regarding to the coordinates stored in the original grid file. On + the reading side, PROJ will accept both conventions (for equivalent georeferencing, + the value of the origin in a PixelIsArea convention is shifted by a half-pixel + towards the upper-left direction). Unspecified behaviour if this GeoKey is absent. + +- Files hosted on the CDN will be tiled, presumably with 256x256 tiles (small + grids that are smaller than 256x256 will use a single strip). On the reading + side, PROJ will accept TIFF files with any strip or tile organization. + Tiling is expressed by specifying the TileWidth, TileHeight, TileOffsets + and TileByteCounts tags. Strip organization is expressed by specifying the + RowsPerStrip, StripByteCounts and StripOffsets tags. + +- Files hosted on the CDN will use `Compression + <https://www.awaresystems.be/imaging/tiff/tifftags/compression.html>`_ = DEFLATE + or LZW (to be determined, possibly with + `Predictor <https://www.awaresystems.be/imaging/tiff/tifftags/predictor.html>`_ = 2 + or 3) + On the reading side, PROJ will accept TIFF files with any compression method + (appropriate for the data types and PhotometricInterpretation considered) + supported by the libtiff build used by PROJ. Of course uncompressed files will be supported. + +- Files hosted on the CDN will use little-endian byte ordering. On the reading + side, libtiff will transparently handle both little-endian and big-endian + ordering. + +- Files hosted on the CDN will use PlanarConfiguration=Separate. + The tools described in a later section will order blocks so that blocks needed + for a given location are close to each other. + On the reading side, PROJ will handle also PlanarConfiguration=Contig. + +- Files hosted on the CDN will generally use Float32 (BitsPerSample=32 and SampleFormat=IEEEFP) + Files may be created using Signed Int 16 ( + `BitsPerSample <https://www.awaresystems.be/imaging/tiff/tifftags/bitspersample.html>`_ =16 and + `SampleFormat <https://www.awaresystems.be/imaging/tiff/tifftags/sampleformat.html>`_ = INT), + Unsigned Int 16 (BitsPerSample=16 and SampleFormat=UINT), Signed Int 32 or Unsigned Int 32 generally with an + associate scale/offset. + On the reading side, only those three data types will be supported as well. + +- Files hosted on the CDN will have a `PhotometricInterpretation + <https://www.awaresystems.be/imaging/tiff/tifftags/photometricinterpretation.html>`_ = MinIsBlack. + It will be assumed, and ignored on the reading side. + +- Files hosted on the CDN will nominally have: + + * `SamplesPerPixel <https://www.awaresystems.be/imaging/tiff/tifftags/samplesperpixel.html>`_ = 2 + for horizontal shift grid, with the first sample being the longitude offset + and the second sample being the latitude offset. + + * SamplesPerPixel = 1 for vertical shift grids. + + In the future, different values of SamplesPerPixel may be used to accomodate + for other needs. For example for deformation models, SamplesPerPixel = 3 to combine + horizontal and vertical adjustments. + And even for the current identified needs of horizontal or vertical shifts, + more samples may be present (to indicate for example uncertainties), but + will be ignored by PROJ. + + The `ExtraSamples <https://www.awaresystems.be/imaging/tiff/tifftags/extrasamples.html>`_ + tag should be set to a value of SamplesPerPixel - 1 (given the rules that + apply for PhotometricInterpretation = MinIsBlack) + +- The `ImageDescription <https://www.awaresystems.be/imaging/tiff/tifftags/imagedescription.html>`_ + tag may be used to convey extra information about the name, provenance, version + and last updated date of the grid. + Will be set when possible fo files hosted on the CDN. + Ignored by PROJ. + +- The `Copyright <https://www.awaresystems.be/imaging/tiff/tifftags/copyright.html>`_ + tag may be used to convey extra information about the copyright and license of the grid. + Will be set when possible fo files hosted on the CDN. + Ignored by PROJ. + +- The `DateTime <https://www.awaresystems.be/imaging/tiff/tifftags/datetime.html>`_ + tag may be used to convey the date at which the file has been created or + converted. In case of a file conversion, for example from NTv2, this will be + the date at which the conversion has been performed. The ``ImageDescription`` + tag however will contain the latest of the CREATED or UPDATED fields from the NTv2 file. + Will be set when possible fo files hosted on the CDN. + Ignored by PROJ. + +- Files hosted on the CDN will use the `GDAL_NODATA + <https://www.awaresystems.be/imaging/tiff/tifftags/gdal_nodata.html>`_ tag to encode + the value of the nodata / missing value, when it applies to the grid. + + If offset and/or scaling is used, the nodata value corresponds to the raw value, + before applying offset and scaling. + The value found in this tag, if present, will be honoured (to the extent to + which current PROJ code makes use of nodata). + For floating point data, writers are strongly discouraged to use non-finite values + (+/- infinity, NaN) of nodata to maximimize interoperability. + The GDAL_NODATA value applies to all samples of a given TIFF IFD. + +- Files hosted on the CDN will use the `GDAL_METADATA + <https://www.awaresystems.be/imaging/tiff/tifftags/gdal_metadata.html>`_ tag to encode extra + metadata not supported by baseline or extended TIFF. + + * The root XML node should be ``GDALMetadata`` + + * Zero, one or several child XML nodes ``Item`` may be present. + + * A Item should have a ``name`` attribute, and a child text node with its value. + ``role`` and ``sample`` attributes may be present for attributes that have + a special semantics (recognized by GDAL). The value of `sample` should be + a integer value between 0 and number_of_samples - 1. + + * Scale and offset to convert integer raw values to floating point values + may be expressed with XML `Item` elements whose name attribute is respectively + ``SCALE`` and ``OFFSET``, and their ``role`` attribute is respectively ``scale`` + and ``offset``. The decoded value will be: {offset} + {scale} * raw_value_from_geotiff_file + + For a offset value of 1 and scaling of 2, the following payload should be + stored: + + .. code-block:: xml + + <GDALMetadata> + <Item name="OFFSET" sample="0" role="offset">1</Item> + <Item name="SCALE" sample="0" role="scale">2</Item> + </GDALMetadata> + + * The type of the grid must be specified with a `Item` whose ``name`` is set + to ``TYPE``. + + Values recognized by PROJ currently are: + + - ``HORIZONTAL_OFFSET``: implies the presence of at least two samples. + The first sample must contain the latitude offset and the second + sample must contain the longitude offset. + Corresponds to PROJ ``hgridshift`` method. + + - ``VERTICAL_OFFSET_GEOGRAPHIC_TO_VERTICAL``: implies the presence of at least one sample. + The first sample must contain the vertical adjustment. Must be used when + the source/interpolation CRS is a Geographic CRS and the target CRS a Vertical CRS. + Corresponds to PROJ ``vgridshift`` method. + + - ``VERTICAL_OFFSET_VERTICAL_TO_VERTICAL``: implies the presence of at least one sample. + The first sample must contain the vertical adjustment. Must be used when + the source and target CRS are Vertical CRS. + Corresponds to PROJ ``vgridshift`` method. + + - ``GEOCENTRIC_TRANSLATION``: implies the presence of at least 3 samples. + The first 3 samples must be respectively the geocentric adjustments along + the X, Y and Z axis. Must be used when the source and target CRS are + geocentric CRS. The interpolation CRS must be a geographic CRS. + Corresponds to PROJ ``xyzgridshift`` method. + + - ``VELOCITY``: implies the presence of at least 3 samples. + The first 3 samples must be respectively the velocities along + the E(ast), N(orth), U(p) axis in the local topocentric coordinate system. + Corresponds to PROJ ``deformation`` method. + + For example: + + .. code-block:: xml + + <Item name="TYPE">HORIZONTAL_OFFSET</Item> + + * The description of each sample must be specified with a Item whose ``name`` + attribute is set to ``DESCRIPTION`` and ``role`` attribute to ``description``. + + Values recognized by PROJ for this Item are currently: + + + ``latitude_offset``: valid for TYPE=HORIZONTAL_OFFSET. Sample values should be + the value to add a latitude expressed in the CRS encoded in the GeoKeys + to obtain a latitude value expressed in the target CRS. + + + ``longitude_offset``: valid for TYPE=HORIZONTAL_OFFSET. Sample values should be + the value to add a longitude expressed in the CRS encoded in the GeoKeys + to obtain a longitude value expressed in the target CRS. + + + ``geoid_undulation``: valid for TYPE=VERTICAL_OFFSET_GEOGRAPHIC_TO_VERTICAL. + For a source CRS being a geographic CRS and a target CRS being a vertical CRS, + sample values should be the value to add to a geoid-related height (that + is expressed in the target CRS) to + get an ellipsoidal height (that is expressed in the source CRS), also + called the geoid undulation. + Note the possible confusion related to what is the source CRS and target CRS and + the semantics of the value stored (to convert from the source to the target, + one must subtract the value contained in the grid). This is the convention + used by the `EPSG:9665 <https://www.epsg-registry.org/export.htm?gml=urn:ogc:def:method:EPSG::9665>`_ + operation method. + + + ``vertical_offset``: valid for TYPE=VERTICAL_OFFSET_VERTICAL_TO_VERTICAL. + For a source and target CRS being vertical CRS, + sample values should be the value to add to an elevation expressed in the + source CRS to obtain a longitude value expressed in the target CRS. + + + ``x_translation`` / ``y_translation`` / ``z_translation``: valid for + TYPE=GEOCENTRIC_TRANSLATION. + Sample values should be the value to add to the input geocentric coordinates + expressed in the source CRS to geocentric coordinates expressed in the target CRS. + + + ``east_velocity`` / ``north_velocity`` / ``up_velocity``: valid for + TYPE=VELOCITY. + Sample values should be the velocity in a linear/time unit in a ENU local + topocentric coordinate system. + + For example: + + .. code-block:: xml + + <Item name="DESCRIPTION" sample="0" role="description">latitude_offset</Item> + <Item name="DESCRIPTION" sample="1" role="description">longitude_offset</Item> + + Other values may be used (not used by PROJ): + + + ``latitude_offset_accuracy``: valid for TYPE=HORIZONTAL_OFFSET. Sample values should be + the accuracy of corresponding latitude_offset samples. Generally in metre (if converted from NTv2) + + + ``longitude_offset_accuracy``: valid for TYPE=HORIZONTAL_OFFSET. Sample values should be + the accuracy of corresponding longitude_offset samples. Generally in metre (if converted from NTv2) + + * The sign convention for the values of the ``longitude_offset`` channel + should be indicated with an Item named ``positive_value`` whose value + can be ``west`` or ``east``. NTv2 products originally use a ``west`` + convention, but when converting from them to GeoTIFF, the sign of those + samples will be inverted so they use a more natural ``east`` convention. + If this item is absent, the default value is ``east``. + + * The unit of the values stored in the grid must be specified for each + sample through an Item of name ``UNITTYPE`` and role ``unittype`` + Valid values should be the name of entries from the EPSG ``unitofmeasure`` + table. To maximize interoperability, writers are strongly encouraged to + limit themselves to the following values: + + For linear units: + + - ``metre`` (default value assumed if absent for vertical shift grid files, and value used for files stored on PROJ CDN) + - ``US survey foot`` + + For angular units: + + - ``degree`` + - ``arc-second`` (default value assumed if absent for longitude and latitude offset samples of horizontal shift grid files, and value used for files stored on PROJ CDN) + + For velocity units: + + - ``millimetres per year`` + + The longitude and latitude offset samples should use the same unit. + The geocentric translation samples should use the same unit. + The velocity samples should use the same unit. + + Example: + + .. code-block:: xml + + <Item name="UNITTYPE" sample="0" role="unittype">arc-second</Item> + <Item name="UNITTYPE" sample="1" role="unittype">arc-second</Item> + + * The ``target_crs_epsg_code`` metadata item should be present. + For a horizontal shift grid, this is the EPSG + code of the target geographic CRS. For a vertical shift grid, this is the + EPSG code of a the target vertical CRS. + If the target CRS has no associated EPSG code, ``target_crs_wkt`` must be + used. + Ignored by PROJ currently. + + * The ``target_crs_wkt`` metadata item must be present if the + ``target_crs_epsg_code`` cannot be used. + Its value should be a valid WKT string according to + `WKT:2015 <http://docs.opengeospatial.org/is/12-063r5/12-063r5.html>`_ + or `WKT:2019 <hhttp://docs.opengeospatial.org/is/18-010r7/18-010r7.html>`_ + Ignored by PROJ currently. + + * The ``source_crs_epsg_code`` metadata item must be present if the source + and interpolation CRS are not the same (typical use case is vertical CRS to vertical CRS + transformation), because the GeoKeys encode the interpolation CRS and not the source CRS. + If the source CRS has no associated EPSG code, ``source_crs_wkt`` must be + used. + Ignored by PROJ currently. + + * The ``source_crs_wkt`` metadata item must be present if the + ``source_crs_epsg_code`` cannot be used. + Its value should be a valid WKT string according to WKT:2015 or WKT:2019. + Ignored by PROJ currently. + + * The ``interpolation_crs_wkt`` metadata item may be present if the GeoKeys + cannot be used to express reliably the interpolation CRS. + Its value should be a valid WKT string according to WKT:2015 or WKT:2019. + Ignored by PROJ currently. + + * The ``recommended_interpolation_method`` metadata item may be present to + describe the method to use to interpolation values at locations not + coincident with nodes stored in the grid file. Potential values: ``bilinear``, + ``bicubic``. + Ignored by PROJ currently. + + * The ``area_of_use`` metadata item can be used to indicate plain text information + about the area of use of the grid (like "USA - Wisconsin"). In case of multiple + subgrids, it should be set only on the first one, but applies to the whole + set of grids, not just the first one. + + * The ``grid_name`` metadata item should be present if there are + subgrids for this grid (that is grids whose extent is contained in the extent + of this grid), or if this is a subgrid. + It is intended to be a relatively short identifier + Will be ignored by PROJ (this information can be inferred by the grids extent) + + * The ``parent_grid_name`` metadata item should be present if this is a + subgrid and its value should be equal to the paren's ``grid_name`` + Will be ignored by PROJ (this information can be inferred by the grids extent) + + * The ``number_of_nested_grids`` metadata item should be present if there are + subgrids for this grid (that is grids whose extent is contained in the extent + of this grid). + Will be ignored by PROJ (this information can be inferred by the grids extent) + +Example ++++++++ + +https://github.com/rouault/sample_proj_gtiff_grids/blob/master/ntf_r93.tif has +been converted from https://github.com/OSGeo/proj-datumgrid/blob/master/ntf_r93.gsb +with https://github.com/rouault/sample_proj_gtiff_grids/blob/master/ntv2_to_gtiff.py + +:: + + $ tiffinfo ntf_r93.tif + + TIFF Directory at offset 0x4e (78) + Image Width: 156 Image Length: 111 + Bits/Sample: 32 + Sample Format: IEEE floating point + Compression Scheme: AdobeDeflate + Photometric Interpretation: min-is-black + Extra Samples: 3<unspecified, unspecified, unspecified> + Samples/Pixel: 4 + Rows/Strip: 111 + Planar Configuration: separate image planes + ImageDescription: NTF (EPSG:4275) to RGF93 (EPSG:4171). Converted from ntf_r93.gsb (version IGN07_01, last updated on 2007-10-31) + DateTime: 2019:12:09 00:00:00 + Copyright: Derived from work by IGN France. Open License https://www.etalab.gouv.fr/wp-content/uploads/2014/05/Open_Licence.pdf + Tag 33550: 0.100000,0.100000,0.000000 + Tag 33922: 0.000000,0.000000,0.000000,-5.500000,52.000000,0.000000 + Tag 34735: 1,1,1,3,1024,0,1,2,1025,0,1,2,2048,0,1,4275 + Tag 42112: <GDALMetadata> + <Item name="grid_name">FRANCE</Item> + <Item name="target_crs_epsg_code">4171</Item> + <Item name="TYPE">HORIZONTAL_OFFSET</Item> + <Item name="UNITTYPE" sample="0" role="unittype">arc-second</Item> + <Item name="DESCRIPTION" sample="0" role="description">latitude_offset</Item> + <Item name="positive_value" sample="1">east</Item> + <Item name="UNITTYPE" sample="1" role="unittype">arc-second</Item> + <Item name="DESCRIPTION" sample="1" role="description">longitude_offset</Item> + <Item name="UNITTYPE" sample="2" role="unittype">arc-second</Item> + <Item name="DESCRIPTION" sample="2" role="description">latitude_offset_accuracy</Item> + <Item name="UNITTYPE" sample="3" role="unittype">arc-second</Item> + <Item name="DESCRIPTION" sample="3" role="description">longitude_offset_accuracy</Item> + </GDALMetadata> + + Predictor: floating point predictor 3 (0x3) + + +:: + + $ listgeo ntf_r93.tif + + Geotiff_Information: + Version: 1 + Key_Revision: 1.1 + Tagged_Information: + ModelTiepointTag (2,3): + 0 0 0 + -5.5 52 0 + ModelPixelScaleTag (1,3): + 0.1 0.1 0 + End_Of_Tags. + Keyed_Information: + GTModelTypeGeoKey (Short,1): ModelTypeGeographic + GTRasterTypeGeoKey (Short,1): RasterPixelIsPoint + GeodeticCRSGeoKey (Short,1): Code-4275 (NTF) + End_Of_Keys. + End_Of_Geotiff. + + GCS: 4275/NTF + Datum: 6275/Nouvelle Triangulation Francaise + Ellipsoid: 7011/Clarke 1880 (IGN) (6378249.20,6356515.00) + Prime Meridian: 8901/Greenwich (0.000000/ 0d 0' 0.00"E) + Projection Linear Units: User-Defined (1.000000m) + + Corner Coordinates: + Upper Left ( 5d30' 0.00"W, 52d 0' 0.00"N) + Lower Left ( 5d30' 0.00"W, 40d54' 0.00"N) + Upper Right ( 10d 6' 0.00"E, 52d 0' 0.00"N) + Lower Right ( 10d 6' 0.00"E, 40d54' 0.00"N) + Center ( 2d18' 0.00"E, 46d27' 0.00"N) + +:: + + $ gdalinfo ntf_r93.tif + + Driver: GTiff/GeoTIFF + Files: ntf_r93.tif + Size is 156, 111 + Coordinate System is: + GEOGCRS["NTF", + DATUM["Nouvelle Triangulation Francaise", + ELLIPSOID["Clarke 1880 (IGN)",6378249.2,293.466021293627, + LENGTHUNIT["metre",1]]], + PRIMEM["Greenwich",0, + ANGLEUNIT["degree",0.0174532925199433]], + CS[ellipsoidal,2], + AXIS["geodetic latitude (Lat)",north, + ORDER[1], + ANGLEUNIT["degree",0.0174532925199433]], + AXIS["geodetic longitude (Lon)",east, + ORDER[2], + ANGLEUNIT["degree",0.0174532925199433]], + ID["EPSG",4275]] + Data axis to CRS axis mapping: 2,1 + Origin = (-5.550000000000000,52.049999999999997) + Pixel Size = (0.100000000000000,-0.100000000000000) + Metadata: + AREA_OR_POINT=Point + grid_name=FRANCE + target_crs_epsg_code=4171 + TIFFTAG_DATETIME=2019:12:09 00:00:00 + TIFFTAG_IMAGEDESCRIPTION=NTF (EPSG:4275) to RGF93 (EPSG:4171). Converted from ntf_r93.gsb (version IGN07_01, last updated on 2007-10-31) + TYPE=HORIZONTAL_OFFSET + Image Structure Metadata: + COMPRESSION=DEFLATE + INTERLEAVE=BAND + Corner Coordinates: + Upper Left ( -5.5500000, 52.0500000) ( 5d33' 0.00"W, 52d 3' 0.00"N) + Lower Left ( -5.5500000, 40.9500000) ( 5d33' 0.00"W, 40d57' 0.00"N) + Upper Right ( 10.0500000, 52.0500000) ( 10d 3' 0.00"E, 52d 3' 0.00"N) + Lower Right ( 10.0500000, 40.9500000) ( 10d 3' 0.00"E, 40d57' 0.00"N) + Center ( 2.2500000, 46.5000000) ( 2d15' 0.00"E, 46d30' 0.00"N) + Band 1 Block=156x111 Type=Float32, ColorInterp=Gray + Description = latitude_offset + Unit Type: arc-second + Band 2 Block=156x111 Type=Float32, ColorInterp=Undefined + Description = longitude_offset + Unit Type: arc-second + Metadata: + positive_value=east + Band 3 Block=156x111 Type=Float32, ColorInterp=Undefined + Description = latitude_offset_accuracy + Unit Type: arc-second + Band 4 Block=156x111 Type=Float32, ColorInterp=Undefined + Description = longitude_offset_accuracy + Unit Type: arc-second + +Multi-grid storage +++++++++++++++++++ + +Formats like NTv2 can contain multiple subgrids. This can be transposed to +TIFF by using several IFD chained together with the last 4 bytes (or 8 bytes +for BigTIFF) of an IFD pointing to the offset of the next one. + +The first IFD should have a full description according to the +:ref:`General description <gtg_general_description>`. +Subsequent IFD might have a more compact description, omitting for example, CRS +information if it is identical to the main IFD (which should be the case for +the currently envisionned use cases), or Copyright / ImageDescription metadata +items. + +Each IFD will have its +`NewSubfileType <https://www.awaresystems.be/imaging/tiff/tifftags/newsubfiletype.html>`_ +tag set to 0. + +If a low-resolution grid is available, it should be put before subgrids of +higher-resolution in the chain of IFD linking. On reading, PROJ will use the +value from the highest-resoluted grid that contains the point of interest. + +For efficient reading from the network, files hosted on the CDN will use +a layout similar to the one described in the `low level paragraph of the Cloud Optimized GeoTIFF +GDAL driver page <https://gdal.org/drivers/raster/cog.html#low-level>`_ + +The layout for a file converted from NTv2 will for example be: + +- TIFF/BigTIFF header/signature and pointer to first IFD (Image File Directory) +- "ghost area" indicating the generated process +- IFD of the first grid, followed by TIFF tags values, excluding the TileOffsets and TileByteCounts arrays +- ... +- IFD of the last grid, followed by TIFF tags values, excluding the GDAL_METADATA tag, TileOffsets and TileByteCounts arrays +- TileOffsets and TileByteCounts arrays for first IFD +- ... +- TileOffsets and TileByteCounts arrays for last IFD +- Value of GDAL_METADATA tag for IFDs following the first IFD +- First IFD: Data corresponding to latitude offset of Block_0_0 +- First IFD: Data corresponding to longitude offset of Block_0_0 +- First IFD: Data corresponding to latitude offset of Block_0_1 +- First IFD: Data corresponding to longitude offset of Block_0_1 +- ... +- First IFD: Data corresponding to latitude offset of Block_n_m +- First IFD: Data corresponding to longitude offset of Block_n_m +- ... +- Last IFD: Data corresponding to latitude offset of Block_0_0 +- Last IFD: Data corresponding to longitude offset of Block_0_0 +- Last IFD: Data corresponding to latitude offset of Block_0_1 +- Last IFD: Data corresponding to longitude offset of Block_0_1 +- ... +- Last IFD: Data corresponding to latitude offset of Block_n_m +- Last IFD: Data corresponding to longitude offset of Block_n_m + +If longitude_offset_accuracy and latitude_offset_accuracy are present, this +will be followed by: + +- First IFD: Data corresponding to latitude offset accuracy of Block_0_0 +- First IFD: Data corresponding to longitude offset accuracy of Block_0_0 +- ... +- First IFD: Data corresponding to latitude offset accuracy of Block_n_m +- First IFD: Data corresponding to longitude offset accuracy of Block_n_m +- ... +- Last IFD: Data corresponding to latitude offset accuracy of Block_0_0 +- Last IFD: Data corresponding to longitude offset accuracy of Block_0_0 +- ... +- Last IFD: Data corresponding to latitude offset accuracy of Block_n_m +- Last IFD: Data corresponding to longitude offset accuracy of Block_n_m + +.. note:: + + TIFF has another mechanism to link IFDs, the SubIFD tag. This potentially + enables to define a hiearchy of IFDs (similar to HDF5 groups). There is no + support for that in most TIFF-using software, notably GDAL, and no compelling + need to have a nested hiearchy, so "flat" organization with the standard IFD chaining + mechanism is adopted. + +Examples of multi-grid dataset +++++++++++++++++++++++++++++++ + +https://github.com/rouault/sample_proj_gtiff_grids/blob/master/GDA94_GDA2020_conformal.tif has +been converted from https://github.com/OSGeo/proj-datumgrid/blob/master/oceania/GDA94_GDA2020_conformal.gsb +with https://github.com/rouault/sample_proj_gtiff_grids/blob/master/ntv2_to_gtiff.py + +It contains 5 subgrids. All essential metadata to list the subgrids and their +georeferencing is contained within the first 3 KB of the file. + +The file size is 4.8 MB using DEFLATE compression and floating-point predictor. +To be compared with the 83 MB of the original .gsb file. + +https://github.com/rouault/sample_proj_gtiff_grids/blob/master/ntv2_0.tif has +been converted from https://github.com/OSGeo/proj-datumgrid/blob/master/north-america/ntv2_0.gsb + +It contains 114 subgrids. All essential metadata to list the subgrids and their +georeferencing is contained within the first 40 KB of the file. diff --git a/docs/source/specifications/index.rst b/docs/source/specifications/index.rst new file mode 100644 index 00000000..1853ca53 --- /dev/null +++ b/docs/source/specifications/index.rst @@ -0,0 +1,14 @@ +.. _specifications: + +================================================================================ +Specifications +================================================================================ + +PROJ implements a number of extensions to standards, that are described below +for the sake of broader interoperability. + +.. toctree:: + :maxdepth: 1 + + projjson + geodetictiffgrids diff --git a/docs/source/usage/projjson.rst b/docs/source/specifications/projjson.rst index 6bcce585..6bcce585 100644 --- a/docs/source/usage/projjson.rst +++ b/docs/source/specifications/projjson.rst diff --git a/docs/source/usage/differences.rst b/docs/source/usage/differences.rst index e85ba039..be163176 100644 --- a/docs/source/usage/differences.rst +++ b/docs/source/usage/differences.rst @@ -139,3 +139,9 @@ cs2cs Removed ``-ld`` option from application, since it promoted use of deprecated paramters like ``+towgs`` and ``+datum``. + +UTF-8 adoption +-------------- + +The value of all path, filenames passed to PROJ through function calls, PROJ +strings or environment variables should be encoded in UTF-8. diff --git a/docs/source/usage/environmentvars.rst b/docs/source/usage/environmentvars.rst index 457432a0..31a74e9a 100644 --- a/docs/source/usage/environmentvars.rst +++ b/docs/source/usage/environmentvars.rst @@ -56,3 +56,21 @@ done by setting the variable with no content:: Set the debug level of PROJ. The default debug level is zero, which results in no debug output when using PROJ. A number from 1-3, whit 3 being the most verbose setting. + +.. envvar:: PROJ_NETWORK + + .. versionadded:: 7.0.0 + + If set to ON, enable the capability to use remote grids stored on CDN + (Content Delivery Network) storage, when grids are not available locally. + Alternatively, the :c:func:`proj_context_set_enable_network` function can + be used. + +.. envvar:: PROJ_NETWORK_ENDPOINT + + .. versionadded:: 7.0.0 + + Define the endpoint of the CDN storage. Normally defined through the proj.ini + configuration file locale in :envvar:`PROJ_LIB`. + Alternatively, the :c:func:`proj_context_set_url_endpoint` function can + be used. diff --git a/docs/source/usage/index.rst b/docs/source/usage/index.rst index c31c6dce..a074b18b 100644 --- a/docs/source/usage/index.rst +++ b/docs/source/usage/index.rst @@ -17,5 +17,4 @@ command line applications or the C API that is a part of the software package. transformation environmentvars differences - projjson - + network diff --git a/docs/source/usage/network.rst b/docs/source/usage/network.rst new file mode 100644 index 00000000..295232b0 --- /dev/null +++ b/docs/source/usage/network.rst @@ -0,0 +1,102 @@ +.. _network: + +================================================================================ +Network capabilities +================================================================================ + +.. versionadded:: 7.0 + +PROJ 7.0 has introduced, per :ref:`rfc4`, the capability to work with grid files +that are not installed on the local machine where PROJ is executed. + +This enables to transparently download the parts of grids that are needed to +perform a coordinate transformation. + +.. _cdn_grids: + +CDN of GeoTIFF grids +-------------------- + +Files are accessed by default through a CDN (Content Delivery Network), +accessible through https://cdn.proj.org, that contains :ref:`geodetictiffgrids` +datasets which are mirrored and managed by the +https://github.com/OSGeo/proj-datumgrid-geotiff/ GitHub project. +Files in the CDN are designed to be used by PROJ 7 or later, but any software +project wishing to use the CDN for shifting support are encouraged to +participate in the project and leverage the CDN. + +How to enable network capabilities ? +------------------------------------ + +This capability assumes that PROJ has been build against `libcurl`, and that +the user authorizes network access. + +Authorizing network access can be done in multiple ways: + + - enabling / uncommenting the ``network = on`` line of :ref:`proj-ini` + - definiting the :envvar:`PROJ_NETWORK` environment variable to ON + - or using the :cpp:func:`proj_context_set_enable_network` with a + ``enabled`` = TRUE value. + +.. note:: + + Instead of using the `libcurl` implementation, an application using the PROJ + API can supply its own network implementation through C function callbacks + with :cpp:func:`proj_context_set_network_callbacks`. Enabling network use + must still be done with one of the above mentionned method. + +Setting endpoint +---------------- + +When this is enabled, and a grid is not found in the various locations where +:ref:`resource files are looked for <resource_file_paths>`, PROJ will then +attempt at loading the file from a remote server, which defaults to +https://cdn.proj.org in :ref:`proj-ini`. This location can be changed with +the :envvar:`PROJ_NETWORK_ENDPOINT` environment variable or with +:cpp:func:`proj_context_set_url_endpoint`. + +Caching +------- + +To avoid repeated access to network, a local cache of downloaded chunks of grids +is implemented as SQLite3 database, ``cache.db``, stored in the +:ref:`PROJ user writable directory <user_writable_directory>`. + +This local caching is enabled by default (can be changed in :ref:`proj-ini` or +with :cpp:func:`proj_grid_cache_set_enable`). The default maximum size of the +cache is 300 MB, which is more than half of the total size of grids available, +at time of writing. This size can also be customized in :ref:`proj-ini` or +with :cpp:func:`proj_grid_cache_set_max_size` + +Download API +------------ + +When on-demand loading of grid is not desirable, the PROJ API also offers the +capability to download whole grids in the +:ref:`PROJ user writable directory <user_writable_directory>` by using the +:cpp:func:`proj_is_download_needed` and :cpp:func:`proj_download_file` functions. + + +Mirroring +--------- + +If you are able, you are encouraged to mirror the grids via AWS S3 command line: + +:: + + aws s3 sync s3://cdn.proj.org . + +If direct S3 access is not possible, you can also use wget to locally mirror the +data: + +:: + + wget --mirror https://cdn.proj.org/ + +Acknowledgments +--------------- + +The s3://cdn.proj.org bucket is hosted by the +`Amazon Public Datasets program <https://aws.amazon.com/opendata/public-datasets/>`_. +CDN services are provided by the AWS Public Dataset team via +`CloudFront <https://aws.amazon.com/cloudfront/>`_ diff --git a/include/proj/coordinateoperation.hpp b/include/proj/coordinateoperation.hpp index 81d82474..c7459416 100644 --- a/include/proj/coordinateoperation.hpp +++ b/include/proj/coordinateoperation.hpp @@ -144,10 +144,12 @@ class PROJ_GCC_DLL CoordinateOperation : public common::ObjectUsage, /** \brief Return grids needed by an operation. */ PROJ_DLL virtual std::set<GridDescription> - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const = 0; + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const = 0; PROJ_DLL bool - isPROJInstantiable(const io::DatabaseContextPtr &databaseContext) const; + isPROJInstantiable(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const; PROJ_DLL bool hasBallparkTransformation() const; @@ -601,7 +603,8 @@ class PROJ_GCC_DLL SingleOperation : virtual public CoordinateOperation { std::vector<metadata::PositionalAccuracyNNPtr>()); PROJ_DLL std::set<GridDescription> - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override; + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override; PROJ_DLL std::list<std::string> validateParameters() const; @@ -1680,7 +1683,8 @@ class PROJ_GCC_DLL ConcatenatedOperation final : public CoordinateOperation { bool checkExtent); // throw InvalidOperation PROJ_DLL std::set<GridDescription> - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override; + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override; PROJ_PRIVATE : @@ -1810,6 +1814,12 @@ class PROJ_GCC_DLL CoordinateOperationContext { /** Ignore grid availability at all. Results will be presented as if * all grids were available. */ IGNORE_GRID_AVAILABILITY, + + /** Results will be presented as if grids known to PROJ (that is + * registered in the grid_alternatives table of its database) were + * available. Used typically when networking is enabled. + */ + KNOWN_AVAILABLE, }; PROJ_DLL void setGridAvailabilityUse(GridAvailabilityUse use); diff --git a/include/proj/internal/coordinateoperation_internal.hpp b/include/proj/internal/coordinateoperation_internal.hpp index 361261c1..a75427c2 100644 --- a/include/proj/internal/coordinateoperation_internal.hpp +++ b/include/proj/internal/coordinateoperation_internal.hpp @@ -180,8 +180,10 @@ class InverseConversion : public Conversion, public InverseCoordinateOperation { // 'osgeo::proj::operation::SingleOperation::osgeo::proj::operation::SingleOperation::gridsNeeded' // via dominance std::set<GridDescription> - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override { - return SingleOperation::gridsNeeded(databaseContext); + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override { + return SingleOperation::gridsNeeded(databaseContext, + considerKnownGridsAsAvailable); } #endif @@ -232,8 +234,10 @@ class InverseTransformation : public Transformation, // 'osgeo::proj::operation::SingleOperation::osgeo::proj::operation::SingleOperation::gridsNeeded' // via dominance std::set<GridDescription> - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override { - return SingleOperation::gridsNeeded(databaseContext); + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override { + return SingleOperation::gridsNeeded(databaseContext, + considerKnownGridsAsAvailable); } #endif @@ -274,7 +278,8 @@ class PROJBasedOperation : public SingleOperation { bool hasRoughTransformation); std::set<GridDescription> - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override; + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override; protected: PROJBasedOperation(const PROJBasedOperation &) = default; diff --git a/include/proj/io.hpp b/include/proj/io.hpp index 37941901..e2f48dd4 100644 --- a/include/proj/io.hpp +++ b/include/proj/io.hpp @@ -832,6 +832,7 @@ class PROJ_GCC_DLL DatabaseContext { bool &inverse) const; PROJ_DLL bool lookForGridInfo(const std::string &projFilename, + bool considerKnownGridsAsAvailable, std::string &fullFilename, std::string &packageName, std::string &url, bool &directDownload, bool &openLicense, @@ -1055,7 +1056,8 @@ class PROJ_GCC_DLL AuthorityFactory { const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, bool tryReverseOrder = false, + bool considerKnownGridsAsAvailable, bool discardSuperseded, + bool tryReverseOrder = false, bool reportOnlyIntersectingTransformations = false, const metadata::ExtentPtr &intersectingExtent1 = nullptr, const metadata::ExtentPtr &intersectingExtent2 = nullptr) const; @@ -1065,7 +1067,7 @@ class PROJ_GCC_DLL AuthorityFactory { const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, + bool considerKnownGridsAsAvailable, bool discardSuperseded, const std::vector<std::pair<std::string, std::string>> &intermediateCRSAuthCodes, ObjectType allowedIntermediateObjectType = ObjectType::CRS, @@ -1131,7 +1133,7 @@ class PROJ_GCC_DLL AuthorityFactory { const std::string &sourceCRSCode, const crs::CRSNNPtr &targetCRS, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, + bool considerKnownGridsAsAvailable, bool discardSuperseded, const std::vector<std::string> &allowedAuthorities, const metadata::ExtentPtr &intersectingExtent1, const metadata::ExtentPtr &intersectingExtent2) const; diff --git a/scripts/doxygen.sh b/scripts/doxygen.sh index 48646237..380afcfa 100755 --- a/scripts/doxygen.sh +++ b/scripts/doxygen.sh @@ -34,7 +34,11 @@ fi mkdir -p docs/build/tmp_breathe python scripts/generate_breathe_friendly_general_doc.py rm -rf docs/build/xml/ -(cat Doxyfile; printf "GENERATE_HTML=NO\nGENERATE_XML=YES\nINPUT= src/iso19111 include/proj src/proj.h docs/build/tmp_breathe/general_doc.dox.reworked.h") | doxygen - > docs/build/tmp_breathe/docs_log.txt 2>&1 + +# Ugly hack to workaround a bug of Doxygen 1.8.17 that erroneously detect proj_network_get_header_value_cbk_type/ as a variable +sed "s/const char\* (\*proj_network_get_header_value_cbk_type/CONST_CHAR\* (\*proj_network_get_header_value_cbk_type/" < src/proj.h > docs/build/tmp_breathe/proj.h + +(cat Doxyfile; printf "GENERATE_HTML=NO\nGENERATE_XML=YES\nINPUT= src/iso19111 include/proj docs/build/tmp_breathe/proj.h src/filemanager.cpp src/networkfilemanager.cpp docs/build/tmp_breathe/general_doc.dox.reworked.h") | doxygen - > docs/build/tmp_breathe/docs_log.txt 2>&1 if grep -i warning docs/build/tmp_breathe/docs_log.txt; then echo "Doxygen warnings found" && cat docs/build/tmp_breathe/docs_log.txt && /bin/false; else @@ -56,5 +60,9 @@ done sed "s/Convention/Convention_/g" < ${TOPDIR}/docs/build/xml/classosgeo_1_1proj_1_1io_1_1WKTFormatter.xml | sed "s/WKT2_2018/_WKT2_2018/g" | sed "s/WKT2_2019/_WKT2_2019/g" | sed "s/WKT2_2015/_WKT2_2015/g" | sed "s/WKT1_GDAL/_WKT1_GDAL/g" | sed "s/WKT1_ESRI/_WKT1_ESRI/g" > ${TOPDIR}/docs/build/xml/classosgeo_1_1proj_1_1io_1_1WKTFormatter.xml.tmp mv ${TOPDIR}/docs/build/xml/classosgeo_1_1proj_1_1io_1_1WKTFormatter.xml.tmp ${TOPDIR}/docs/build/xml/classosgeo_1_1proj_1_1io_1_1WKTFormatter.xml +# Ugly hack to workaround a bug of Doxygen 1.8.17 that erroneously detect proj_network_get_header_value_cbk_type/ as a variable +sed "s/CONST_CHAR/const char/" < ${TOPDIR}/docs/build/xml/proj_8h.xml > ${TOPDIR}/docs/build/xml/proj_8h.xml.tmp +mv ${TOPDIR}/docs/build/xml/proj_8h.xml.tmp ${TOPDIR}/docs/build/xml/proj_8h.xml + popd > /dev/null || exit diff --git a/scripts/reference_exported_symbols.txt b/scripts/reference_exported_symbols.txt index 49d6e5d2..07043504 100644 --- a/scripts/reference_exported_symbols.txt +++ b/scripts/reference_exported_symbols.txt @@ -281,6 +281,26 @@ osgeo::proj::datum::TemporalDatum::temporalOrigin() const osgeo::proj::datum::VerticalReferenceFrame::create(osgeo::proj::util::PropertyMap const&, osgeo::proj::util::optional<std::string> const&, osgeo::proj::util::optional<osgeo::proj::datum::RealizationMethod> const&) osgeo::proj::datum::VerticalReferenceFrame::realizationMethod() const osgeo::proj::datum::VerticalReferenceFrame::~VerticalReferenceFrame() +osgeo::proj::GenericShiftGrid::~GenericShiftGrid() +osgeo::proj::GenericShiftGrid::GenericShiftGrid(std::string const&, int, int, osgeo::proj::ExtentAndRes const&) +osgeo::proj::GenericShiftGrid::gridAt(double, double) const +osgeo::proj::GenericShiftGridSet::~GenericShiftGridSet() +osgeo::proj::GenericShiftGridSet::GenericShiftGridSet() +osgeo::proj::GenericShiftGridSet::gridAt(double, double) const +osgeo::proj::GenericShiftGridSet::open(projCtx_t*, std::string const&) +osgeo::proj::GenericShiftGridSet::reassign_context(projCtx_t*) +osgeo::proj::GenericShiftGridSet::reopen(projCtx_t*) +osgeo::proj::Grid::~Grid() +osgeo::proj::Grid::Grid(std::string const&, int, int, osgeo::proj::ExtentAndRes const&) +osgeo::proj::HorizontalShiftGrid::gridAt(double, double) const +osgeo::proj::HorizontalShiftGrid::~HorizontalShiftGrid() +osgeo::proj::HorizontalShiftGrid::HorizontalShiftGrid(std::string const&, int, int, osgeo::proj::ExtentAndRes const&) +osgeo::proj::HorizontalShiftGridSet::gridAt(double, double) const +osgeo::proj::HorizontalShiftGridSet::~HorizontalShiftGridSet() +osgeo::proj::HorizontalShiftGridSet::HorizontalShiftGridSet() +osgeo::proj::HorizontalShiftGridSet::open(projCtx_t*, std::string const&) +osgeo::proj::HorizontalShiftGridSet::reassign_context(projCtx_t*) +osgeo::proj::HorizontalShiftGridSet::reopen(projCtx_t*) osgeo::proj::internal::ci_equal(std::string const&, char const*) osgeo::proj::internal::ci_equal(std::string const&, std::string const&) osgeo::proj::internal::ci_find(std::string const&, char const*) @@ -301,8 +321,8 @@ osgeo::proj::io::AuthorityFactory::create(dropbox::oxygen::nn<std::shared_ptr<os osgeo::proj::io::AuthorityFactory::createEllipsoid(std::string const&) const osgeo::proj::io::AuthorityFactory::createExtent(std::string const&) const osgeo::proj::io::AuthorityFactory::createFromCoordinateReferenceSystemCodes(std::string const&, std::string const&) const -osgeo::proj::io::AuthorityFactory::createFromCoordinateReferenceSystemCodes(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, bool, bool, std::shared_ptr<osgeo::proj::metadata::Extent> const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&) const -osgeo::proj::io::AuthorityFactory::createFromCRSCodesWithIntermediates(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, std::vector<std::pair<std::string, std::string>, std::allocator<std::pair<std::string, std::string> > > const&, osgeo::proj::io::AuthorityFactory::ObjectType, std::vector<std::string, std::allocator<std::string> > const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&) const +osgeo::proj::io::AuthorityFactory::createFromCoordinateReferenceSystemCodes(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, bool, bool, bool, std::shared_ptr<osgeo::proj::metadata::Extent> const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&) const +osgeo::proj::io::AuthorityFactory::createFromCRSCodesWithIntermediates(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, bool, std::vector<std::pair<std::string, std::string>, std::allocator<std::pair<std::string, std::string> > > const&, osgeo::proj::io::AuthorityFactory::ObjectType, std::vector<std::string, std::allocator<std::string> > const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&) const osgeo::proj::io::AuthorityFactory::createGeodeticCRS(std::string const&) const osgeo::proj::io::AuthorityFactory::createGeodeticDatum(std::string const&) const osgeo::proj::io::AuthorityFactory::createGeographicCRS(std::string const&) const @@ -332,7 +352,7 @@ osgeo::proj::io::DatabaseContext::getDatabaseStructure() const osgeo::proj::io::DatabaseContext::getMetadata(char const*) const osgeo::proj::io::DatabaseContext::getPath() const osgeo::proj::io::DatabaseContext::getSqliteHandle() const -osgeo::proj::io::DatabaseContext::lookForGridInfo(std::string const&, std::string&, std::string&, std::string&, bool&, bool&, bool&) const +osgeo::proj::io::DatabaseContext::lookForGridInfo(std::string const&, bool, std::string&, std::string&, std::string&, bool&, bool&, bool&) const osgeo::proj::io::FactoryException::~FactoryException() osgeo::proj::io::FactoryException::FactoryException(char const*) osgeo::proj::io::FactoryException::FactoryException(osgeo::proj::io::FactoryException const&) @@ -467,7 +487,7 @@ osgeo::proj::metadata::VerticalExtent::~VerticalExtent() osgeo::proj::operation::ConcatenatedOperation::~ConcatenatedOperation() osgeo::proj::operation::ConcatenatedOperation::createComputeMetadata(std::vector<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::operation::CoordinateOperation> >, std::allocator<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::operation::CoordinateOperation> > > > const&, bool) osgeo::proj::operation::ConcatenatedOperation::create(osgeo::proj::util::PropertyMap const&, std::vector<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::operation::CoordinateOperation> >, std::allocator<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::operation::CoordinateOperation> > > > const&, std::vector<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::metadata::PositionalAccuracy> >, std::allocator<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::metadata::PositionalAccuracy> > > > const&) -osgeo::proj::operation::ConcatenatedOperation::gridsNeeded(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&) const +osgeo::proj::operation::ConcatenatedOperation::gridsNeeded(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&, bool) const osgeo::proj::operation::ConcatenatedOperation::inverse() const osgeo::proj::operation::ConcatenatedOperation::operations() const osgeo::proj::operation::Conversion::~Conversion() @@ -577,7 +597,7 @@ osgeo::proj::operation::CoordinateOperationFactory::createOperation(dropbox::oxy osgeo::proj::operation::CoordinateOperationFactory::createOperations(dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::crs::CRS> > const&, dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::crs::CRS> > const&, dropbox::oxygen::nn<std::unique_ptr<osgeo::proj::operation::CoordinateOperationContext, std::default_delete<osgeo::proj::operation::CoordinateOperationContext> > > const&) const osgeo::proj::operation::CoordinateOperation::hasBallparkTransformation() const osgeo::proj::operation::CoordinateOperation::interpolationCRS() const -osgeo::proj::operation::CoordinateOperation::isPROJInstantiable(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&) const +osgeo::proj::operation::CoordinateOperation::isPROJInstantiable(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&, bool) const osgeo::proj::operation::CoordinateOperation::normalizeForVisualization() const osgeo::proj::operation::CoordinateOperation::operationVersion() const osgeo::proj::operation::CoordinateOperation::shallowClone() const @@ -623,7 +643,7 @@ osgeo::proj::operation::ParameterValue::value() const osgeo::proj::operation::ParameterValue::valueFile() const osgeo::proj::operation::PointMotionOperation::~PointMotionOperation() osgeo::proj::operation::SingleOperation::createPROJBased(osgeo::proj::util::PropertyMap const&, std::string const&, std::shared_ptr<osgeo::proj::crs::CRS> const&, std::shared_ptr<osgeo::proj::crs::CRS> const&, std::vector<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::metadata::PositionalAccuracy> >, std::allocator<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::metadata::PositionalAccuracy> > > > const&) -osgeo::proj::operation::SingleOperation::gridsNeeded(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&) const +osgeo::proj::operation::SingleOperation::gridsNeeded(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&, bool) const osgeo::proj::operation::SingleOperation::method() const osgeo::proj::operation::SingleOperation::parameterValue(int) const osgeo::proj::operation::SingleOperation::parameterValueMeasure(int) const @@ -698,6 +718,15 @@ osgeo::proj::util::PropertyMap::set(std::string const&, std::string const&) osgeo::proj::util::PropertyMap::set(std::string const&, std::vector<std::string, std::allocator<std::string> > const&) osgeo::proj::util::UnsupportedOperationException::~UnsupportedOperationException() osgeo::proj::util::UnsupportedOperationException::UnsupportedOperationException(osgeo::proj::util::UnsupportedOperationException const&) +osgeo::proj::VerticalShiftGrid::gridAt(double, double) const +osgeo::proj::VerticalShiftGridSet::gridAt(double, double) const +osgeo::proj::VerticalShiftGridSet::open(projCtx_t*, std::string const&) +osgeo::proj::VerticalShiftGridSet::reassign_context(projCtx_t*) +osgeo::proj::VerticalShiftGridSet::reopen(projCtx_t*) +osgeo::proj::VerticalShiftGridSet::~VerticalShiftGridSet() +osgeo::proj::VerticalShiftGridSet::VerticalShiftGridSet() +osgeo::proj::VerticalShiftGrid::~VerticalShiftGrid() +osgeo::proj::VerticalShiftGrid::VerticalShiftGrid(std::string const&, int, int, osgeo::proj::ExtentAndRes const&) pj_acquire_lock pj_add_type_crs_if_needed(std::string const&) pj_apply_gridshift @@ -709,6 +738,9 @@ pj_chomp(char*) pj_cleanup_lock pj_clear_initcache pj_compare_datums +pj_context_get_grid_cache_filename(projCtx_t*) +pj_context_get_user_writable_directory(projCtx_t*, bool) +pj_context_is_network_enabled(projCtx_t*) pj_ctx_alloc pj_ctx_fclose pj_ctx_fgets @@ -796,9 +828,14 @@ proj_context_get_use_proj4_init_rules proj_context_guess_wkt_dialect proj_context_set_autoclose_database proj_context_set_database_path +proj_context_set_enable_network +proj_context_set_fileapi proj_context_set_file_finder +proj_context_set_network_callbacks proj_context_set(PJconsts*, projCtx_t*) proj_context_set_search_paths +proj_context_set_sqlite3_vfs_name +proj_context_set_url_endpoint proj_context_use_proj4_init_rules proj_convert_conversion_to_other_method proj_coord @@ -928,6 +965,7 @@ proj_cs_get_axis_info proj_cs_get_type proj_destroy proj_dmstor +proj_download_file proj_ellipsoid_get_parameters proj_errno proj_errno_reset @@ -953,6 +991,11 @@ proj_get_scope proj_get_source_crs proj_get_target_crs proj_get_type +proj_grid_cache_clear +proj_grid_cache_set_enable +proj_grid_cache_set_filename +proj_grid_cache_set_max_size +proj_grid_cache_set_ttl proj_grid_get_info_from_database proj_grid_info proj_identify @@ -962,6 +1005,7 @@ proj_int_list_destroy proj_is_crs proj_is_deprecated proj_is_derived_crs +proj_is_download_needed proj_is_equivalent_to proj_is_equivalent_to_with_ctx proj_list_angular_units diff --git a/scripts/reformat_cpp.sh b/scripts/reformat_cpp.sh index a8002a6c..20c32b29 100755 --- a/scripts/reformat_cpp.sh +++ b/scripts/reformat_cpp.sh @@ -15,7 +15,11 @@ esac TOPDIR="$SCRIPT_DIR/.." -for i in "$TOPDIR"/include/proj/*.hpp "$TOPDIR"/include/proj/internal/*.hpp "$TOPDIR"/src/iso19111/*.cpp "$TOPDIR"/test/unit/*.cpp "$TOPDIR"/src/apps/projinfo.cpp "$TOPDIR"/src/tracing.cpp; do +for i in "$TOPDIR"/include/proj/*.hpp "$TOPDIR"/include/proj/internal/*.hpp \ + "$TOPDIR"/src/iso19111/*.cpp "$TOPDIR"/test/unit/*.cpp "$TOPDIR"/src/apps/projinfo.cpp \ + "$TOPDIR"/src/tracing.cpp "$TOPDIR"/src/grids.hpp "$TOPDIR"/src/grids.cpp \ + "$TOPDIR"/src/filemanager.hpp "$TOPDIR"/src/filemanager.cpp \ + "$TOPDIR"/src/sqlite3_utils.hpp "$TOPDIR"/src/sqlite3_utils.cpp ; do if ! echo "$i" | grep -q "lru_cache.hpp"; then "$SCRIPT_DIR"/reformat.sh "$i"; fi diff --git a/src/4D_api.cpp b/src/4D_api.cpp index 087cac5c..dabd44a0 100644 --- a/src/4D_api.cpp +++ b/src/4D_api.cpp @@ -47,6 +47,8 @@ #include "proj_internal.h" #include <math.h> #include "geodesic.h" +#include "grids.hpp" +#include "filemanager.hpp" #include "proj/common.hpp" #include "proj/coordinateoperation.hpp" @@ -262,6 +264,9 @@ similarly, but prefers the 2D resp. 3D interfaces if available. } PJ_COORD res = direction == PJ_FWD ? pj_fwd4d( coord, alt.pj ) : pj_inv4d( coord, alt.pj ); + if( proj_errno(alt.pj) == PJD_ERR_NETWORK_ERROR ) { + return proj_coord_error (); + } if( res.xyzt.x != HUGE_VAL ) { return res; } @@ -290,7 +295,7 @@ similarly, but prefers the 2D resp. 3D interfaces if available. auto coordOperation = dynamic_cast< NS_PROJ::operation::CoordinateOperation*>(alt.pj->iso_obj.get()); if( coordOperation ) { - if( coordOperation->gridsNeeded(dbContext).empty() ) { + if( coordOperation->gridsNeeded(dbContext, true).empty() ) { if( P->iCurCoordOp != i ) { if (proj_log_level(P->ctx, PJ_LOG_TELL) >= PJ_LOG_DEBUG) { std::string msg("Using coordinate operation "); @@ -1146,7 +1151,10 @@ PJ *proj_create_crs_to_crs_from_pj (PJ_CONTEXT *ctx, const PJ *source_crs, cons proj_operation_factory_context_set_spatial_criterion( ctx, operation_ctx, PROJ_SPATIAL_CRITERION_PARTIAL_INTERSECTION); proj_operation_factory_context_set_grid_availability_use( - ctx, operation_ctx, PROJ_GRID_AVAILABILITY_DISCARD_OPERATION_IF_MISSING_GRID); + ctx, operation_ctx, + pj_context_is_network_enabled(ctx) ? + PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE: + PROJ_GRID_AVAILABILITY_DISCARD_OPERATION_IF_MISSING_GRID); auto op_list = proj_create_operations(ctx, source_crs, target_crs, operation_ctx); @@ -1476,10 +1484,20 @@ PJ_INFO proj_info (void) { /* build search path string */ auto ctx = pj_get_default_ctx(); if (!ctx || ctx->search_paths.empty()) { - const char *envPROJ_LIB = getenv("PROJ_LIB"); - buf = path_append(buf, envPROJ_LIB, &buf_size); + // Env var mostly for testing purposes and being independent from + // an existing installation + const char* ignoreUserWritableDirectory = + getenv("PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY"); + if( ignoreUserWritableDirectory == nullptr || + ignoreUserWritableDirectory[0] == '\0' ) { + buf = path_append(buf, + pj_context_get_user_writable_directory(ctx, false).c_str(), + &buf_size); + } + const std::string envPROJ_LIB = NS_PROJ::FileManager::getProjLibEnvVar(ctx); + buf = path_append(buf, envPROJ_LIB.empty() ? nullptr : envPROJ_LIB.c_str(), &buf_size); #ifdef PROJ_LIB - if (envPROJ_LIB == nullptr) { + if (envPROJ_LIB.empty()) { buf = path_append(buf, PROJ_LIB, &buf_size); } #endif @@ -1583,43 +1601,65 @@ PJ_GRID_INFO proj_grid_info(const char *gridname) { /*PJ_CONTEXT *ctx = proj_context_create(); */ PJ_CONTEXT *ctx = pj_get_default_ctx(); - PJ_GRIDINFO *gridinfo = pj_gridinfo_init(ctx, gridname); memset(&grinfo, 0, sizeof(PJ_GRID_INFO)); - /* in case the grid wasn't found */ - if (gridinfo->filename == nullptr || gridinfo->ct == nullptr) { - pj_gridinfo_free(ctx, gridinfo); - strcpy(grinfo.format, "missing"); - return grinfo; - } - - /* The string copies below are automatically null-terminated due to */ - /* the memset above, so strncpy is safe */ + const auto fillGridInfo = [&grinfo, ctx, gridname] + (const NS_PROJ::Grid& grid, const std::string& format) + { + const auto& extent = grid.extentAndRes(); - /* name of grid */ - strncpy (grinfo.gridname, gridname, sizeof(grinfo.gridname) - 1); + /* name of grid */ + strncpy (grinfo.gridname, gridname, sizeof(grinfo.gridname) - 1); - /* full path of grid */ - pj_find_file(ctx, gridname, grinfo.filename, sizeof(grinfo.filename) - 1); + /* full path of grid */ + pj_find_file(ctx, gridname, grinfo.filename, sizeof(grinfo.filename) - 1); - /* grid format */ - strncpy (grinfo.format, gridinfo->format, sizeof(grinfo.format) - 1); + /* grid format */ + strncpy (grinfo.format, format.c_str(), sizeof(grinfo.format) - 1); - /* grid size */ - grinfo.n_lon = gridinfo->ct->lim.lam; - grinfo.n_lat = gridinfo->ct->lim.phi; + /* grid size */ + grinfo.n_lon = grid.width(); + grinfo.n_lat = grid.height(); - /* cell size */ - grinfo.cs_lon = gridinfo->ct->del.lam; - grinfo.cs_lat = gridinfo->ct->del.phi; + /* cell size */ + grinfo.cs_lon = extent.resLon; + grinfo.cs_lat = extent.resLat; - /* bounds of grid */ - grinfo.lowerleft = gridinfo->ct->ll; - grinfo.upperright.lam = grinfo.lowerleft.lam + grinfo.n_lon*grinfo.cs_lon; - grinfo.upperright.phi = grinfo.lowerleft.phi + grinfo.n_lat*grinfo.cs_lat; + /* bounds of grid */ + grinfo.lowerleft.lam = extent.westLon; + grinfo.lowerleft.phi = extent.southLat; + grinfo.upperright.lam = extent.eastLon; + grinfo.upperright.phi = extent.northLat; + }; - pj_gridinfo_free(ctx, gridinfo); + { + const auto gridSet = NS_PROJ::VerticalShiftGridSet::open(ctx, gridname); + if( gridSet ) + { + const auto& grids = gridSet->grids(); + if( !grids.empty() ) + { + const auto& grid = grids.front(); + fillGridInfo(*grid, gridSet->format()); + return grinfo; + } + } + } + { + const auto gridSet = NS_PROJ::HorizontalShiftGridSet::open(ctx, gridname); + if( gridSet ) + { + const auto& grids = gridSet->grids(); + if( !grids.empty() ) + { + const auto& grid = grids.front(); + fillGridInfo(*grid, gridSet->format()); + return grinfo; + } + } + } + strcpy(grinfo.format, "missing"); return grinfo; } @@ -1763,3 +1803,4 @@ PJ_FACTORS proj_factors(PJ *P, PJ_COORD lp) { return factors; } + diff --git a/src/Makefile.am b/src/Makefile.am index 62821ff9..8ab30a33 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,7 +7,7 @@ TESTS = geodtest check_PROGRAMS = geodtest AM_CPPFLAGS = -DPROJ_LIB=\"$(pkgdatadir)\" \ - -DMUTEX_@MUTEX_SETTING@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ + -DMUTEX_@MUTEX_SETTING@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ @TIFF_CFLAGS@ @TIFF_ENABLED_FLAGS@ @CURL_CFLAGS@ @CURL_ENABLED_FLAGS@ AM_CXXFLAGS = @CXX_WFLAGS@ @FLTO_FLAG@ include_HEADERS = proj.h proj_experimental.h proj_constants.h proj_api.h geodesic.h \ @@ -43,7 +43,7 @@ geodtest_LDADD = libproj.la lib_LTLIBRARIES = libproj.la libproj_la_LDFLAGS = -no-undefined -version-info 18:0:3 -libproj_la_LIBADD = @SQLITE3_LIBS@ +libproj_la_LIBADD = @SQLITE3_LIBS@ @TIFF_LIBS@ @CURL_LIBS@ libproj_la_SOURCES = \ pj_list.h proj_internal.h \ @@ -183,23 +183,22 @@ libproj_la_SOURCES = \ transformations/horner.cpp \ transformations/molodensky.cpp \ transformations/vgridshift.cpp \ + transformations/xyzgridshift.cpp \ \ aasincos.cpp adjlon.cpp \ dmstor.cpp auth.cpp \ deriv.cpp ell_set.cpp ellps.cpp errno.cpp \ factors.cpp fwd.cpp init.cpp inv.cpp \ list.cpp malloc.cpp mlfn.cpp msfn.cpp proj_mdist.cpp \ - open_lib.cpp param.cpp phi2.cpp pr_list.cpp \ + param.cpp phi2.cpp pr_list.cpp \ qsfn.cpp strerrno.cpp \ tsfn.cpp units.cpp ctx.cpp log.cpp zpoly1.cpp rtodms.cpp \ release.cpp gauss.cpp \ fileapi.cpp \ \ - gc_reader.cpp gridcatalog.cpp \ - nad_cvt.cpp nad_init.cpp nad_intr.cpp \ - apply_gridshift.cpp datums.cpp datum_set.cpp transform.cpp \ - geocent.cpp geocent.h utils.cpp gridinfo.cpp gridlist.cpp \ - mutex.cpp initcache.cpp apply_vgridshift.cpp geodesic.c \ + datums.cpp datum_set.cpp transform.cpp \ + geocent.cpp geocent.h utils.cpp \ + mutex.cpp initcache.cpp geodesic.c \ strtod.cpp \ \ 4D_api.cpp pipeline.cpp \ @@ -213,7 +212,15 @@ libproj_la_SOURCES = \ proj_json_streaming_writer.hpp \ proj_json_streaming_writer.cpp \ \ - tracing.cpp + tracing.cpp \ + \ + grids.hpp \ + grids.cpp \ + filemanager.hpp \ + filemanager.cpp \ + networkfilemanager.cpp \ + sqlite3_utils.hpp \ + sqlite3_utils.cpp # The sed hack is to please MSVC diff --git a/src/apply_gridshift.cpp b/src/apply_gridshift.cpp deleted file mode 100644 index 39e7c3b5..00000000 --- a/src/apply_gridshift.cpp +++ /dev/null @@ -1,366 +0,0 @@ -/****************************************************************************** - * Project: PROJ.4 - * Purpose: Apply datum shifts based on grid shift files (normally NAD27 to - * NAD83 or the reverse). This module is responsible for keeping - * a list of loaded grids, and calling with each one that is - * allowed for a given datum (expressed as the nadgrids= parameter). - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ****************************************************************************** - * Copyright (c) 2000, Frank Warmerdam <warmerdam@pobox.com> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - *****************************************************************************/ - -#define PJ_LIB__ - -#include <math.h> -#include <stdio.h> -#include <string.h> - -#include "proj.h" -#include "proj_internal.h" - -/************************************************************************/ -/* pj_apply_gridshift() */ -/* */ -/* This is the externally callable interface - part of the */ -/* public API - though it is not used internally any more and I */ -/* doubt it is used by any other applications. But we preserve */ -/* it to honour our public api. */ -/************************************************************************/ - -int pj_apply_gridshift( projCtx ctx, const char *nadgrids, int inverse, - long point_count, int point_offset, - double *x, double *y, double *z ) - -{ - PJ_GRIDINFO **gridlist; - int grid_count; - int ret; - - gridlist = pj_gridlist_from_nadgrids( ctx, nadgrids, &grid_count ); - - if( gridlist == nullptr || grid_count == 0 ) - { - pj_dalloc( gridlist ); - return ctx->last_errno; - } - - ret = pj_apply_gridshift_3( ctx, gridlist, grid_count, inverse, - point_count, point_offset, x, y, z ); - - /* - ** Note this frees the array of grid list pointers, but not the grids - ** which is as intended. The grids themselves live on. - */ - pj_dalloc( gridlist ); - - return ret; -} - -/************************************************************************/ -/* pj_apply_gridshift_2() */ -/* */ -/* This implementation uses the gridlist from a coordinate */ -/* system definition. If the gridlist has not yet been */ -/* populated in the coordinate system definition we set it up */ -/* now. */ -/************************************************************************/ - -int pj_apply_gridshift_2( PJ *defn, int inverse, - long point_count, int point_offset, - double *x, double *y, double *z ) - -{ - if( defn->catalog_name != nullptr ) - return pj_gc_apply_gridshift( defn, inverse, point_count, point_offset, - x, y, z ); - - if( defn->gridlist == nullptr ) - { - defn->gridlist = - pj_gridlist_from_nadgrids( pj_get_ctx( defn ), - pj_param(defn->ctx, defn->params,"snadgrids").s, - &(defn->gridlist_count) ); - - if( defn->gridlist == nullptr || defn->gridlist_count == 0 ) - return defn->ctx->last_errno; - } - - return pj_apply_gridshift_3( pj_get_ctx( defn ), - defn->gridlist, defn->gridlist_count, inverse, - point_count, point_offset, x, y, z ); -} - -/************************************************************************/ -/* find_ctable() */ -/* */ -/* Determine which grid is the correct given an input coordinate. */ -/************************************************************************/ - -struct CTABLE* find_ctable(projCtx ctx, PJ_LP input, int grid_count, PJ_GRIDINFO **tables) { - int itable; - - /* keep trying till we find a table that works */ - for( itable = 0; itable < grid_count; itable++ ) - { - - PJ_GRIDINFO *gi = tables[itable]; - struct CTABLE *ct = gi->ct; - double epsilon = (fabs(ct->del.phi)+fabs(ct->del.lam))/10000.0; - /* skip tables that don't match our point at all. */ - if ( ct->ll.phi - epsilon > input.phi - || ct->ll.lam - epsilon > input.lam - || (ct->ll.phi + (ct->lim.phi-1) * ct->del.phi + epsilon < input.phi) - || (ct->ll.lam + (ct->lim.lam-1) * ct->del.lam + epsilon < input.lam) ) { - continue; - } - - /* If we have child nodes, check to see if any of them apply. */ - while( gi->child ) - { - PJ_GRIDINFO *child; - - for( child = gi->child; child != nullptr; child = child->next ) - { - struct CTABLE *ct1 = child->ct; - epsilon = (fabs(ct1->del.phi)+fabs(ct1->del.lam))/10000.0; - - if( ct1->ll.phi - epsilon > input.phi - || ct1->ll.lam - epsilon > input.lam - || (ct1->ll.phi+(ct1->lim.phi-1)*ct1->del.phi + epsilon < input.phi) - || (ct1->ll.lam+(ct1->lim.lam-1)*ct1->del.lam + epsilon < input.lam) ) { - continue; - } - break; - } - - /* If we didn't find a child then nothing more to do */ - if( child == nullptr ) break; - - /* Otherwise use the child, first checking it's children */ - gi = child; - ct = child->ct; - } - /* load the grid shift info if we don't have it. */ - if( ct->cvs == nullptr) { - if (!pj_gridinfo_load( ctx, gi ) ) { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return nullptr; - } - } - /* if we get this far we have found a suitable grid */ - return ct; - } - - return nullptr; -} - -/************************************************************************/ -/* pj_apply_gridshift_3() */ -/* */ -/* This is the real workhorse, given a gridlist. */ -/************************************************************************/ - -int pj_apply_gridshift_3( projCtx ctx, PJ_GRIDINFO **gridlist, int gridlist_count, - int inverse, long point_count, int point_offset, - double *x, double *y, double *z ) -{ - int i; - struct CTABLE *ct; - static int debug_count = 0; - (void) z; - - if( gridlist== nullptr || gridlist_count == 0 ) - { - pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); - return PJD_ERR_FAILED_TO_LOAD_GRID; - } - - ctx->last_errno = 0; - - for( i = 0; i < point_count; i++ ) - { - long io = i * point_offset; - PJ_LP input, output; - int itable; - - input.phi = y[io]; - input.lam = x[io]; - output.phi = HUGE_VAL; - output.lam = HUGE_VAL; - - ct = find_ctable(ctx, input, gridlist_count, gridlist); - if( ct != nullptr ) - { - output = nad_cvt( ctx, input, inverse, ct, gridlist_count, gridlist); - - if ( output.lam != HUGE_VAL && debug_count++ < 20 ) - pj_log( ctx, PJ_LOG_DEBUG_MINOR, "pj_apply_gridshift(): used %s", ct->id ); - } - - if ( output.lam == HUGE_VAL ) - { - if( ctx->debug_level >= PJ_LOG_DEBUG_MAJOR ) - { - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "pj_apply_gridshift(): failed to find a grid shift table for\n" - " location (%.7fdW,%.7fdN)", - x[io] * RAD_TO_DEG, - y[io] * RAD_TO_DEG ); - for( itable = 0; itable < gridlist_count; itable++ ) - { - PJ_GRIDINFO *gi = gridlist[itable]; - if( itable == 0 ) - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, " tried: %s", gi->gridname ); - else - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, ",%s", gi->gridname ); - } - } - - /* - * We don't actually have any machinery currently to set the - * following macro, so this is mostly kept here to make it clear - * how we ought to operate if we wanted to make it super clear - * that an error has occurred when points are outside our available - * datum shift areas. But if this is on, we will find that "low - * value" points on the fringes of some datasets will completely - * fail causing lots of problems when it is more or less ok to - * just not apply a datum shift. So rather than deal with - * that we just fallback to no shift. (see also bug #45). - */ -#ifdef ERR_GRID_AREA_TRANSIENT_SEVERE - y[io] = HUGE_VAL; - x[io] = HUGE_VAL; -#else - /* leave x/y unshifted. */ -#endif - } - else - { - y[io] = output.phi; - x[io] = output.lam; - } - } - - return 0; -} - -/**********************************************/ -int proj_hgrid_init(PJ* P, const char *grids) { -/********************************************** - - Initizalize and populate list of horizontal - grids. - - Takes a PJ-object and the plus-parameter - name that is used in the proj-string to - specify the grids to load, e.g. "+grids". - The + should be left out here. - - Returns the number of loaded grids. - -***********************************************/ - - /* prepend "s" to the "grids" string to allow usage with pj_param */ - char *sgrids = (char *) pj_malloc( (strlen(grids)+1+1) *sizeof(char) ); - sprintf(sgrids, "%s%s", "s", grids); - - if (P->gridlist == nullptr) { - P->gridlist = pj_gridlist_from_nadgrids( - P->ctx, - pj_param(P->ctx, P->params, sgrids).s, - &(P->gridlist_count) - ); - - if( P->gridlist == nullptr || P->gridlist_count == 0 ) { - pj_dealloc(sgrids); - return 0; - } - } - - if (P->gridlist_count == 0) { - proj_errno_set(P, PJD_ERR_FAILED_TO_LOAD_GRID); - } - - pj_dealloc(sgrids); - return P->gridlist_count; -} - -/********************************************/ -/* proj_hgrid_value() */ -/* */ -/* Return coordinate offset in grid */ -/********************************************/ -PJ_LP proj_hgrid_value(PJ *P, PJ_LP lp) { - struct CTABLE *ct; - PJ_LP out = proj_coord_error().lp; - - ct = find_ctable(P->ctx, lp, P->gridlist_count, P->gridlist); - if (ct == nullptr) { - pj_ctx_set_errno( P->ctx, PJD_ERR_GRID_AREA); - return out; - } - - /* normalize input to ll origin */ - lp.lam -= ct->ll.lam; - lp.phi -= ct->ll.phi; - - lp.lam = adjlon(lp.lam - M_PI) + M_PI; - - out = nad_intr(lp, ct); - - if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) { - pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); - } - - return out; -} - -PJ_LP proj_hgrid_apply(PJ *P, PJ_LP lp, PJ_DIRECTION direction) { - struct CTABLE *ct; - int inverse; - PJ_LP out; - - out.lam = HUGE_VAL; out.phi = HUGE_VAL; - - ct = find_ctable(P->ctx, lp, P->gridlist_count, P->gridlist); - - if (ct == nullptr || ct->cvs == nullptr) { - if( P->gridlist_count == 1 && - strcmp(P->gridlist[0]->gridname, "null") == 0) { - // TODO: remove this particular case that is put there just to be - // able to handle longitudes outside of -180,180 - out = lp; - } else { - pj_ctx_set_errno( P->ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - } - return out; - } - - inverse = direction == PJ_FWD ? 0 : 1; - out = nad_cvt(P->ctx, lp, inverse, ct, P->gridlist_count, P->gridlist); - - if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) - pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); - - return out; - -} diff --git a/src/apply_vgridshift.cpp b/src/apply_vgridshift.cpp deleted file mode 100644 index daa44858..00000000 --- a/src/apply_vgridshift.cpp +++ /dev/null @@ -1,361 +0,0 @@ -/****************************************************************************** - * Project: PROJ.4 - * Purpose: Apply vertical datum shifts based on grid shift files, normally - * geoid grids mapping WGS84 to NAVD88 or something similar. - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ****************************************************************************** - * Copyright (c) 2010, Frank Warmerdam <warmerdam@pobox.com> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - *****************************************************************************/ - -#define PJ_LIB__ - -#include <assert.h> -#include <stdio.h> -#include <string.h> - -#include <math.h> -#include "proj_internal.h" - -static int is_nodata(float value, double vmultiplier) -{ - /* nodata? */ - /* GTX official nodata value if -88.88880f, but some grids also */ - /* use other big values for nodata (e.g naptrans2008.gtx has */ - /* nodata values like -2147479936), so test them too */ - return value * vmultiplier > 1000 || value * vmultiplier < -1000 || value == -88.88880f; -} - -static double read_vgrid_value( PJ *defn, PJ_LP input, double vmultiplier, int *gridlist_count_p, PJ_GRIDINFO **tables, struct CTABLE *ct) { - int itable = 0; - double value = HUGE_VAL; - double grid_x, grid_y; - long grid_ix, grid_iy; - long grid_ix2, grid_iy2; - float *cvs; - /* do not deal with NaN coordinates */ - /* cppcheck-suppress duplicateExpression */ - if( isnan(input.phi) || isnan(input.lam) ) - itable = *gridlist_count_p; - - /* keep trying till we find a table that works */ - for ( ; itable < *gridlist_count_p; itable++ ) - { - PJ_GRIDINFO *gi = tables[itable]; - - ct = gi->ct; - - /* skip tables that don't match our point at all (latitude check). */ - if( ct->ll.phi > input.phi - || ct->ll.phi + (ct->lim.phi-1) * ct->del.phi < input.phi ) - continue; - - bool fullWorldLongExtent = false; - if( fabs(ct->lim.lam * ct->del.lam - 2 * M_PI) < 1e-10 ) - { - fullWorldLongExtent = true; - } - - /* skip tables that don't match our point at all (longitude check). */ - else if( ct->ll.lam > input.lam - || ct->ll.lam + (ct->lim.lam-1) * ct->del.lam < input.lam ) - continue; - - /* If we have child nodes, check to see if any of them apply. */ - while( gi->child != nullptr ) - { - PJ_GRIDINFO *child; - - for( child = gi->child; child != nullptr; child = child->next ) - { - struct CTABLE *ct1 = child->ct; - - fullWorldLongExtent = false; - - if( ct1->ll.phi > input.phi - || ct1->ll.phi+(ct1->lim.phi-1)*ct1->del.phi < input.phi) - continue; - - if( fabs(ct1->lim.lam * ct1->del.lam - 2 * M_PI) < 1e-10 ) - { - fullWorldLongExtent = true; - } - else if( ct1->ll.lam > input.lam - || ct1->ll.lam+(ct1->lim.lam-1)*ct1->del.lam < input.lam) - continue; - - break; - } - - /* we didn't find a more refined child node to use, so go with current grid */ - if( child == nullptr ) - { - break; - } - - /* Otherwise let's try for childrens children .. */ - gi = child; - ct = child->ct; - } - - /* load the grid shift info if we don't have it. */ - if( ct->cvs == nullptr ) - { - pj_gridinfo_load( pj_get_ctx(defn), gi ); - } - if( ct->cvs == nullptr ) - { - pj_ctx_set_errno( defn->ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return PJD_ERR_FAILED_TO_LOAD_GRID; - } - - /* Interpolation a location within the grid */ - grid_x = (input.lam - ct->ll.lam) / ct->del.lam; - if( fullWorldLongExtent ) { - // The first fmod goes to ]-lim, lim[ range - // So we add lim again to be in ]0, 2*lim[ and fmod again - grid_x = fmod( - fmod(grid_x + ct->lim.lam, ct->lim.lam) + ct->lim.lam, - ct->lim.lam); - } - grid_y = (input.phi - ct->ll.phi) / ct->del.phi; - grid_ix = lround(floor(grid_x)); - assert(grid_ix >= 0 && grid_ix < ct->lim.lam); - grid_iy = lround(floor(grid_y)); - assert(grid_iy >= 0 && grid_iy < ct->lim.phi); - grid_x -= grid_ix; - grid_y -= grid_iy; - - grid_ix2 = grid_ix + 1; - if( grid_ix2 >= ct->lim.lam ) { - if( fullWorldLongExtent ) { - grid_ix2 = 0; - } else { - grid_ix2 = ct->lim.lam - 1; - } - } - grid_iy2 = grid_iy + 1; - if( grid_iy2 >= ct->lim.phi ) - grid_iy2 = ct->lim.phi - 1; - - cvs = (float *) ct->cvs; - { - float value_a = cvs[grid_ix + grid_iy * ct->lim.lam]; - float value_b = cvs[grid_ix2 + grid_iy * ct->lim.lam]; - float value_c = cvs[grid_ix + grid_iy2 * ct->lim.lam]; - float value_d = cvs[grid_ix2 + grid_iy2 * ct->lim.lam]; - double total_weight = 0.0; - int n_weights = 0; - value = 0.0f; - if( !is_nodata(value_a, vmultiplier) ) - { - double weight = (1.0-grid_x) * (1.0-grid_y); - value += value_a * weight; - total_weight += weight; - n_weights ++; - } - if( !is_nodata(value_b, vmultiplier) ) - { - double weight = (grid_x) * (1.0-grid_y); - value += value_b * weight; - total_weight += weight; - n_weights ++; - } - if( !is_nodata(value_c, vmultiplier) ) - { - double weight = (1.0-grid_x) * (grid_y); - value += value_c * weight; - total_weight += weight; - n_weights ++; - } - if( !is_nodata(value_d, vmultiplier) ) - { - double weight = (grid_x) * (grid_y); - value += value_d * weight; - total_weight += weight; - n_weights ++; - } - if( n_weights == 0 ) - value = HUGE_VAL; - else if( n_weights != 4 ) - value /= total_weight; - } - - } - - return value * vmultiplier; -} - -/************************************************************************/ -/* pj_apply_vgridshift() */ -/* */ -/* This implementation takes uses the gridlist from a coordinate */ -/* system definition. If the gridlist has not yet been */ -/* populated in the coordinate system definition we set it up */ -/* now. */ -/************************************************************************/ -int pj_apply_vgridshift( PJ *defn, const char *listname, - PJ_GRIDINFO ***gridlist_p, - int *gridlist_count_p, - int inverse, - long point_count, int point_offset, - double *x, double *y, double *z ) - -{ - int i; - static int debug_count = 0; - PJ_GRIDINFO **tables; - struct CTABLE ct; - - if( *gridlist_p == nullptr ) - { - *gridlist_p = - pj_gridlist_from_nadgrids( pj_get_ctx(defn), - pj_param(defn->ctx,defn->params,listname).s, - gridlist_count_p ); - - if( *gridlist_p == nullptr || *gridlist_count_p == 0 ) - return defn->ctx->last_errno; - } - - if( *gridlist_count_p == 0 ) - { - pj_ctx_set_errno( defn->ctx, PJD_ERR_FAILED_TO_LOAD_GRID); - return PJD_ERR_FAILED_TO_LOAD_GRID; - } - - tables = *gridlist_p; - defn->ctx->last_errno = 0; - - for( i = 0; i < point_count; i++ ) - { - double value; - long io = i * point_offset; - PJ_LP input; - - input.phi = y[io]; - input.lam = x[io]; - - value = read_vgrid_value(defn, input, 1.0, gridlist_count_p, tables, &ct); - - if( inverse ) - z[io] -= value; - else - z[io] += value; - if( value != HUGE_VAL ) - { - if( debug_count++ < 20 ) { - proj_log_trace(defn, "pj_apply_gridshift(): used %s", ct.id); - break; - } - } - - if( value == HUGE_VAL ) - { - int itable; - std::string gridlist; - - proj_log_debug(defn, - "pj_apply_vgridshift(): failed to find a grid shift table for\n" - " location (%.7fdW,%.7fdN)", - x[io] * RAD_TO_DEG, - y[io] * RAD_TO_DEG ); - - for( itable = 0; itable < *gridlist_count_p; itable++ ) - { - PJ_GRIDINFO *gi = tables[itable]; - if( itable == 0 ) - gridlist += " tried: "; - else - gridlist += ','; - gridlist += gi->gridname; - } - - proj_log_debug(defn, "%s", gridlist.c_str()); - pj_ctx_set_errno( defn->ctx, PJD_ERR_GRID_AREA ); - - return PJD_ERR_GRID_AREA; - } - } - - return 0; -} - -/**********************************************/ -int proj_vgrid_init(PJ* P, const char *grids) { -/********************************************** - - Initizalize and populate gridlist. - - Takes a PJ-object and the plus-parameter - name that is used in the proj-string to - specify the grids to load, e.g. "+grids". - The + should be left out here. - - Returns the number of loaded grids. - -***********************************************/ - - /* prepend "s" to the "grids" string to allow usage with pj_param */ - char *sgrids = (char *) pj_malloc( (strlen(grids)+1+1) *sizeof(char) ); - sprintf(sgrids, "%s%s", "s", grids); - - if (P->vgridlist_geoid == nullptr) { - P->vgridlist_geoid = pj_gridlist_from_nadgrids( - P->ctx, - pj_param(P->ctx, P->params, sgrids).s, - &(P->vgridlist_geoid_count) - ); - - if( P->vgridlist_geoid == nullptr || P->vgridlist_geoid_count == 0 ) { - pj_dealloc(sgrids); - return 0; - } - } - - if (P->vgridlist_geoid_count == 0) { - proj_errno_set(P, PJD_ERR_FAILED_TO_LOAD_GRID); - } - - pj_dealloc(sgrids); - return P->vgridlist_geoid_count; -} - -/***********************************************/ -double proj_vgrid_value(PJ *P, PJ_LP lp, double vmultiplier){ -/*********************************************** - - Read grid value at position lp in grids loaded - with proj_grid_init. - - Returns the grid value of the given coordinate. - -************************************************/ - - struct CTABLE used_grid; - double value; - memset(&used_grid, 0, sizeof(struct CTABLE)); - - value = read_vgrid_value(P, lp, vmultiplier, &(P->vgridlist_geoid_count), P->vgridlist_geoid, &used_grid); - proj_log_trace(P, "proj_vgrid_value: (%f, %f) = %f", lp.lam*RAD_TO_DEG, lp.phi*RAD_TO_DEG, value); - - return value; -} diff --git a/src/apps/gie.cpp b/src/apps/gie.cpp index 6a67b55d..d9907776 100644 --- a/src/apps/gie.cpp +++ b/src/apps/gie.cpp @@ -1153,6 +1153,7 @@ static const struct errno_vs_err_const lookup[] = { {"pjd_err_inconsistent_unit" , -59}, {"pjd_err_mutually_exclusive_args" , -60}, {"pjd_err_generic_error" , -61}, + {"pjd_err_network_error" , -62}, {"pjd_err_dont_skip" , 5555}, {"pjd_err_unknown" , 9999}, {"pjd_err_enomem" , ENOMEM}, diff --git a/src/apps/projinfo.cpp b/src/apps/projinfo.cpp index bef47746..27ea278a 100644 --- a/src/apps/projinfo.cpp +++ b/src/apps/projinfo.cpp @@ -87,7 +87,8 @@ static void usage() { << " [--spatial-test contains|intersects]" << std::endl << " [--crs-extent-use none|both|intersection|smallest]" << std::endl - << " [--grid-check none|discard_missing|sort] " + << " [--grid-check " + "none|discard_missing|sort|known_available] " "[--show-superseded]" << std::endl << " [--pivot-crs always|if_no_direct_transformation|" @@ -554,7 +555,7 @@ static void outputObject( auto op = dynamic_cast<CoordinateOperation *>(obj.get()); if (op && dbContext && getenv("PROJINFO_NO_GRID_CHECK") == nullptr) { try { - auto setGrids = op->gridsNeeded(dbContext); + auto setGrids = op->gridsNeeded(dbContext, false); bool firstWarning = true; for (const auto &grid : setGrids) { if (!grid.available) { @@ -570,6 +571,7 @@ static void outputObject( if (!grid.url.empty()) { std::cout << " at " << grid.url; } + std::cout << ", or on CDN"; } else if (!grid.url.empty()) { std::cout << " Can be obtained at " << grid.url; } @@ -584,8 +586,9 @@ static void outputObject( // --------------------------------------------------------------------------- -static void outputOperationSummary(const CoordinateOperationNNPtr &op, - const DatabaseContextPtr &dbContext) { +static void outputOperationSummary( + const CoordinateOperationNNPtr &op, const DatabaseContextPtr &dbContext, + CoordinateOperationContext::GridAvailabilityUse gridAvailabilityUse) { auto ids = op->identifiers(); if (!ids.empty()) { std::cout << *(ids[0]->codeSpace()) << ":" << ids[0]->code(); @@ -631,10 +634,16 @@ static void outputOperationSummary(const CoordinateOperationNNPtr &op, if (dbContext && getenv("PROJINFO_NO_GRID_CHECK") == nullptr) { try { - auto setGrids = op->gridsNeeded(dbContext); + auto setGrids = op->gridsNeeded(dbContext, false); for (const auto &grid : setGrids) { if (!grid.available) { std::cout << ", at least one grid missing"; + if (gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE && + !grid.packageName.empty()) { + std::cout << " on the system, but available on CDN"; + } break; } } @@ -731,7 +740,7 @@ static void outputOperations( } if (summary) { for (const auto &op : list) { - outputOperationSummary(op, dbContext); + outputOperationSummary(op, dbContext, gridAvailabilityUse); } } else { bool first = true; @@ -744,7 +753,7 @@ static void outputOperations( std::cout << "-------------------------------------" << std::endl; std::cout << "Operation No. " << (i + 1) << ":" << std::endl << std::endl; - outputOperationSummary(op, dbContext); + outputOperationSummary(op, dbContext, gridAvailabilityUse); std::cout << std::endl; outputObject(dbContext, op, allowUseIntermediateCRS, outputOpt); } @@ -777,7 +786,9 @@ int main(int argc, char **argv) { CoordinateOperationContext::SourceTargetCRSExtentUse::SMALLEST; bool buildBoundCRSToWGS84 = false; CoordinateOperationContext::GridAvailabilityUse gridAvailabilityUse = - CoordinateOperationContext::GridAvailabilityUse::USE_FOR_SORTING; + pj_context_is_network_enabled(nullptr) + ? CoordinateOperationContext::GridAvailabilityUse::KNOWN_AVAILABLE + : CoordinateOperationContext::GridAvailabilityUse::USE_FOR_SORTING; CoordinateOperationContext::IntermediateCRSUse allowUseIntermediateCRS = CoordinateOperationContext::IntermediateCRSUse:: IF_NO_DIRECT_TRANSFORMATION; @@ -992,6 +1003,9 @@ int main(int argc, char **argv) { } else if (ci_equal(value, "sort")) { gridAvailabilityUse = CoordinateOperationContext:: GridAvailabilityUse::USE_FOR_SORTING; + } else if (ci_equal(value, "known_available")) { + gridAvailabilityUse = CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE; } else { std::cerr << "Unrecognized value for option --grid-check: " << value << std::endl; diff --git a/src/ctx.cpp b/src/ctx.cpp index bcb6e1cc..6172b3c8 100644 --- a/src/ctx.cpp +++ b/src/ctx.cpp @@ -33,6 +33,7 @@ #include "proj_experimental.h" #include "proj_internal.h" +#include "filemanager.hpp" /************************************************************************/ /* pj_get_ctx() */ @@ -60,9 +61,9 @@ void pj_set_ctx( projPJ pj, projCtx ctx ) if (pj==nullptr) return; pj->ctx = ctx; - if( pj->is_pipeline ) + if( pj->reassign_context ) { - pj_pipeline_assign_context_to_steps(pj, ctx); + pj->reassign_context(pj, ctx); } for( const auto &alt: pj->alternativeCoordinateOperations ) { @@ -95,7 +96,8 @@ projCtx_t projCtx_t::createDefault() projCtx_t ctx; ctx.debug_level = PJ_LOG_NONE; ctx.logger = pj_stderr_logger; - ctx.fileapi = pj_get_default_fileapi(); + ctx.fileapi_legacy = pj_get_default_fileapi(); + NS_PROJ::FileManager::fillDefaultNetworkInterface(&ctx); if( getenv("PROJ_DEBUG") != nullptr ) { @@ -133,12 +135,13 @@ projCtx_t::projCtx_t(const projCtx_t& other) debug_level = other.debug_level; logger = other.logger; logger_app_data = other.logger_app_data; - fileapi = other.fileapi; + fileapi_legacy = other.fileapi_legacy; epsg_file_exists = other.epsg_file_exists; set_search_paths(other.search_paths); file_finder = other.file_finder; file_finder_legacy = other.file_finder_legacy; file_finder_user_data = other.file_finder_user_data; + networking = other.networking; } /************************************************************************/ @@ -258,28 +261,3 @@ void *pj_ctx_get_app_data( projCtx ctx ) return nullptr; return ctx->logger_app_data; } - -/************************************************************************/ -/* pj_ctx_set_fileapi() */ -/************************************************************************/ - -void pj_ctx_set_fileapi( projCtx ctx, projFileAPI *fileapi ) - -{ - if (nullptr==ctx) - return; - ctx->fileapi = fileapi; -} - -/************************************************************************/ -/* pj_ctx_get_fileapi() */ -/************************************************************************/ - -projFileAPI *pj_ctx_get_fileapi( projCtx ctx ) - -{ - if (nullptr==ctx) - return nullptr; - return ctx->fileapi; -} - diff --git a/src/datum_set.cpp b/src/datum_set.cpp index 873d7be5..15d51613 100644 --- a/src/datum_set.cpp +++ b/src/datum_set.cpp @@ -41,7 +41,7 @@ int pj_datum_set(projCtx ctx, paralist *pl, PJ *projdef) { - const char *name, *towgs84, *nadgrids, *catalog; + const char *name, *towgs84, *nadgrids; projdef->datum_type = PJD_UNKNOWN; @@ -118,25 +118,6 @@ int pj_datum_set(projCtx ctx, paralist *pl, PJ *projdef) } /* -------------------------------------------------------------------- */ -/* Check for grid catalog parameter, and optional date. */ -/* -------------------------------------------------------------------- */ - else if( (catalog = pj_param(ctx, pl,"scatalog").s) != nullptr ) - { - const char *date; - - projdef->datum_type = PJD_GRIDSHIFT; - projdef->catalog_name = pj_strdup(catalog); - if (!projdef->catalog_name) { - pj_ctx_set_errno(ctx, ENOMEM); - return 1; - } - - date = pj_param(ctx, pl, "sdate").s; - if( date != nullptr) - projdef->datum_date = pj_gc_parsedate( ctx, date); - } - -/* -------------------------------------------------------------------- */ /* Check for towgs84 parameter. */ /* -------------------------------------------------------------------- */ else if( (towgs84 = pj_param(ctx, pl,"stowgs84").s) != nullptr ) diff --git a/src/fileapi.cpp b/src/fileapi.cpp index 70c7b5de..70be2502 100644 --- a/src/fileapi.cpp +++ b/src/fileapi.cpp @@ -34,6 +34,7 @@ #include "proj.h" #include "proj_internal.h" +#include "filemanager.hpp" static PAFile stdio_fopen(projCtx ctx, const char *filename, const char *access); @@ -141,7 +142,7 @@ static void stdio_fclose(PAFile file) PAFile pj_ctx_fopen(projCtx ctx, const char *filename, const char *access) { - return ctx->fileapi->FOpen(ctx, filename, access); + return ctx->fileapi_legacy->FOpen(ctx, filename, access); } /************************************************************************/ @@ -149,7 +150,7 @@ PAFile pj_ctx_fopen(projCtx ctx, const char *filename, const char *access) /************************************************************************/ size_t pj_ctx_fread(projCtx ctx, void *buffer, size_t size, size_t nmemb, PAFile file) { - return ctx->fileapi->FRead(buffer, size, nmemb, file); + return ctx->fileapi_legacy->FRead(buffer, size, nmemb, file); } /************************************************************************/ @@ -157,7 +158,7 @@ size_t pj_ctx_fread(projCtx ctx, void *buffer, size_t size, size_t nmemb, PAFile /************************************************************************/ int pj_ctx_fseek(projCtx ctx, PAFile file, long offset, int whence) { - return ctx->fileapi->FSeek(file, offset, whence); + return ctx->fileapi_legacy->FSeek(file, offset, whence); } /************************************************************************/ @@ -165,7 +166,7 @@ int pj_ctx_fseek(projCtx ctx, PAFile file, long offset, int whence) /************************************************************************/ long pj_ctx_ftell(projCtx ctx, PAFile file) { - return ctx->fileapi->FTell(file); + return ctx->fileapi_legacy->FTell(file); } /************************************************************************/ @@ -173,7 +174,7 @@ long pj_ctx_ftell(projCtx ctx, PAFile file) /************************************************************************/ void pj_ctx_fclose(projCtx ctx, PAFile file) { - ctx->fileapi->FClose(file); + ctx->fileapi_legacy->FClose(file); } /************************************************************************/ @@ -212,3 +213,28 @@ char *pj_ctx_fgets(projCtx ctx, char *line, int size, PAFile file) } return line; } + +/************************************************************************/ +/* pj_ctx_set_fileapi() */ +/************************************************************************/ + +void pj_ctx_set_fileapi( projCtx ctx, projFileAPI *fileapi ) + +{ + if (nullptr==ctx) + return; + ctx->fileapi_legacy = fileapi; +} + +/************************************************************************/ +/* pj_ctx_get_fileapi() */ +/************************************************************************/ + +projFileAPI *pj_ctx_get_fileapi( projCtx ctx ) + +{ + if (nullptr==ctx) + return nullptr; + return ctx->fileapi_legacy; +} + diff --git a/src/filemanager.cpp b/src/filemanager.cpp new file mode 100644 index 00000000..005e734b --- /dev/null +++ b/src/filemanager.cpp @@ -0,0 +1,1733 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: File manager + * Author: Even Rouault, <even.rouault at spatialys.com> + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef FROM_PROJ_CPP +#define FROM_PROJ_CPP +#endif +#define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS + +#include <errno.h> +#include <stdlib.h> + +#include <algorithm> +#include <limits> +#include <string> + +#include "filemanager.hpp" +#include "proj.h" +#include "proj/internal/internal.hpp" +#include "proj_internal.h" + +#include <sys/stat.h> + +#ifdef _WIN32 +#include <shlobj.h> +#else +#include <sys/types.h> +#include <unistd.h> +#endif + +//! @cond Doxygen_Suppress + +#define STR_HELPER(x) #x +#define STR(x) STR_HELPER(x) + +using namespace NS_PROJ::internal; + +NS_PROJ_START + +// --------------------------------------------------------------------------- + +File::File(const std::string &name) : name_(name) {} + +// --------------------------------------------------------------------------- + +File::~File() = default; + +#ifdef _WIN32 + +/* The bulk of utf8towc()/utf8fromwc() is derived from the utf.c module from + * FLTK. It was originally downloaded from: + * http://svn.easysw.com/public/fltk/fltk/trunk/src/utf.c + * And already used by GDAL + */ +/************************************************************************/ +/* ==================================================================== */ +/* UTF.C code from FLTK with some modifications. */ +/* ==================================================================== */ +/************************************************************************/ + +/* Set to 1 to turn bad UTF8 bytes into ISO-8859-1. If this is to zero + they are instead turned into the Unicode REPLACEMENT CHARACTER, of + value 0xfffd. + If this is on utf8decode will correctly map most (perhaps all) + human-readable text that is in ISO-8859-1. This may allow you + to completely ignore character sets in your code because virtually + everything is either ISO-8859-1 or UTF-8. +*/ +#define ERRORS_TO_ISO8859_1 1 + +/* Set to 1 to turn bad UTF8 bytes in the 0x80-0x9f range into the + Unicode index for Microsoft's CP1252 character set. You should + also set ERRORS_TO_ISO8859_1. With this a huge amount of more + available text (such as all web pages) are correctly converted + to Unicode. +*/ +#define ERRORS_TO_CP1252 1 + +/* A number of Unicode code points are in fact illegal and should not + be produced by a UTF-8 converter. Turn this on will replace the + bytes in those encodings with errors. If you do this then converting + arbitrary 16-bit data to UTF-8 and then back is not an identity, + which will probably break a lot of software. +*/ +#define STRICT_RFC3629 0 + +#if ERRORS_TO_CP1252 +// Codes 0x80..0x9f from the Microsoft CP1252 character set, translated +// to Unicode: +constexpr unsigned short cp1252[32] = { + 0x20ac, 0x0081, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008d, 0x017d, 0x008f, + 0x0090, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0x009d, 0x017e, 0x0178}; +#endif + +/************************************************************************/ +/* utf8decode() */ +/************************************************************************/ + +/* + Decode a single UTF-8 encoded character starting at \e p. The + resulting Unicode value (in the range 0-0x10ffff) is returned, + and \e len is set the number of bytes in the UTF-8 encoding + (adding \e len to \e p will point at the next character). + + If \a p points at an illegal UTF-8 encoding, including one that + would go past \e end, or where a code is uses more bytes than + necessary, then *reinterpret_cast<const unsigned char*>(p) is translated as +though it is + in the Microsoft CP1252 character set and \e len is set to 1. + Treating errors this way allows this to decode almost any + ISO-8859-1 or CP1252 text that has been mistakenly placed where + UTF-8 is expected, and has proven very useful. + + If you want errors to be converted to error characters (as the + standards recommend), adding a test to see if the length is + unexpectedly 1 will work: + +\code + if( *p & 0x80 ) + { // What should be a multibyte encoding. + code = utf8decode(p, end, &len); + if( len<2 ) code = 0xFFFD; // Turn errors into REPLACEMENT CHARACTER. + } + else + { // Handle the 1-byte utf8 encoding: + code = *p; + len = 1; + } +\endcode + + Direct testing for the 1-byte case (as shown above) will also + speed up the scanning of strings where the majority of characters + are ASCII. +*/ +static unsigned utf8decode(const char *p, const char *end, int *len) { + unsigned char c = *reinterpret_cast<const unsigned char *>(p); + if (c < 0x80) { + *len = 1; + return c; +#if ERRORS_TO_CP1252 + } else if (c < 0xa0) { + *len = 1; + return cp1252[c - 0x80]; +#endif + } else if (c < 0xc2) { + goto FAIL; + } + if (p + 1 >= end || (p[1] & 0xc0) != 0x80) + goto FAIL; + if (c < 0xe0) { + *len = 2; + return ((p[0] & 0x1f) << 6) + ((p[1] & 0x3f)); + } else if (c == 0xe0) { + if ((reinterpret_cast<const unsigned char *>(p))[1] < 0xa0) + goto FAIL; + goto UTF8_3; +#if STRICT_RFC3629 + } else if (c == 0xed) { + // RFC 3629 says surrogate chars are illegal. + if ((reinterpret_cast<const unsigned char *>(p))[1] >= 0xa0) + goto FAIL; + goto UTF8_3; + } else if (c == 0xef) { + // 0xfffe and 0xffff are also illegal characters. + if ((reinterpret_cast<const unsigned char *>(p))[1] == 0xbf && + (reinterpret_cast<const unsigned char *>(p))[2] >= 0xbe) + goto FAIL; + goto UTF8_3; +#endif + } else if (c < 0xf0) { + UTF8_3: + if (p + 2 >= end || (p[2] & 0xc0) != 0x80) + goto FAIL; + *len = 3; + return ((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + ((p[2] & 0x3f)); + } else if (c == 0xf0) { + if ((reinterpret_cast<const unsigned char *>(p))[1] < 0x90) + goto FAIL; + goto UTF8_4; + } else if (c < 0xf4) { + UTF8_4: + if (p + 3 >= end || (p[2] & 0xc0) != 0x80 || (p[3] & 0xc0) != 0x80) + goto FAIL; + *len = 4; +#if STRICT_RFC3629 + // RFC 3629 says all codes ending in fffe or ffff are illegal: + if ((p[1] & 0xf) == 0xf && + (reinterpret_cast<const unsigned char *>(p))[2] == 0xbf && + (reinterpret_cast<const unsigned char *>(p))[3] >= 0xbe) + goto FAIL; +#endif + return ((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12) + + ((p[2] & 0x3f) << 6) + ((p[3] & 0x3f)); + } else if (c == 0xf4) { + if ((reinterpret_cast<const unsigned char *>(p))[1] > 0x8f) + goto FAIL; // After 0x10ffff. + goto UTF8_4; + } else { + FAIL: + *len = 1; +#if ERRORS_TO_ISO8859_1 + return c; +#else + return 0xfffd; // Unicode REPLACEMENT CHARACTER +#endif + } +} + +/************************************************************************/ +/* utf8towc() */ +/************************************************************************/ + +/* Convert a UTF-8 sequence into an array of wchar_t. These + are used by some system calls, especially on Windows. + + \a src points at the UTF-8, and \a srclen is the number of bytes to + convert. + + \a dst points at an array to write, and \a dstlen is the number of + locations in this array. At most \a dstlen-1 words will be + written there, plus a 0 terminating word. Thus this function + will never overwrite the buffer and will always return a + zero-terminated string. If \a dstlen is zero then \a dst can be + null and no data is written, but the length is returned. + + The return value is the number of words that \e would be written + to \a dst if it were long enough, not counting the terminating + zero. If the return value is greater or equal to \a dstlen it + indicates truncation, you can then allocate a new array of size + return+1 and call this again. + + Errors in the UTF-8 are converted as though each byte in the + erroneous string is in the Microsoft CP1252 encoding. This allows + ISO-8859-1 text mistakenly identified as UTF-8 to be printed + correctly. + + Notice that sizeof(wchar_t) is 2 on Windows and is 4 on Linux + and most other systems. Where wchar_t is 16 bits, Unicode + characters in the range 0x10000 to 0x10ffff are converted to + "surrogate pairs" which take two words each (this is called UTF-16 + encoding). If wchar_t is 32 bits this rather nasty problem is + avoided. +*/ +static unsigned utf8towc(const char *src, unsigned srclen, wchar_t *dst, + unsigned dstlen) { + const char *p = src; + const char *e = src + srclen; + unsigned count = 0; + if (dstlen) + while (true) { + if (p >= e) { + dst[count] = 0; + return count; + } + if (!(*p & 0x80)) { + // ASCII + dst[count] = *p++; + } else { + int len = 0; + unsigned ucs = utf8decode(p, e, &len); + p += len; +#ifdef _WIN32 + if (ucs < 0x10000) { + dst[count] = static_cast<wchar_t>(ucs); + } else { + // Make a surrogate pair: + if (count + 2 >= dstlen) { + dst[count] = 0; + count += 2; + break; + } + dst[count] = static_cast<wchar_t>( + (((ucs - 0x10000u) >> 10) & 0x3ff) | 0xd800); + dst[++count] = static_cast<wchar_t>((ucs & 0x3ff) | 0xdc00); + } +#else + dst[count] = static_cast<wchar_t>(ucs); +#endif + } + if (++count == dstlen) { + dst[count - 1] = 0; + break; + } + } + // We filled dst, measure the rest: + while (p < e) { + if (!(*p & 0x80)) { + p++; + } else { + int len = 0; +#ifdef _WIN32 + const unsigned ucs = utf8decode(p, e, &len); + p += len; + if (ucs >= 0x10000) + ++count; +#else + utf8decode(p, e, &len); + p += len; +#endif + } + ++count; + } + + return count; +} + +// --------------------------------------------------------------------------- + +struct NonValidUTF8Exception : public std::exception {}; + +// May throw exceptions +static std::wstring UTF8ToWString(const std::string &str) { + std::wstring wstr; + wstr.resize(str.size()); + wstr.resize(utf8towc(str.data(), static_cast<unsigned>(str.size()), + &wstr[0], static_cast<unsigned>(wstr.size()) + 1)); + for (const auto ch : wstr) { + if (ch == 0xfffd) { + throw NonValidUTF8Exception(); + } + } + return wstr; +} + +// --------------------------------------------------------------------------- + +/************************************************************************/ +/* utf8fromwc() */ +/************************************************************************/ +/* Turn "wide characters" as returned by some system calls + (especially on Windows) into UTF-8. + + Up to \a dstlen bytes are written to \a dst, including a null + terminator. The return value is the number of bytes that would be + written, not counting the null terminator. If greater or equal to + \a dstlen then if you malloc a new array of size n+1 you will have + the space needed for the entire string. If \a dstlen is zero then + nothing is written and this call just measures the storage space + needed. + + \a srclen is the number of words in \a src to convert. On Windows + this is not necessarily the number of characters, due to there + possibly being "surrogate pairs" in the UTF-16 encoding used. + On Unix wchar_t is 32 bits and each location is a character. + + On Unix if a src word is greater than 0x10ffff then this is an + illegal character according to RFC 3629. These are converted as + though they are 0xFFFD (REPLACEMENT CHARACTER). Characters in the + range 0xd800 to 0xdfff, or ending with 0xfffe or 0xffff are also + illegal according to RFC 3629. However I encode these as though + they are legal, so that utf8towc will return the original data. + + On Windows "surrogate pairs" are converted to a single character + and UTF-8 encoded (as 4 bytes). Mismatched halves of surrogate + pairs are converted as though they are individual characters. +*/ +static unsigned int utf8fromwc(char *dst, unsigned dstlen, const wchar_t *src, + unsigned srclen) { + unsigned int i = 0; + unsigned int count = 0; + if (dstlen) + while (true) { + if (i >= srclen) { + dst[count] = 0; + return count; + } + unsigned int ucs = src[i++]; + if (ucs < 0x80U) { + dst[count++] = static_cast<char>(ucs); + if (count >= dstlen) { + dst[count - 1] = 0; + break; + } + } else if (ucs < 0x800U) { + // 2 bytes. + if (count + 2 >= dstlen) { + dst[count] = 0; + count += 2; + break; + } + dst[count++] = 0xc0 | static_cast<char>(ucs >> 6); + dst[count++] = 0x80 | static_cast<char>(ucs & 0x3F); +#ifdef _WIN32 + } else if (ucs >= 0xd800 && ucs <= 0xdbff && i < srclen && + src[i] >= 0xdc00 && src[i] <= 0xdfff) { + // Surrogate pair. + unsigned int ucs2 = src[i++]; + ucs = 0x10000U + ((ucs & 0x3ff) << 10) + (ucs2 & 0x3ff); +// All surrogate pairs turn into 4-byte utf8. +#else + } else if (ucs >= 0x10000) { + if (ucs > 0x10ffff) { + ucs = 0xfffd; + goto J1; + } +#endif + if (count + 4 >= dstlen) { + dst[count] = 0; + count += 4; + break; + } + dst[count++] = 0xf0 | static_cast<char>(ucs >> 18); + dst[count++] = 0x80 | static_cast<char>((ucs >> 12) & 0x3F); + dst[count++] = 0x80 | static_cast<char>((ucs >> 6) & 0x3F); + dst[count++] = 0x80 | static_cast<char>(ucs & 0x3F); + } else { +#ifndef _WIN32 + J1: +#endif + // All others are 3 bytes: + if (count + 3 >= dstlen) { + dst[count] = 0; + count += 3; + break; + } + dst[count++] = 0xe0 | static_cast<char>(ucs >> 12); + dst[count++] = 0x80 | static_cast<char>((ucs >> 6) & 0x3F); + dst[count++] = 0x80 | static_cast<char>(ucs & 0x3F); + } + } + + // We filled dst, measure the rest: + while (i < srclen) { + unsigned int ucs = src[i++]; + if (ucs < 0x80U) { + count++; + } else if (ucs < 0x800U) { + // 2 bytes. + count += 2; +#ifdef _WIN32 + } else if (ucs >= 0xd800 && ucs <= 0xdbff && i < srclen - 1 && + src[i + 1] >= 0xdc00 && src[i + 1] <= 0xdfff) { + // Surrogate pair. + ++i; +#else + } else if (ucs >= 0x10000 && ucs <= 0x10ffff) { +#endif + count += 4; + } else { + count += 3; + } + } + return count; +} + +// --------------------------------------------------------------------------- + +static std::string WStringToUTF8(const std::wstring &wstr) { + std::string str; + str.resize(wstr.size()); + str.resize(utf8fromwc(&str[0], static_cast<unsigned>(str.size() + 1), + wstr.data(), static_cast<unsigned>(wstr.size()))); + return str; +} + +// --------------------------------------------------------------------------- + +static std::string Win32Recode(const char *src, unsigned src_code_page, + unsigned dst_code_page) { + // Convert from source code page to Unicode. + + // Compute the length in wide characters. + int wlen = MultiByteToWideChar(src_code_page, MB_ERR_INVALID_CHARS, src, -1, + nullptr, 0); + if (wlen == 0 && GetLastError() == ERROR_NO_UNICODE_TRANSLATION) { + return std::string(); + } + + // Do the actual conversion. + std::wstring wbuf; + wbuf.resize(wlen); + MultiByteToWideChar(src_code_page, 0, src, -1, &wbuf[0], wlen); + + // Convert from Unicode to destination code page. + + // Compute the length in chars. + int len = WideCharToMultiByte(dst_code_page, 0, &wbuf[0], -1, nullptr, 0, + nullptr, nullptr); + + // Do the actual conversion. + std::string out; + out.resize(len); + WideCharToMultiByte(dst_code_page, 0, &wbuf[0], -1, &out[0], len, nullptr, + nullptr); + out.resize(strlen(out.c_str())); + + return out; +} + +// --------------------------------------------------------------------------- + +class FileWin32 : public File { + PJ_CONTEXT *m_ctx; + HANDLE m_handle; + + FileWin32(const FileWin32 &) = delete; + FileWin32 &operator=(const FileWin32 &) = delete; + + protected: + FileWin32(const std::string &name, PJ_CONTEXT *ctx, HANDLE handle) + : File(name), m_ctx(ctx), m_handle(handle) {} + + public: + ~FileWin32() override; + + size_t read(void *buffer, size_t sizeBytes) override; + size_t write(const void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } + + // We may lie, but the real use case is only for network files + bool hasChanged() const override { return false; } + + static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename, + FileAccess access); +}; + +// --------------------------------------------------------------------------- + +FileWin32::~FileWin32() { CloseHandle(m_handle); } + +// --------------------------------------------------------------------------- + +size_t FileWin32::read(void *buffer, size_t sizeBytes) { + DWORD dwSizeRead = 0; + size_t nResult = 0; + + if (!ReadFile(m_handle, buffer, static_cast<DWORD>(sizeBytes), &dwSizeRead, + nullptr)) + nResult = 0; + else + nResult = dwSizeRead; + + return nResult; +} + +// --------------------------------------------------------------------------- + +size_t FileWin32::write(const void *buffer, size_t sizeBytes) { + DWORD dwSizeWritten = 0; + size_t nResult = 0; + + if (!WriteFile(m_handle, buffer, static_cast<DWORD>(sizeBytes), + &dwSizeWritten, nullptr)) + nResult = 0; + else + nResult = dwSizeWritten; + + return nResult; +} + +// --------------------------------------------------------------------------- + +bool FileWin32::seek(unsigned long long offset, int whence) { + LONG dwMoveMethod, dwMoveHigh; + uint32_t nMoveLow; + LARGE_INTEGER li; + + switch (whence) { + case SEEK_CUR: + dwMoveMethod = FILE_CURRENT; + break; + case SEEK_END: + dwMoveMethod = FILE_END; + break; + case SEEK_SET: + default: + dwMoveMethod = FILE_BEGIN; + break; + } + + li.QuadPart = offset; + nMoveLow = li.LowPart; + dwMoveHigh = li.HighPart; + + SetLastError(0); + SetFilePointer(m_handle, nMoveLow, &dwMoveHigh, dwMoveMethod); + + return GetLastError() == NO_ERROR; +} + +// --------------------------------------------------------------------------- + +unsigned long long FileWin32::tell() { + LARGE_INTEGER li; + + li.HighPart = 0; + li.LowPart = SetFilePointer(m_handle, 0, &(li.HighPart), FILE_CURRENT); + + return static_cast<unsigned long long>(li.QuadPart); +} +// --------------------------------------------------------------------------- + +std::unique_ptr<File> FileWin32::open(PJ_CONTEXT *ctx, const char *filename, + FileAccess access) { + DWORD dwDesiredAccess = access == FileAccess::READ_ONLY + ? GENERIC_READ + : GENERIC_READ | GENERIC_WRITE; + DWORD dwCreationDisposition = + access == FileAccess::CREATE ? CREATE_ALWAYS : OPEN_EXISTING; + DWORD dwFlagsAndAttributes = (dwDesiredAccess == GENERIC_READ) + ? FILE_ATTRIBUTE_READONLY + : FILE_ATTRIBUTE_NORMAL; + try { + HANDLE hFile = CreateFileW( + UTF8ToWString(std::string(filename)).c_str(), dwDesiredAccess, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, + dwCreationDisposition, dwFlagsAndAttributes, nullptr); + return std::unique_ptr<File>(hFile != INVALID_HANDLE_VALUE + ? new FileWin32(filename, ctx, hFile) + : nullptr); + } catch (const std::exception &e) { + pj_log(ctx, PJ_LOG_DEBUG, "%s", e.what()); + return nullptr; + } +} +#else + +// --------------------------------------------------------------------------- + +class FileStdio : public File { + PJ_CONTEXT *m_ctx; + FILE *m_fp; + + FileStdio(const FileStdio &) = delete; + FileStdio &operator=(const FileStdio &) = delete; + + protected: + FileStdio(const std::string &name, PJ_CONTEXT *ctx, FILE *fp) + : File(name), m_ctx(ctx), m_fp(fp) {} + + public: + ~FileStdio() override; + + size_t read(void *buffer, size_t sizeBytes) override; + size_t write(const void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } + + // We may lie, but the real use case is only for network files + bool hasChanged() const override { return false; } + + static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename, + FileAccess access); +}; + +// --------------------------------------------------------------------------- + +FileStdio::~FileStdio() { fclose(m_fp); } + +// --------------------------------------------------------------------------- + +size_t FileStdio::read(void *buffer, size_t sizeBytes) { + return fread(buffer, 1, sizeBytes, m_fp); +} + +// --------------------------------------------------------------------------- + +size_t FileStdio::write(const void *buffer, size_t sizeBytes) { + return fwrite(buffer, 1, sizeBytes, m_fp); +} + +// --------------------------------------------------------------------------- + +bool FileStdio::seek(unsigned long long offset, int whence) { + // TODO one day: use 64-bit offset compatible API + if (offset != static_cast<unsigned long long>(static_cast<long>(offset))) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Attempt at seeking to a 64 bit offset. Not supported yet"); + return false; + } + return fseek(m_fp, static_cast<long>(offset), whence) == 0; +} + +// --------------------------------------------------------------------------- + +unsigned long long FileStdio::tell() { + // TODO one day: use 64-bit offset compatible API + return ftell(m_fp); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<File> FileStdio::open(PJ_CONTEXT *ctx, const char *filename, + FileAccess access) { + auto fp = fopen(filename, + access == FileAccess::READ_ONLY + ? "rb" + : access == FileAccess::READ_UPDATE ? "r+b" : "w+b"); + return std::unique_ptr<File>(fp ? new FileStdio(filename, ctx, fp) + : nullptr); +} + +#endif // _WIN32 + +// --------------------------------------------------------------------------- + +#ifndef REMOVE_LEGACY_SUPPORT + +class FileLegacyAdapter : public File { + PJ_CONTEXT *m_ctx; + PAFile m_fp; + + FileLegacyAdapter(const FileLegacyAdapter &) = delete; + FileLegacyAdapter &operator=(const FileLegacyAdapter &) = delete; + + protected: + FileLegacyAdapter(const std::string &name, PJ_CONTEXT *ctx, PAFile fp) + : File(name), m_ctx(ctx), m_fp(fp) {} + + public: + ~FileLegacyAdapter() override; + + size_t read(void *buffer, size_t sizeBytes) override; + size_t write(const void *, size_t) override { return 0; } + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } + + // We may lie, but the real use case is only for network files + bool hasChanged() const override { return false; } + + static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename, + FileAccess access); +}; + +// --------------------------------------------------------------------------- + +FileLegacyAdapter::~FileLegacyAdapter() { pj_ctx_fclose(m_ctx, m_fp); } + +// --------------------------------------------------------------------------- + +size_t FileLegacyAdapter::read(void *buffer, size_t sizeBytes) { + return pj_ctx_fread(m_ctx, buffer, 1, sizeBytes, m_fp); +} + +// --------------------------------------------------------------------------- + +bool FileLegacyAdapter::seek(unsigned long long offset, int whence) { + if (offset != static_cast<unsigned long long>(static_cast<long>(offset))) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Attempt at seeking to a 64 bit offset. Not supported yet"); + return false; + } + return pj_ctx_fseek(m_ctx, m_fp, static_cast<long>(offset), whence) == 0; +} + +// --------------------------------------------------------------------------- + +unsigned long long FileLegacyAdapter::tell() { + return pj_ctx_ftell(m_ctx, m_fp); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<File> +FileLegacyAdapter::open(PJ_CONTEXT *ctx, const char *filename, FileAccess) { + auto fid = pj_ctx_fopen(ctx, filename, "rb"); + return std::unique_ptr<File>(fid ? new FileLegacyAdapter(filename, ctx, fid) + : nullptr); +} + +#endif // REMOVE_LEGACY_SUPPORT + +// --------------------------------------------------------------------------- + +class FileApiAdapter : public File { + PJ_CONTEXT *m_ctx; + PROJ_FILE_HANDLE *m_fp; + + FileApiAdapter(const FileApiAdapter &) = delete; + FileApiAdapter &operator=(const FileApiAdapter &) = delete; + + protected: + FileApiAdapter(const std::string &name, PJ_CONTEXT *ctx, + PROJ_FILE_HANDLE *fp) + : File(name), m_ctx(ctx), m_fp(fp) {} + + public: + ~FileApiAdapter() override; + + size_t read(void *buffer, size_t sizeBytes) override; + size_t write(const void *, size_t) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } + + // We may lie, but the real use case is only for network files + bool hasChanged() const override { return false; } + + static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename, + FileAccess access); +}; + +// --------------------------------------------------------------------------- + +FileApiAdapter::~FileApiAdapter() { + m_ctx->fileApi.close_cbk(m_ctx, m_fp, m_ctx->fileApi.user_data); +} + +// --------------------------------------------------------------------------- + +size_t FileApiAdapter::read(void *buffer, size_t sizeBytes) { + return m_ctx->fileApi.read_cbk(m_ctx, m_fp, buffer, sizeBytes, + m_ctx->fileApi.user_data); +} + +// --------------------------------------------------------------------------- + +size_t FileApiAdapter::write(const void *buffer, size_t sizeBytes) { + return m_ctx->fileApi.write_cbk(m_ctx, m_fp, buffer, sizeBytes, + m_ctx->fileApi.user_data); +} + +// --------------------------------------------------------------------------- + +bool FileApiAdapter::seek(unsigned long long offset, int whence) { + return m_ctx->fileApi.seek_cbk(m_ctx, m_fp, static_cast<long long>(offset), + whence, m_ctx->fileApi.user_data) != 0; +} + +// --------------------------------------------------------------------------- + +unsigned long long FileApiAdapter::tell() { + return m_ctx->fileApi.tell_cbk(m_ctx, m_fp, m_ctx->fileApi.user_data); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<File> FileApiAdapter::open(PJ_CONTEXT *ctx, + const char *filename, + FileAccess eAccess) { + PROJ_OPEN_ACCESS eCAccess = PROJ_OPEN_ACCESS_READ_ONLY; + switch (eAccess) { + case FileAccess::READ_ONLY: + // Initialized above + break; + case FileAccess::READ_UPDATE: + eCAccess = PROJ_OPEN_ACCESS_READ_UPDATE; + break; + case FileAccess::CREATE: + eCAccess = PROJ_OPEN_ACCESS_CREATE; + break; + } + auto fp = + ctx->fileApi.open_cbk(ctx, filename, eCAccess, ctx->fileApi.user_data); + return std::unique_ptr<File>(fp ? new FileApiAdapter(filename, ctx, fp) + : nullptr); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<File> FileManager::open(PJ_CONTEXT *ctx, const char *filename, + FileAccess access) { + if (starts_with(filename, "http://") || starts_with(filename, "https://")) { + if (!pj_context_is_network_enabled(ctx)) { + pj_log( + ctx, PJ_LOG_ERROR, + "Attempt at accessing remote resource not authorized. Either " + "set PROJ_NETWORK=ON or " + "proj_context_set_enable_network(ctx, TRUE)"); + return nullptr; + } + return pj_network_file_open(ctx, filename); + } +#ifndef REMOVE_LEGACY_SUPPORT + // If the user has specified a legacy fileapi, use it + if (ctx->fileapi_legacy != pj_get_default_fileapi()) { + return FileLegacyAdapter::open(ctx, filename, access); + } +#endif + if (ctx->fileApi.open_cbk != nullptr) { + return FileApiAdapter::open(ctx, filename, access); + } +#ifdef _WIN32 + return FileWin32::open(ctx, filename, access); +#else + return FileStdio::open(ctx, filename, access); +#endif +} + +// --------------------------------------------------------------------------- + +bool FileManager::exists(PJ_CONTEXT *ctx, const char *filename) { + if (ctx->fileApi.exists_cbk) { + return ctx->fileApi.exists_cbk(ctx, filename, ctx->fileApi.user_data) != + 0; + } + +#ifdef _WIN32 + struct __stat64 buf; + try { + return _wstat64(UTF8ToWString(filename).c_str(), &buf) == 0; + } catch (const std::exception &e) { + pj_log(ctx, PJ_LOG_DEBUG, "%s", e.what()); + return false; + } +#else + (void)ctx; + struct stat sStat; + return stat(filename, &sStat) == 0; +#endif +} + +// --------------------------------------------------------------------------- + +bool FileManager::mkdir(PJ_CONTEXT *ctx, const char *filename) { + if (ctx->fileApi.mkdir_cbk) { + return ctx->fileApi.mkdir_cbk(ctx, filename, ctx->fileApi.user_data) != + 0; + } + +#ifdef _WIN32 + try { + return _wmkdir(UTF8ToWString(filename).c_str()) == 0; + } catch (const std::exception &e) { + pj_log(ctx, PJ_LOG_DEBUG, "%s", e.what()); + return false; + } +#else + (void)ctx; + return ::mkdir(filename, 0755) == 0; +#endif +} + +// --------------------------------------------------------------------------- + +bool FileManager::unlink(PJ_CONTEXT *ctx, const char *filename) { + if (ctx->fileApi.unlink_cbk) { + return ctx->fileApi.unlink_cbk(ctx, filename, ctx->fileApi.user_data) != + 0; + } + +#ifdef _WIN32 + try { + return _wunlink(UTF8ToWString(filename).c_str()) == 0; + } catch (const std::exception &e) { + pj_log(ctx, PJ_LOG_DEBUG, "%s", e.what()); + return false; + } +#else + (void)ctx; + return ::unlink(filename) == 0; +#endif +} + +// --------------------------------------------------------------------------- + +bool FileManager::rename(PJ_CONTEXT *ctx, const char *oldPath, + const char *newPath) { + if (ctx->fileApi.rename_cbk) { + return ctx->fileApi.rename_cbk(ctx, oldPath, newPath, + ctx->fileApi.user_data) != 0; + } + +#ifdef _WIN32 + try { + return _wrename(UTF8ToWString(oldPath).c_str(), + UTF8ToWString(newPath).c_str()) == 0; + } catch (const std::exception &e) { + pj_log(ctx, PJ_LOG_DEBUG, "%s", e.what()); + return false; + } +#else + (void)ctx; + return ::rename(oldPath, newPath) == 0; +#endif +} + +// --------------------------------------------------------------------------- + +std::string FileManager::getProjLibEnvVar(PJ_CONTEXT *ctx) { + if (!ctx->env_var_proj_lib.empty()) { + return ctx->env_var_proj_lib; + } + (void)ctx; + std::string str; + const char *envvar = getenv("PROJ_LIB"); + if (!envvar) + return str; + str = envvar; +#ifdef _WIN32 + // Assume this is UTF-8. If not try to convert from ANSI page + bool looksLikeUTF8 = false; + try { + UTF8ToWString(envvar); + looksLikeUTF8 = true; + } catch (const std::exception &) { + } + if (!looksLikeUTF8 || !exists(ctx, envvar)) { + str = Win32Recode(envvar, CP_ACP, CP_UTF8); + if (str.empty() || !exists(ctx, str.c_str())) + str = envvar; + } +#endif + ctx->env_var_proj_lib = str; + return str; +} + +NS_PROJ_END + +//! @endcond + +// --------------------------------------------------------------------------- + +/** Set a file API + * + * All callbacks should be provided (non NULL pointers). If read-only usage + * is intended, then the callbacks might have a dummy implementation. + * + * \note Those callbacks will not be used for SQLite3 database access. If + * custom I/O is desired for that, then proj_context_set_sqlite3_vfs_name() + * should be used. + * + * @param ctx PROJ context, or NULL + * @param fileapi Pointer to file API structure (content will be copied). + * @param user_data Arbitrary pointer provided by the user, and passed to the + * above callbacks. May be NULL. + * @return TRUE in case of success. + * @since 7.0 + */ +int proj_context_set_fileapi(PJ_CONTEXT *ctx, const PROJ_FILE_API *fileapi, + void *user_data) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (!fileapi) { + return false; + } + if (fileapi->version != 1) { + return false; + } + if (!fileapi->open_cbk || !fileapi->close_cbk || !fileapi->read_cbk || + !fileapi->write_cbk || !fileapi->seek_cbk || !fileapi->tell_cbk || + !fileapi->exists_cbk || !fileapi->mkdir_cbk || !fileapi->unlink_cbk || + !fileapi->rename_cbk) { + return false; + } + ctx->fileApi.open_cbk = fileapi->open_cbk; + ctx->fileApi.close_cbk = fileapi->close_cbk; + ctx->fileApi.read_cbk = fileapi->read_cbk; + ctx->fileApi.write_cbk = fileapi->write_cbk; + ctx->fileApi.seek_cbk = fileapi->seek_cbk; + ctx->fileApi.tell_cbk = fileapi->tell_cbk; + ctx->fileApi.exists_cbk = fileapi->exists_cbk; + ctx->fileApi.mkdir_cbk = fileapi->mkdir_cbk; + ctx->fileApi.unlink_cbk = fileapi->unlink_cbk; + ctx->fileApi.rename_cbk = fileapi->rename_cbk; + ctx->fileApi.user_data = user_data; + return true; +} + +// --------------------------------------------------------------------------- + +/** Set the name of a custom SQLite3 VFS. + * + * This should be a valid SQLite3 VFS name, such as the one passed to the + * sqlite3_vfs_register(). See https://www.sqlite.org/vfs.html + * + * It will be used to read proj.db or create&access the cache.db file in the + * PROJ user writable directory. + * + * @param ctx PROJ context, or NULL + * @param name SQLite3 VFS name. If NULL is passed, default implementation by + * SQLite will be used. + * @since 7.0 + */ +void proj_context_set_sqlite3_vfs_name(PJ_CONTEXT *ctx, const char *name) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + ctx->custom_sqlite3_vfs_name = name ? name : std::string(); +} + +// --------------------------------------------------------------------------- + +//! @cond Doxygen_Suppress + +// --------------------------------------------------------------------------- + +static void CreateDirectoryRecursively(PJ_CONTEXT *ctx, + const std::string &path) { + if (NS_PROJ::FileManager::exists(ctx, path.c_str())) + return; + auto pos = path.find_last_of("/\\"); + if (pos == 0 || pos == std::string::npos) + return; + CreateDirectoryRecursively(ctx, path.substr(0, pos)); + NS_PROJ::FileManager::mkdir(ctx, path.c_str()); +} + +// --------------------------------------------------------------------------- + +std::string pj_context_get_user_writable_directory(PJ_CONTEXT *ctx, + bool create) { + if (ctx->user_writable_directory.empty()) { + // For testing purposes only + const char *env_var_PROJ_USER_WRITABLE_DIRECTORY = + getenv("PROJ_USER_WRITABLE_DIRECTORY"); + if (env_var_PROJ_USER_WRITABLE_DIRECTORY && + env_var_PROJ_USER_WRITABLE_DIRECTORY[0] != '\0') { + ctx->user_writable_directory = env_var_PROJ_USER_WRITABLE_DIRECTORY; + } + } + if (ctx->user_writable_directory.empty()) { + std::string path; +#ifdef _WIN32 + std::wstring wPath; + wPath.resize(MAX_PATH); + if (SHGetFolderPathW(nullptr, CSIDL_LOCAL_APPDATA, nullptr, 0, + &wPath[0]) == S_OK) { + wPath.resize(wcslen(wPath.data())); + path = NS_PROJ::WStringToUTF8(wPath); + } else { + const char *local_app_data = getenv("LOCALAPPDATA"); + if (!local_app_data) { + local_app_data = getenv("TEMP"); + if (!local_app_data) { + local_app_data = "c:/users"; + } + } + path = local_app_data; + } +#else + const char *xdg_data_home = getenv("XDG_DATA_HOME"); + if (xdg_data_home != nullptr) { + path = xdg_data_home; + } else { + const char *home = getenv("HOME"); + if (home) { +#if defined(__MACH__) && defined(__APPLE__) + path = std::string(home) + "/Library/Application Support"; +#else + path = std::string(home) + "/.local/share"; +#endif + } else { + path = "/tmp"; + } + } +#endif + path += "/proj"; + ctx->user_writable_directory = path; + } + if (create) { + CreateDirectoryRecursively(ctx, ctx->user_writable_directory); + } + return ctx->user_writable_directory; +} + +// --------------------------------------------------------------------------- + +#ifdef WIN32 +static const char dir_chars[] = "/\\"; +#else +static const char dir_chars[] = "/"; +#endif + +static bool is_tilde_slash(const char *name) { + return *name == '~' && strchr(dir_chars, name[1]); +} + +static bool is_rel_or_absolute_filename(const char *name) { + return strchr(dir_chars, *name) || + (*name == '.' && strchr(dir_chars, name[1])) || + (!strncmp(name, "..", 2) && strchr(dir_chars, name[2])) || + (name[0] != '\0' && name[1] == ':' && strchr(dir_chars, name[2])); +} + +// --------------------------------------------------------------------------- + +#ifdef _WIN32 +static const char *get_path_from_win32_projlib(PJ_CONTEXT *ctx, + const char *name, + std::string &out) { + /* Check if proj.db lieves in a share/proj dir parallel to bin/proj.dll */ + /* Based in + * https://stackoverflow.com/questions/9112893/how-to-get-path-to-executable-in-c-running-on-windows + */ + + DWORD path_size = 1024; + + std::wstring wout; + for (;;) { + wout.clear(); + wout.resize(path_size); + DWORD result = GetModuleFileNameW(nullptr, &wout[0], path_size - 1); + DWORD last_error = GetLastError(); + + if (result == 0) { + return nullptr; + } else if (result == path_size - 1) { + if (ERROR_INSUFFICIENT_BUFFER != last_error) { + return nullptr; + } + path_size = path_size * 2; + } else { + break; + } + } + // Now remove the program's name. It was (example) + // "C:\programs\gmt6\bin\gdal_translate.exe" + wout.resize(wcslen(wout.c_str())); + out = NS_PROJ::WStringToUTF8(wout); + size_t k = out.size(); + while (k > 0 && out[--k] != '\\') { + } + out.resize(k); + + out += "/../share/proj/"; + out += name; + + return NS_PROJ::FileManager::exists(ctx, out.c_str()) ? out.c_str() + : nullptr; +} +#endif + +/************************************************************************/ +/* pj_open_lib_internal() */ +/************************************************************************/ + +#ifdef WIN32 +static const char dirSeparator = ';'; +#else +static const char dirSeparator = ':'; +#endif + +static const char *proj_lib_name = +#ifdef PROJ_LIB + PROJ_LIB; +#else + nullptr; +#endif + +static bool dontReadUserWritableDirectory() { + // Env var mostly for testing purposes and being independent from + // an existing installation + const char *envVar = getenv("PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY"); + return envVar != nullptr && envVar[0] != '\0'; +} + +static void * +pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, + void *(*open_file)(projCtx, const char *, const char *), + char *out_full_filename, size_t out_full_filename_size) { + try { + std::string fname; + const char *sysname = nullptr; + void *fid = nullptr; + std::string projLib; + + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + + if (out_full_filename != nullptr && out_full_filename_size > 0) + out_full_filename[0] = '\0'; + + /* check if ~/name */ + if (is_tilde_slash(name)) + if ((sysname = getenv("HOME")) != nullptr) { + fname = sysname; + fname += DIR_CHAR; + fname += name; + sysname = fname.c_str(); + } else + return nullptr; + + /* or fixed path: /name, ./name or ../name */ + else if (is_rel_or_absolute_filename(name)) { + sysname = name; +#ifdef _WIN32 + try { + NS_PROJ::UTF8ToWString(name); + } catch (const std::exception &) { + fname = NS_PROJ::Win32Recode(name, CP_ACP, CP_UTF8); + sysname = fname.c_str(); + } +#endif + } + + else if (starts_with(name, "http://") || starts_with(name, "https://")) + sysname = name; + + /* or try to use application provided file finder */ + else if (ctx->file_finder != nullptr && + (sysname = ctx->file_finder( + ctx, name, ctx->file_finder_user_data)) != nullptr) + ; + + else if (ctx->file_finder_legacy != nullptr && + (sysname = ctx->file_finder_legacy(name)) != nullptr) + ; + + /* The user has search paths set */ + else if (!ctx->search_paths.empty()) { + for (const auto &path : ctx->search_paths) { + try { + fname = path; + fname += DIR_CHAR; + fname += name; + sysname = fname.c_str(); + fid = open_file(ctx, sysname, mode); + } catch (const std::exception &) { + } + if (fid) + break; + } + } + + else if (!dontReadUserWritableDirectory() && + (fid = open_file( + ctx, (pj_context_get_user_writable_directory(ctx, false) + + DIR_CHAR + name) + .c_str(), + mode)) != nullptr) { + fname = pj_context_get_user_writable_directory(ctx, false); + fname += DIR_CHAR; + fname += name; + sysname = fname.c_str(); + } + + /* if is environment PROJ_LIB defined */ + else if (!(projLib = NS_PROJ::FileManager::getProjLibEnvVar(ctx)) + .empty()) { + auto paths = NS_PROJ::internal::split(projLib, dirSeparator); + for (const auto &path : paths) { + fname = path; + fname += DIR_CHAR; + fname += name; + sysname = fname.c_str(); + fid = open_file(ctx, sysname, mode); + if (fid) + break; + } +#ifdef _WIN32 + /* check if it lives in a ../share/proj dir of the proj dll */ + } else if ((sysname = get_path_from_win32_projlib(ctx, name, fname)) != + nullptr) { +#endif + /* or hardcoded path */ + } else if ((sysname = proj_lib_name) != nullptr) { + fname = sysname; + fname += DIR_CHAR; + fname += name; + sysname = fname.c_str(); + /* just try it bare bones */ + } else { + sysname = name; + } + + assert(sysname); // to make Coverity Scan happy + if (fid != nullptr || + (fid = open_file(ctx, sysname, mode)) != nullptr) { + if (out_full_filename != nullptr && out_full_filename_size > 0) { + // cppcheck-suppress nullPointer + strncpy(out_full_filename, sysname, out_full_filename_size); + out_full_filename[out_full_filename_size - 1] = '\0'; + } + errno = 0; + } + + if (ctx->last_errno == 0 && errno != 0) + pj_ctx_set_errno(ctx, errno); + + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "pj_open_lib(%s): call fopen(%s) - %s", + name, sysname, fid == nullptr ? "failed" : "succeeded"); + + return (fid); + } catch (const std::exception &) { + + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "pj_open_lib(%s): out of memory", name); + + return nullptr; + } +} + +/************************************************************************/ +/* pj_open_file_with_manager() */ +/************************************************************************/ + +static void *pj_open_file_with_manager(projCtx ctx, const char *name, + const char * /* mode */) { + return NS_PROJ::FileManager::open(ctx, name, NS_PROJ::FileAccess::READ_ONLY) + .release(); +} + +/************************************************************************/ +/* FileManager::open_resource_file() */ +/************************************************************************/ + +std::unique_ptr<NS_PROJ::File> +NS_PROJ::FileManager::open_resource_file(projCtx ctx, const char *name) { + + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + + auto file = std::unique_ptr<NS_PROJ::File>( + reinterpret_cast<NS_PROJ::File *>(pj_open_lib_internal( + ctx, name, "rb", pj_open_file_with_manager, nullptr, 0))); + + // Retry with a .tif extension if the file name doesn't end with .tif + if (file == nullptr && !is_tilde_slash(name) && + !is_rel_or_absolute_filename(name) && !starts_with(name, "http://") && + !starts_with(name, "https://") && strcmp(name, "proj.db") != 0 && + strstr(name, ".tif") == nullptr) { + std::string filename(name); + auto pos = filename.rfind('.'); + if (pos + 4 == filename.size()) { + filename = filename.substr(0, pos) + ".tif"; + file.reset(reinterpret_cast<NS_PROJ::File *>( + pj_open_lib_internal(ctx, filename.c_str(), "rb", + pj_open_file_with_manager, nullptr, 0))); + } else { + // For example for resource files like 'alaska' + filename += ".tif"; + file.reset(reinterpret_cast<NS_PROJ::File *>( + pj_open_lib_internal(ctx, filename.c_str(), "rb", + pj_open_file_with_manager, nullptr, 0))); + } + if (file) { + pj_ctx_set_errno(ctx, 0); + } + } + + if (file == nullptr && !is_tilde_slash(name) && + !is_rel_or_absolute_filename(name) && !starts_with(name, "http://") && + !starts_with(name, "https://") && pj_context_is_network_enabled(ctx)) { + std::string remote_file(pj_context_get_url_endpoint(ctx)); + if (!remote_file.empty()) { + if (remote_file.back() != '/') { + remote_file += '/'; + } + remote_file += name; + auto pos = remote_file.rfind('.'); + if (pos + 4 == remote_file.size()) { + remote_file = remote_file.substr(0, pos) + ".tif"; + file = open(ctx, remote_file.c_str(), + NS_PROJ::FileAccess::READ_ONLY); + if (file) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Using %s", + remote_file.c_str()); + pj_ctx_set_errno(ctx, 0); + } + } else { + // For example for resource files like 'alaska' + auto remote_file_tif = remote_file + ".tif"; + file = open(ctx, remote_file_tif.c_str(), + NS_PROJ::FileAccess::READ_ONLY); + if (file) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Using %s", + remote_file_tif.c_str()); + pj_ctx_set_errno(ctx, 0); + } else { + // Init files + file = open(ctx, remote_file.c_str(), + NS_PROJ::FileAccess::READ_ONLY); + if (file) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Using %s", + remote_file.c_str()); + pj_ctx_set_errno(ctx, 0); + } + } + } + } + } + return file; +} + +/************************************************************************/ +/* pj_open_lib() */ +/************************************************************************/ + +#ifndef REMOVE_LEGACY_SUPPORT + +// Used by following legacy function +static void *pj_ctx_fopen_adapter(projCtx ctx, const char *name, + const char *mode) { + return pj_ctx_fopen(ctx, name, mode); +} + +// Legacy function +PAFile pj_open_lib(projCtx ctx, const char *name, const char *mode) { + return (PAFile)pj_open_lib_internal(ctx, name, mode, pj_ctx_fopen_adapter, + nullptr, 0); +} + +#endif // REMOVE_LEGACY_SUPPORT + +/************************************************************************/ +/* pj_find_file() */ +/************************************************************************/ + +/** Returns the full filename corresponding to a proj resource file specified + * as a short filename. + * + * @param ctx context. + * @param short_filename short filename (e.g. egm96_15.gtx). Must not be NULL. + * @param out_full_filename output buffer, of size out_full_filename_size, that + * will receive the full filename on success. + * Will be zero-terminated. + * @param out_full_filename_size size of out_full_filename. + * @return 1 if the file was found, 0 otherwise. + */ +int pj_find_file(projCtx ctx, const char *short_filename, + char *out_full_filename, size_t out_full_filename_size) { + auto f = reinterpret_cast<NS_PROJ::File *>(pj_open_lib_internal( + ctx, short_filename, "rb", pj_open_file_with_manager, out_full_filename, + out_full_filename_size)); + if (f != nullptr) { + delete f; + return 1; + } + return 0; +} + +/************************************************************************/ +/* pj_context_get_url_endpoint() */ +/************************************************************************/ + +std::string pj_context_get_url_endpoint(PJ_CONTEXT *ctx) { + if (!ctx->endpoint.empty()) { + return ctx->endpoint; + } + pj_load_ini(ctx); + return ctx->endpoint; +} + +/************************************************************************/ +/* trim() */ +/************************************************************************/ + +static std::string trim(const std::string &s) { + const auto first = s.find_first_not_of(' '); + const auto last = s.find_last_not_of(' '); + if (first == std::string::npos || last == std::string::npos) { + return std::string(); + } + return s.substr(first, last - first + 1); +} + +/************************************************************************/ +/* pj_load_ini() */ +/************************************************************************/ + +void pj_load_ini(projCtx ctx) { + if (ctx->iniFileLoaded) + return; + + const char *endpoint_from_env = getenv("PROJ_NETWORK_ENDPOINT"); + if (endpoint_from_env && endpoint_from_env[0] != '\0') { + ctx->endpoint = endpoint_from_env; + } + + ctx->iniFileLoaded = true; + auto file = std::unique_ptr<NS_PROJ::File>( + reinterpret_cast<NS_PROJ::File *>(pj_open_lib_internal( + ctx, "proj.ini", "rb", pj_open_file_with_manager, nullptr, 0))); + if (!file) + return; + file->seek(0, SEEK_END); + const auto filesize = file->tell(); + if (filesize == 0 || filesize > 100 * 1024U) + return; + file->seek(0, SEEK_SET); + std::string content; + content.resize(static_cast<size_t>(filesize)); + const auto nread = file->read(&content[0], content.size()); + if (nread != content.size()) + return; + content += '\n'; + size_t pos = 0; + while (pos != std::string::npos) { + const auto eol = content.find_first_of("\r\n", pos); + if (eol == std::string::npos) { + break; + } + + const auto equal = content.find('=', pos); + if (equal < eol) { + const auto key = trim(content.substr(pos, equal - pos)); + const auto value = + trim(content.substr(equal + 1, eol - (equal + 1))); + if (ctx->endpoint.empty() && key == "cdn_endpoint") { + ctx->endpoint = value; + } else if (key == "network") { + const char *enabled = getenv("PROJ_NETWORK"); + if (enabled == nullptr || enabled[0] == '\0') { + ctx->networking.enabled = ci_equal(value, "ON") || + ci_equal(value, "YES") || + ci_equal(value, "TRUE"); + } + } else if (key == "cache_enabled") { + ctx->gridChunkCache.enabled = ci_equal(value, "ON") || + ci_equal(value, "YES") || + ci_equal(value, "TRUE"); + } else if (key == "cache_size_MB") { + const int val = atoi(value.c_str()); + ctx->gridChunkCache.max_size = + val > 0 ? static_cast<long long>(val) * 1024 * 1024 : -1; + } else if (key == "cache_ttl_sec") { + ctx->gridChunkCache.ttl = atoi(value.c_str()); + } + } + + pos = content.find_first_not_of("\r\n", eol); + } +} + +//! @endcond + +/************************************************************************/ +/* pj_set_finder() */ +/************************************************************************/ + +void pj_set_finder(const char *(*new_finder)(const char *)) + +{ + auto ctx = pj_get_default_ctx(); + if (ctx) { + ctx->file_finder_legacy = new_finder; + } +} + +/************************************************************************/ +/* proj_context_set_file_finder() */ +/************************************************************************/ + +/** \brief Assign a file finder callback to a context. + * + * This callback will be used whenever PROJ must open one of its resource files + * (proj.db database, grids, etc...) + * + * The callback will be called with the context currently in use at the moment + * where it is used (not necessarily the one provided during this call), and + * with the provided user_data (which may be NULL). + * The user_data must remain valid during the whole lifetime of the context. + * + * A finder set on the default context will be inherited by contexts created + * later. + * + * @param ctx PROJ context, or NULL for the default context. + * @param finder Finder callback. May be NULL + * @param user_data User data provided to the finder callback. May be NULL. + * + * @since PROJ 6.0 + */ +void proj_context_set_file_finder(PJ_CONTEXT *ctx, proj_file_finder finder, + void *user_data) { + if (!ctx) + ctx = pj_get_default_ctx(); + if (!ctx) + return; + ctx->file_finder = finder; + ctx->file_finder_user_data = user_data; +} + +/************************************************************************/ +/* proj_context_set_search_paths() */ +/************************************************************************/ + +/** \brief Sets search paths. + * + * Those search paths will be used whenever PROJ must open one of its resource + * files + * (proj.db database, grids, etc...) + * + * If set on the default context, they will be inherited by contexts created + * later. + * + * Starting with PROJ 7.0, the path(s) should be encoded in UTF-8. + * + * @param ctx PROJ context, or NULL for the default context. + * @param count_paths Number of paths. 0 if paths == NULL. + * @param paths Paths. May be NULL. + * + * @since PROJ 6.0 + */ +void proj_context_set_search_paths(PJ_CONTEXT *ctx, int count_paths, + const char *const *paths) { + if (!ctx) + ctx = pj_get_default_ctx(); + if (!ctx) + return; + try { + std::vector<std::string> vector_of_paths; + for (int i = 0; i < count_paths; i++) { + vector_of_paths.emplace_back(paths[i]); + } + ctx->set_search_paths(vector_of_paths); + } catch (const std::exception &) { + } +} + +/************************************************************************/ +/* pj_set_searchpath() */ +/* */ +/* Path control for callers that can't practically provide */ +/* pj_set_finder() style callbacks. Call with (0,NULL) as args */ +/* to clear the searchpath set. */ +/************************************************************************/ + +void pj_set_searchpath(int count, const char **path) { + proj_context_set_search_paths(nullptr, count, + const_cast<const char *const *>(path)); +} diff --git a/src/filemanager.hpp b/src/filemanager.hpp new file mode 100644 index 00000000..554bd325 --- /dev/null +++ b/src/filemanager.hpp @@ -0,0 +1,100 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: File manager + * Author: Even Rouault, <even.rouault at spatialys.com> + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef FILEMANAGER_HPP_INCLUDED +#define FILEMANAGER_HPP_INCLUDED + +#include <memory> + +#include "proj.h" +#include "proj/util.hpp" + +//! @cond Doxygen_Suppress + +NS_PROJ_START + +class File; + +enum class FileAccess { + READ_ONLY, // "rb" + READ_UPDATE, // "r+b" + CREATE, // "w+b" +}; + +class FileManager { + private: + FileManager() = delete; + + public: + // "Low-level" interface. + static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename, + FileAccess access); + static bool exists(PJ_CONTEXT *ctx, const char *filename); + static bool mkdir(PJ_CONTEXT *ctx, const char *filename); + static bool unlink(PJ_CONTEXT *ctx, const char *filename); + static bool rename(PJ_CONTEXT *ctx, const char *oldPath, + const char *newPath); + static std::string getProjLibEnvVar(PJ_CONTEXT *ctx); + + // "High-level" interface, honoring PROJ_LIB and the like. + static std::unique_ptr<File> open_resource_file(PJ_CONTEXT *ctx, + const char *name); + + static void fillDefaultNetworkInterface(PJ_CONTEXT *ctx); + + static void clearMemoryCache(); +}; + +// --------------------------------------------------------------------------- + +class File { + protected: + std::string name_; + explicit File(const std::string &name); + + public: + virtual ~File(); + virtual size_t read(void *buffer, size_t sizeBytes) = 0; + virtual size_t write(const void *buffer, size_t sizeBytes) = 0; + virtual bool seek(unsigned long long offset, int whence = SEEK_SET) = 0; + virtual unsigned long long tell() = 0; + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; + virtual bool hasChanged() const = 0; + + const std::string &name() const { return name_; } +}; + +// --------------------------------------------------------------------------- + +std::unique_ptr<File> pj_network_file_open(PJ_CONTEXT *ctx, + const char *filename); + +NS_PROJ_END + +//! @endcond Doxygen_Suppress + +#endif // FILEMANAGER_HPP_INCLUDED diff --git a/src/gc_reader.cpp b/src/gc_reader.cpp deleted file mode 100644 index def52a11..00000000 --- a/src/gc_reader.cpp +++ /dev/null @@ -1,248 +0,0 @@ -/****************************************************************************** - * Project: PROJ.4 - * Purpose: Code to read a grid catalog from a .cvs file. - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ****************************************************************************** - * Copyright (c) 2012, Frank Warmerdam <warmerdam@pobox.com> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - *****************************************************************************/ - -#define PJ_LIB__ - -#include <ctype.h> -#include <errno.h> -#include <stdlib.h> -#include <string.h> - -#include "proj.h" -#include "proj_internal.h" - -static int gc_readentry(projCtx ctx, PAFile fid, PJ_GridCatalogEntry *entry); - -/************************************************************************/ -/* pj_gc_readcatalog() */ -/* */ -/* Read a grid catalog from a .csv file. */ -/************************************************************************/ - -PJ_GridCatalog *pj_gc_readcatalog( projCtx ctx, const char *catalog_name ) -{ - PAFile fid; - PJ_GridCatalog *catalog; - int entry_max; - char line[302]; - - fid = pj_open_lib( ctx, catalog_name, "r" ); - if (fid == nullptr) - return nullptr; - - /* discard title line */ - pj_ctx_fgets(ctx, line, sizeof(line)-1, fid); - - catalog = (PJ_GridCatalog *) calloc(1,sizeof(PJ_GridCatalog)); - if( !catalog ) - { - pj_ctx_set_errno(ctx, ENOMEM); - pj_ctx_fclose(ctx, fid); - return nullptr; - } - - catalog->catalog_name = pj_strdup(catalog_name); - if (!catalog->catalog_name) { - pj_ctx_set_errno(ctx, ENOMEM); - free(catalog); - pj_ctx_fclose(ctx, fid); - return nullptr; - } - - entry_max = 10; - catalog->entries = (PJ_GridCatalogEntry *) - malloc(entry_max * sizeof(PJ_GridCatalogEntry)); - if (!catalog->entries) { - pj_ctx_set_errno(ctx, ENOMEM); - free(catalog->catalog_name); - free(catalog); - pj_ctx_fclose(ctx, fid); - return nullptr; - } - - while( gc_readentry( ctx, fid, - catalog->entries+catalog->entry_count) == 0) - { - catalog->entry_count++; - - if( catalog->entry_count == entry_max ) - { - PJ_GridCatalogEntry* new_entries; - entry_max = entry_max * 2; - new_entries = (PJ_GridCatalogEntry *) - realloc(catalog->entries, - entry_max * sizeof(PJ_GridCatalogEntry)); - if (new_entries == nullptr ) - { - int i; - for( i = 0; i < catalog->entry_count; i++ ) - free( catalog->entries[i].definition ); - free( catalog->entries ); - free( catalog->catalog_name ); - free( catalog ); - pj_ctx_fclose(ctx, fid); - return nullptr; - } - catalog->entries = new_entries; - } - } - - pj_ctx_fclose(ctx, fid); - - return catalog; -} - -/************************************************************************/ -/* gc_read_csv_line() */ -/* */ -/* Simple csv line splitter with fixed maximum line size and */ -/* token count. */ -/************************************************************************/ - -static int gc_read_csv_line( projCtx ctx, PAFile fid, - char **tokens, int max_tokens ) -{ - char line[302]; - - while( pj_ctx_fgets(ctx, line, sizeof(line)-1, fid) != nullptr ) - { - char *next = line; - int token_count = 0; - - while( isspace(*next) ) - next++; - - /* skip blank and comment lines */ - if( next[0] == '#' || next[0] == '\0' ) - continue; - - while( token_count < max_tokens && *next != '\0' ) - { - const char *start = next; - char* token; - - while( *next != '\0' && *next != ',' ) - next++; - - if( *next == ',' ) - { - *next = '\0'; - next++; - } - - token = pj_strdup(start); - if (!token) { - while (token_count > 0) - free(tokens[--token_count]); - pj_ctx_set_errno(ctx, ENOMEM); - return 0; - } - tokens[token_count++] = token; - } - - return token_count; - } - - return 0; -} - -/************************************************************************/ -/* pj_gc_parsedate() */ -/* */ -/* Parse a date into a floating point year value. Acceptable */ -/* values are "yyyy.fraction" and "yyyy-mm-dd". Anything else */ -/* returns 0.0. */ -/************************************************************************/ - -double pj_gc_parsedate( projCtx ctx, const char *date_string ) -{ - (void) ctx; - - if( strlen(date_string) == 10 - && date_string[4] == '-' && date_string[7] == '-' ) - { - int year = atoi(date_string); - int month = atoi(date_string+5); - int day = atoi(date_string+8); - - /* simplified calculation so we don't need to know all about months */ - return year + ((month-1) * 31 + (day-1)) / 372.0; - } - else - { - return pj_atof(date_string); - } -} - - -/************************************************************************/ -/* gc_readentry() */ -/* */ -/* Read one catalog entry from the file */ -/* */ -/* Format: */ -/* gridname,ll_long,ll_lat,ur_long,ur_lat,priority,date */ -/************************************************************************/ - -static int gc_readentry(projCtx ctx, PAFile fid, PJ_GridCatalogEntry *entry) -{ -#define MAX_TOKENS 30 - char *tokens[MAX_TOKENS]; - int token_count, i; - int error = 0; - - memset( entry, 0, sizeof(PJ_GridCatalogEntry) ); - - token_count = gc_read_csv_line( ctx, fid, tokens, MAX_TOKENS ); - if( token_count < 5 ) - { - error = 1; /* TODO: need real error codes */ - if( token_count != 0 ) - pj_log( ctx, PJ_LOG_ERROR, "Short line in grid catalog." ); - } - else - { - entry->definition = tokens[0]; - tokens[0] = nullptr; /* We take ownership of tokens[0] */ - entry->region.ll_long = dmstor_ctx( ctx, tokens[1], nullptr ); - entry->region.ll_lat = dmstor_ctx( ctx, tokens[2], nullptr ); - entry->region.ur_long = dmstor_ctx( ctx, tokens[3], nullptr ); - entry->region.ur_lat = dmstor_ctx( ctx, tokens[4], nullptr ); - if( token_count > 5 ) - entry->priority = atoi( tokens[5] ); /* defaults to zero */ - if( token_count > 6 ) - entry->date = pj_gc_parsedate( ctx, tokens[6] ); - } - - for( i = 0; i < token_count; i++ ) - free( tokens[i] ); - - return error; -} - - - diff --git a/src/gridcatalog.cpp b/src/gridcatalog.cpp deleted file mode 100644 index 9b94fef8..00000000 --- a/src/gridcatalog.cpp +++ /dev/null @@ -1,306 +0,0 @@ -/****************************************************************************** - * Project: PROJ.4 - * Purpose: Code in support of grid catalogs - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ****************************************************************************** - * Copyright (c) 2012, Frank Warmerdam <warmerdam@pobox.com> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - *****************************************************************************/ - -#define PJ_LIB__ - -#include <assert.h> -#include <math.h> -#include <stdlib.h> -#include <string.h> - -#include "proj.h" -#include "proj_internal.h" - -static PJ_GridCatalog *grid_catalog_list = nullptr; - -static -PJ_GRIDINFO *pj_gc_findgrid( projCtx_t *ctx, - PJ_GridCatalog *catalog, int after, - PJ_LP location, double date, - PJ_Region *optional_region, - double *grid_date ); - -/************************************************************************/ -/* pj_gc_unloadall() */ -/* */ -/* Deallocate all the grid catalogs (but not the referenced */ -/* grids). */ -/************************************************************************/ - -void pj_gc_unloadall( projCtx ctx ) -{ - (void) ctx; - - while( grid_catalog_list != nullptr ) - { - int i; - PJ_GridCatalog *catalog = grid_catalog_list; - grid_catalog_list = grid_catalog_list->next; - - for( i = 0; i < catalog->entry_count; i++ ) - { - /* we don't own gridinfo - do not free here */ - free( catalog->entries[i].definition ); - } - free( catalog->entries ); - free( catalog->catalog_name ); - free( catalog ); - } -} - -/************************************************************************/ -/* pj_gc_findcatalog() */ -/************************************************************************/ - -PJ_GridCatalog *pj_gc_findcatalog( projCtx ctx, const char *name ) - -{ - PJ_GridCatalog *catalog; - - pj_acquire_lock(); - - for( catalog=grid_catalog_list; catalog != nullptr; catalog = catalog->next ) - { - if( strcmp(catalog->catalog_name, name) == 0 ) - { - pj_release_lock(); - return catalog; - } - } - - pj_release_lock(); - - catalog = pj_gc_readcatalog( ctx, name ); - if( catalog == nullptr ) - return nullptr; - - pj_acquire_lock(); - catalog->next = grid_catalog_list; - grid_catalog_list = catalog; - pj_release_lock(); - - return catalog; -} - -/************************************************************************/ -/* pj_gc_apply_gridshift() */ -/************************************************************************/ - -int pj_gc_apply_gridshift( PJ *defn, int inverse, - long point_count, int point_offset, - double *x, double *y, double *z ) - -{ - int i; - (void) z; - - if( defn->catalog == nullptr ) - { - defn->catalog = pj_gc_findcatalog( defn->ctx, defn->catalog_name ); - if( defn->catalog == nullptr ) - return defn->ctx->last_errno; - } - - defn->ctx->last_errno = 0; - - for( i = 0; i < point_count; i++ ) - { - long io = i * point_offset; - PJ_LP input, output_after, output_before; - double mix_ratio; - PJ_GRIDINFO *gi; - - input.phi = y[io]; - input.lam = x[io]; - - /* make sure we have appropriate "after" shift file available */ - if( defn->last_after_grid == nullptr - || input.lam < defn->last_after_region.ll_long - || input.lam > defn->last_after_region.ur_long - || input.phi < defn->last_after_region.ll_lat - || input.phi > defn->last_after_region.ll_lat ) { - defn->last_after_grid = - pj_gc_findgrid( defn->ctx, defn->catalog, - 1, input, defn->datum_date, - &(defn->last_after_region), - &(defn->last_after_date)); - if( defn->last_after_grid == nullptr ) - { - pj_ctx_set_errno( defn->ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return PJD_ERR_FAILED_TO_LOAD_GRID; - } - } - gi = defn->last_after_grid; - assert( gi->child == nullptr ); - - /* load the grid shift info if we don't have it. */ - if( gi->ct->cvs == nullptr && !pj_gridinfo_load( defn->ctx, gi ) ) - { - pj_ctx_set_errno( defn->ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return PJD_ERR_FAILED_TO_LOAD_GRID; - } - - output_after = nad_cvt( defn->ctx, input, inverse, gi->ct, 0, nullptr ); - if( output_after.lam == HUGE_VAL ) - { - if( defn->ctx->debug_level >= PJ_LOG_DEBUG_MAJOR ) - { - pj_log( defn->ctx, PJ_LOG_DEBUG_MAJOR, - "pj_apply_gridshift(): failed to find a grid shift table for\n" - " location (%.7fdW,%.7fdN)", - x[io] * RAD_TO_DEG, - y[io] * RAD_TO_DEG ); - } - continue; - } - - if( defn->datum_date == 0.0 ) - { - y[io] = output_after.phi; - x[io] = output_after.lam; - continue; - } - - /* make sure we have appropriate "before" shift file available */ - if( defn->last_before_grid == nullptr - || input.lam < defn->last_before_region.ll_long - || input.lam > defn->last_before_region.ur_long - || input.phi < defn->last_before_region.ll_lat - || input.phi > defn->last_before_region.ll_lat ) { - defn->last_before_grid = - pj_gc_findgrid( defn->ctx, defn->catalog, - 0, input, defn->datum_date, - &(defn->last_before_region), - &(defn->last_before_date)); - if( defn->last_before_grid == nullptr ) - { - pj_ctx_set_errno( defn->ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return PJD_ERR_FAILED_TO_LOAD_GRID; - } - } - - gi = defn->last_before_grid; - assert( gi->child == nullptr ); - - /* load the grid shift info if we don't have it. */ - if( gi->ct->cvs == nullptr && !pj_gridinfo_load( defn->ctx, gi ) ) - { - pj_ctx_set_errno( defn->ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return PJD_ERR_FAILED_TO_LOAD_GRID; - } - - output_before = nad_cvt( defn->ctx, input, inverse, gi->ct, 0, nullptr ); - if( output_before.lam == HUGE_VAL ) - { - if( defn->ctx->debug_level >= PJ_LOG_DEBUG_MAJOR ) - { - pj_log( defn->ctx, PJ_LOG_DEBUG_MAJOR, - "pj_apply_gridshift(): failed to find a grid shift table for\n" - " location (%.7fdW,%.7fdN)", - x[io] * RAD_TO_DEG, - y[io] * RAD_TO_DEG ); - } - continue; - } - - mix_ratio = (defn->datum_date - defn->last_before_date) - / (defn->last_after_date - defn->last_before_date); - - y[io] = mix_ratio * output_after.phi - + (1.0-mix_ratio) * output_before.phi; - x[io] = mix_ratio * output_after.lam - + (1.0-mix_ratio) * output_before.lam; - } - - return 0; -} - -/************************************************************************/ -/* pj_c_findgrid() */ -/************************************************************************/ - -static -PJ_GRIDINFO *pj_gc_findgrid( projCtx ctx, PJ_GridCatalog *catalog, int after, - PJ_LP location, double date, - PJ_Region *optional_region, - double *grid_date ) -{ - int iEntry; - PJ_GridCatalogEntry *entry = nullptr; - - for( iEntry = 0; iEntry < catalog->entry_count; iEntry++ ) - { - entry = catalog->entries + iEntry; - - if( (after && entry->date < date) - || (!after && entry->date > date) ) - continue; - - if( location.lam < entry->region.ll_long - || location.lam > entry->region.ur_long - || location.phi < entry->region.ll_lat - || location.phi > entry->region.ur_lat ) - continue; - - if( entry->available == -1 ) - continue; - - break; - } - - if( entry == nullptr ) - { - if( grid_date ) - *grid_date = 0.0; - if( optional_region != nullptr ) - memset( optional_region, 0, sizeof(PJ_Region)); - return nullptr; - } - - if( grid_date ) - *grid_date = entry->date; - - if( optional_region ) - { - - } - - if( entry->gridinfo == nullptr ) - { - PJ_GRIDINFO **gridlist = nullptr; - int grid_count = 0; - gridlist = pj_gridlist_from_nadgrids( ctx, entry->definition, - &grid_count); - // FIXME: this leaks gridlist itself, and memory ownership of - // entry->gridinfo is also confusing. Coverity CID 193539 - if( grid_count == 1 ) - entry->gridinfo = gridlist[0]; - } - - return entry->gridinfo; -} - diff --git a/src/gridinfo.cpp b/src/gridinfo.cpp deleted file mode 100644 index 7f7d930f..00000000 --- a/src/gridinfo.cpp +++ /dev/null @@ -1,988 +0,0 @@ -/****************************************************************************** - * Project: PROJ.4 - * Purpose: Functions for handling individual PJ_GRIDINFO's. Includes - * loaders for all formats but CTABLE (in nad_init.c). - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ****************************************************************************** - * Copyright (c) 2000, Frank Warmerdam <warmerdam@pobox.com> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - *****************************************************************************/ - -#define PJ_LIB__ - -#include <errno.h> -#include <math.h> -#include <stddef.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include "proj_internal.h" - -/************************************************************************/ -/* swap_words() */ -/* */ -/* Convert the byte order of the given word(s) in place. */ -/************************************************************************/ - -static const int byte_order_test = 1; -#define IS_LSB (1 == ((const unsigned char *) (&byte_order_test))[0]) - -static void swap_words( unsigned char *data, int word_size, int word_count ) - -{ - int word; - - for( word = 0; word < word_count; word++ ) - { - int i; - - for( i = 0; i < word_size/2; i++ ) - { - unsigned char t; - - t = data[i]; - data[i] = data[word_size-i-1]; - data[word_size-i-1] = t; - } - - data += word_size; - } -} - -/************************************************************************/ -/* to_double() */ -/* */ -/* Returns a double from the pointed data. */ -/************************************************************************/ - -static double to_double( unsigned char* data ) -{ - double d; - memcpy(&d, data, sizeof(d)); - return d; -} - -/************************************************************************/ -/* pj_gridinfo_free() */ -/************************************************************************/ - -void pj_gridinfo_free( projCtx ctx, PJ_GRIDINFO *gi ) - -{ - if( gi == nullptr ) - return; - - if( gi->child != nullptr ) - { - PJ_GRIDINFO *child, *next; - - for( child = gi->child; child != nullptr; child=next) - { - next=child->next; - pj_gridinfo_free( ctx, child ); - } - } - - if( gi->ct != nullptr ) - nad_free( gi->ct ); - - free( gi->gridname ); - if( gi->filename != nullptr ) - free( gi->filename ); - - pj_dalloc( gi ); -} - -/************************************************************************/ -/* pj_gridinfo_load() */ -/* */ -/* This function is intended to implement delayed loading of */ -/* the data contents of a grid file. The header and related */ -/* stuff are loaded by pj_gridinfo_init(). */ -/************************************************************************/ - -int pj_gridinfo_load( projCtx_t* ctx, PJ_GRIDINFO *gi ) - -{ - struct CTABLE ct_tmp; - - if( gi == nullptr || gi->ct == nullptr ) - return 0; - - pj_acquire_lock(); - if( gi->ct->cvs != nullptr ) - { - pj_release_lock(); - return 1; - } - - memcpy(&ct_tmp, gi->ct, sizeof(struct CTABLE)); - -/* -------------------------------------------------------------------- */ -/* Original platform specific CTable format. */ -/* -------------------------------------------------------------------- */ - if( strcmp(gi->format,"ctable") == 0 ) - { - PAFile fid; - int result; - - fid = pj_open_lib( ctx, gi->filename, "rb" ); - - if( fid == nullptr ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_release_lock(); - return 0; - } - - result = nad_ctable_load( ctx, &ct_tmp, (struct projFileAPI_t*)fid ); - - pj_ctx_fclose( ctx, fid ); - - gi->ct->cvs = ct_tmp.cvs; - pj_release_lock(); - - return result; - } - -/* -------------------------------------------------------------------- */ -/* CTable2 format. */ -/* -------------------------------------------------------------------- */ - else if( strcmp(gi->format,"ctable2") == 0 ) - { - PAFile fid; - int result; - - fid = pj_open_lib( ctx, gi->filename, "rb" ); - - if( fid == nullptr ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_release_lock(); - return 0; - } - - result = nad_ctable2_load( ctx, &ct_tmp, (struct projFileAPI_t*)fid ); - - pj_ctx_fclose( ctx, fid ); - - gi->ct->cvs = ct_tmp.cvs; - - pj_release_lock(); - return result; - } - -/* -------------------------------------------------------------------- */ -/* NTv1 format. */ -/* We process one line at a time. Note that the array storage */ -/* direction (e-w) is different in the NTv1 file and what */ -/* the CTABLE is supposed to have. The phi/lam are also */ -/* reversed, and we have to be aware of byte swapping. */ -/* -------------------------------------------------------------------- */ - else if( strcmp(gi->format,"ntv1") == 0 ) - { - double *row_buf; - int row; - PAFile fid; - - fid = pj_open_lib( ctx, gi->filename, "rb" ); - - if( fid == nullptr ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_release_lock(); - return 0; - } - - pj_ctx_fseek( ctx, fid, gi->grid_offset, SEEK_SET ); - - row_buf = (double *) pj_malloc(gi->ct->lim.lam * sizeof(double) * 2); - ct_tmp.cvs = (FLP *) pj_malloc(gi->ct->lim.lam*gi->ct->lim.phi*sizeof(FLP)); - if( row_buf == nullptr || ct_tmp.cvs == nullptr ) - { - pj_dalloc( row_buf ); - pj_dalloc( ct_tmp.cvs ); - pj_ctx_set_errno( ctx, ENOMEM ); - pj_release_lock(); - return 0; - } - - for( row = 0; row < gi->ct->lim.phi; row++ ) - { - int i; - FLP *cvs; - double *diff_seconds; - - if( pj_ctx_fread( ctx, row_buf, - sizeof(double), gi->ct->lim.lam * 2, fid ) - != (size_t)( 2 * gi->ct->lim.lam ) ) - { - pj_dalloc( row_buf ); - pj_dalloc( ct_tmp.cvs ); - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_release_lock(); - return 0; - } - - if( IS_LSB ) - swap_words( (unsigned char *) row_buf, 8, gi->ct->lim.lam*2 ); - - /* convert seconds to radians */ - diff_seconds = row_buf; - - for( i = 0; i < gi->ct->lim.lam; i++ ) - { - cvs = ct_tmp.cvs + (row) * gi->ct->lim.lam - + (gi->ct->lim.lam - i - 1); - - cvs->phi = (float) (*(diff_seconds++) * ((M_PI/180.0) / 3600.0)); - cvs->lam = (float) (*(diff_seconds++) * ((M_PI/180.0) / 3600.0)); - } - } - - pj_dalloc( row_buf ); - - pj_ctx_fclose( ctx, fid ); - - gi->ct->cvs = ct_tmp.cvs; - pj_release_lock(); - - return 1; - } - -/* -------------------------------------------------------------------- */ -/* NTv2 format. */ -/* We process one line at a time. Note that the array storage */ -/* direction (e-w) is different in the NTv2 file and what */ -/* the CTABLE is supposed to have. The phi/lam are also */ -/* reversed, and we have to be aware of byte swapping. */ -/* -------------------------------------------------------------------- */ - else if( strcmp(gi->format,"ntv2") == 0 ) - { - float *row_buf; - int row; - PAFile fid; - - pj_log( ctx, PJ_LOG_DEBUG_MINOR, - "NTv2 - loading grid %s", gi->ct->id ); - - fid = pj_open_lib( ctx, gi->filename, "rb" ); - - if( fid == nullptr ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_release_lock(); - return 0; - } - - pj_ctx_fseek( ctx, fid, gi->grid_offset, SEEK_SET ); - - row_buf = (float *) pj_malloc(gi->ct->lim.lam * sizeof(float) * 4); - ct_tmp.cvs = (FLP *) pj_malloc(gi->ct->lim.lam*gi->ct->lim.phi*sizeof(FLP)); - if( row_buf == nullptr || ct_tmp.cvs == nullptr ) - { - pj_dalloc( row_buf ); - pj_dalloc( ct_tmp.cvs ); - pj_ctx_set_errno( ctx, ENOMEM ); - pj_release_lock(); - return 0; - } - - for( row = 0; row < gi->ct->lim.phi; row++ ) - { - int i; - FLP *cvs; - float *diff_seconds; - - if( pj_ctx_fread( ctx, row_buf, sizeof(float), - gi->ct->lim.lam*4, fid ) - != (size_t)( 4 * gi->ct->lim.lam ) ) - { - pj_dalloc( row_buf ); - pj_dalloc( ct_tmp.cvs ); - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_release_lock(); - return 0; - } - - if( gi->must_swap ) - swap_words( (unsigned char *) row_buf, 4, - gi->ct->lim.lam*4 ); - - /* convert seconds to radians */ - diff_seconds = row_buf; - - for( i = 0; i < gi->ct->lim.lam; i++ ) - { - cvs = ct_tmp.cvs + (row) * gi->ct->lim.lam - + (gi->ct->lim.lam - i - 1); - - cvs->phi = (float) (*(diff_seconds++) * ((M_PI/180.0) / 3600.0)); - cvs->lam = (float) (*(diff_seconds++) * ((M_PI/180.0) / 3600.0)); - diff_seconds += 2; /* skip accuracy values */ - } - } - - pj_dalloc( row_buf ); - - pj_ctx_fclose( ctx, fid ); - - gi->ct->cvs = ct_tmp.cvs; - - pj_release_lock(); - return 1; - } - -/* -------------------------------------------------------------------- */ -/* GTX format. */ -/* -------------------------------------------------------------------- */ - else if( strcmp(gi->format,"gtx") == 0 ) - { - int words = gi->ct->lim.lam * gi->ct->lim.phi; - PAFile fid; - - fid = pj_open_lib( ctx, gi->filename, "rb" ); - - if( fid == nullptr ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_release_lock(); - return 0; - } - - pj_ctx_fseek( ctx, fid, gi->grid_offset, SEEK_SET ); - - ct_tmp.cvs = (FLP *) pj_malloc(words*sizeof(float)); - if( ct_tmp.cvs == nullptr ) - { - pj_ctx_set_errno( ctx, ENOMEM ); - pj_release_lock(); - return 0; - } - - if( pj_ctx_fread( ctx, ct_tmp.cvs, sizeof(float), words, fid ) - != (size_t)words ) - { - pj_dalloc( ct_tmp.cvs ); - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_release_lock(); - return 0; - } - - if( IS_LSB ) - swap_words( (unsigned char *) ct_tmp.cvs, 4, words ); - - pj_ctx_fclose( ctx, fid ); - gi->ct->cvs = ct_tmp.cvs; - pj_release_lock(); - return 1; - } - - else - { - pj_release_lock(); - return 0; - } -} - -/************************************************************************/ -/* gridinfo_parent() */ -/* */ -/* Seek a parent grid file by name from a grid list */ -/************************************************************************/ - -static PJ_GRIDINFO* gridinfo_parent( PJ_GRIDINFO *gilist, - const char *name, int length ) -{ - while( gilist ) - { - if( strncmp(gilist->ct->id,name,length) == 0 ) return gilist; - if( gilist->child ) - { - PJ_GRIDINFO *parent=gridinfo_parent( gilist->child, name, length ); - if( parent ) return parent; - } - gilist=gilist->next; - } - return gilist; -} - -/************************************************************************/ -/* pj_gridinfo_init_ntv2() */ -/* */ -/* Load a ntv2 (.gsb) file. */ -/************************************************************************/ - -static int pj_gridinfo_init_ntv2( projCtx ctx, PAFile fid, PJ_GRIDINFO *gilist ) -{ - unsigned char header[11*16]; - int num_subfiles, subfile; - int must_swap; - - /* cppcheck-suppress sizeofCalculation */ - STATIC_ASSERT( sizeof(pj_int32) == 4 ); - /* cppcheck-suppress sizeofCalculation */ - STATIC_ASSERT( sizeof(double) == 8 ); - -/* -------------------------------------------------------------------- */ -/* Read the overview header. */ -/* -------------------------------------------------------------------- */ - if( pj_ctx_fread( ctx, header, sizeof(header), 1, fid ) != 1 ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return 0; - } - - if( header[8] == 11 ) - must_swap = !IS_LSB; - else - must_swap = IS_LSB; - -/* -------------------------------------------------------------------- */ -/* Byte swap interesting fields if needed. */ -/* -------------------------------------------------------------------- */ - if( must_swap ) - { - swap_words( header+8, 4, 1 ); - swap_words( header+8+16, 4, 1 ); - swap_words( header+8+32, 4, 1 ); - swap_words( header+8+7*16, 8, 1 ); - swap_words( header+8+8*16, 8, 1 ); - swap_words( header+8+9*16, 8, 1 ); - swap_words( header+8+10*16, 8, 1 ); - } - -/* -------------------------------------------------------------------- */ -/* Get the subfile count out ... all we really use for now. */ -/* -------------------------------------------------------------------- */ - memcpy( &num_subfiles, header+8+32, 4 ); - -/* ==================================================================== */ -/* Step through the subfiles, creating a PJ_GRIDINFO for each. */ -/* ==================================================================== */ - for( subfile = 0; subfile < num_subfiles; subfile++ ) - { - struct CTABLE *ct; - PJ_LP ur; - int gs_count; - PJ_GRIDINFO *gi; - -/* -------------------------------------------------------------------- */ -/* Read header. */ -/* -------------------------------------------------------------------- */ - if( pj_ctx_fread( ctx, header, sizeof(header), 1, fid ) != 1 ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return 0; - } - - if( strncmp((const char *) header,"SUB_NAME",8) != 0 ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return 0; - } - -/* -------------------------------------------------------------------- */ -/* Byte swap interesting fields if needed. */ -/* -------------------------------------------------------------------- */ - if( must_swap ) - { - swap_words( header+8+16*4, 8, 1 ); - swap_words( header+8+16*5, 8, 1 ); - swap_words( header+8+16*6, 8, 1 ); - swap_words( header+8+16*7, 8, 1 ); - swap_words( header+8+16*8, 8, 1 ); - swap_words( header+8+16*9, 8, 1 ); - swap_words( header+8+16*10, 4, 1 ); - } - -/* -------------------------------------------------------------------- */ -/* Initialize a corresponding "ct" structure. */ -/* -------------------------------------------------------------------- */ - ct = (struct CTABLE *) pj_malloc(sizeof(struct CTABLE)); - if (!ct) { - pj_ctx_set_errno(ctx, ENOMEM); - return 0; - } - strncpy( ct->id, (const char *) header + 8, 8 ); - ct->id[8] = '\0'; - - ct->ll.lam = - to_double(header+7*16+8); /* W_LONG */ - ct->ll.phi = to_double(header+4*16+8); /* S_LAT */ - - ur.lam = - to_double(header+6*16+8); /* E_LONG */ - ur.phi = to_double(header+5*16+8); /* N_LAT */ - - ct->del.lam = to_double(header+9*16+8); - ct->del.phi = to_double(header+8*16+8); - - ct->lim.lam = (pj_int32) (fabs(ur.lam-ct->ll.lam)/ct->del.lam + 0.5) + 1; - ct->lim.phi = (pj_int32) (fabs(ur.phi-ct->ll.phi)/ct->del.phi + 0.5) + 1; - - pj_log( ctx, PJ_LOG_DEBUG_MINOR, - "NTv2 %s %dx%d: LL=(%.9g,%.9g) UR=(%.9g,%.9g)", - ct->id, - ct->lim.lam, ct->lim.phi, - ct->ll.lam/3600.0, ct->ll.phi/3600.0, - ur.lam/3600.0, ur.phi/3600.0 ); - - ct->ll.lam *= DEG_TO_RAD/3600.0; - ct->ll.phi *= DEG_TO_RAD/3600.0; - ct->del.lam *= DEG_TO_RAD/3600.0; - ct->del.phi *= DEG_TO_RAD/3600.0; - - memcpy( &gs_count, header + 8 + 16*10, 4 ); - if( gs_count != ct->lim.lam * ct->lim.phi ) - { - pj_log( ctx, PJ_LOG_ERROR, - "GS_COUNT(%d) does not match expected cells (%dx%d=%d)", - gs_count, ct->lim.lam, ct->lim.phi, - ct->lim.lam * ct->lim.phi ); - pj_dalloc(ct); - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return 0; - } - - ct->cvs = nullptr; - -/* -------------------------------------------------------------------- */ -/* Create a new gridinfo for this if we aren't processing the */ -/* 1st subfile, and initialize our grid info. */ -/* -------------------------------------------------------------------- */ - if( subfile == 0 ) - gi = gilist; - else - { - gi = (PJ_GRIDINFO *) pj_calloc(1, sizeof(PJ_GRIDINFO)); - if (!gi) { - pj_dalloc(ct); - pj_gridinfo_free(ctx, gilist); - pj_ctx_set_errno(ctx, ENOMEM); - return 0; - } - - gi->gridname = pj_strdup( gilist->gridname ); - gi->filename = pj_strdup( gilist->filename ); - if (!gi->gridname || !gi->filename) { - pj_gridinfo_free(ctx, gi); - pj_dalloc(ct); - pj_gridinfo_free(ctx, gilist); - pj_ctx_set_errno(ctx, ENOMEM); - return 0; - } - gi->next = nullptr; - } - - gi->must_swap = must_swap; - gi->ct = ct; - gi->format = "ntv2"; - gi->grid_offset = pj_ctx_ftell( ctx, fid ); - -/* -------------------------------------------------------------------- */ -/* Attach to the correct list or sublist. */ -/* -------------------------------------------------------------------- */ - if( strncmp((const char *)header+24,"NONE",4) == 0 ) - { - if( gi != gilist ) - { - PJ_GRIDINFO *lnk; - - for( lnk = gilist; lnk->next != nullptr; lnk = lnk->next ) {} - lnk->next = gi; - } - } - - else - { - PJ_GRIDINFO *lnk; - PJ_GRIDINFO *gp = gridinfo_parent(gilist, - (const char*)header+24,8); - - if( gp == nullptr ) - { - pj_log( ctx, PJ_LOG_ERROR, - "pj_gridinfo_init_ntv2(): " - "failed to find parent %8.8s for %s.", - (const char *) header+24, gi->ct->id ); - - for( lnk = gilist; lnk->next != nullptr; lnk = lnk->next ) {} - lnk->next = gi; - } - else - { - if( gp->child == nullptr ) - { - gp->child = gi; - } - else - { - for( lnk = gp->child; lnk->next != nullptr; lnk = lnk->next ) {} - lnk->next = gi; - } - } - } - -/* -------------------------------------------------------------------- */ -/* Seek past the data. */ -/* -------------------------------------------------------------------- */ - pj_ctx_fseek( ctx, fid, gs_count * 16, SEEK_CUR ); - } - - return 1; -} - -/************************************************************************/ -/* pj_gridinfo_init_ntv1() */ -/* */ -/* Load an NTv1 style Canadian grid shift file. */ -/************************************************************************/ - -static int pj_gridinfo_init_ntv1( projCtx ctx, PAFile fid, PJ_GRIDINFO *gi ) - -{ - unsigned char header[192]; /* 12 records of 16 bytes */ - struct CTABLE *ct; - PJ_LP ur; - - /* cppcheck-suppress sizeofCalculation */ - STATIC_ASSERT( sizeof(pj_int32) == 4 ); - /* cppcheck-suppress sizeofCalculation */ - STATIC_ASSERT( sizeof(double) == 8 ); - -/* -------------------------------------------------------------------- */ -/* Read the header. */ -/* -------------------------------------------------------------------- */ - if( pj_ctx_fread( ctx, header, sizeof(header), 1, fid ) != 1 ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return 0; - } - -/* -------------------------------------------------------------------- */ -/* Regularize fields of interest. */ -/* -------------------------------------------------------------------- */ - if( IS_LSB ) - { - swap_words( header+8, 4, 1 ); - swap_words( header+24, 8, 1 ); - swap_words( header+40, 8, 1 ); - swap_words( header+56, 8, 1 ); - swap_words( header+72, 8, 1 ); - swap_words( header+88, 8, 1 ); - swap_words( header+104, 8, 1 ); - } - - if( *((int *) (header+8)) != 12 ) - { - pj_log( ctx, PJ_LOG_ERROR, - "NTv1 grid shift file has wrong record count, corrupt?" ); - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return 0; - } - -/* -------------------------------------------------------------------- */ -/* Fill in CTABLE structure. */ -/* -------------------------------------------------------------------- */ - ct = (struct CTABLE *) pj_malloc(sizeof(struct CTABLE)); - if (!ct) { - pj_ctx_set_errno(ctx, ENOMEM); - return 0; - } - strcpy( ct->id, "NTv1 Grid Shift File" ); - - ct->ll.lam = - to_double(header+72); - ct->ll.phi = to_double(header+24); - ur.lam = - to_double(header+56); - ur.phi = to_double(header+40); - ct->del.lam = to_double(header+104); - ct->del.phi = to_double(header+88); - ct->lim.lam = (pj_int32) (fabs(ur.lam-ct->ll.lam)/ct->del.lam + 0.5) + 1; - ct->lim.phi = (pj_int32) (fabs(ur.phi-ct->ll.phi)/ct->del.phi + 0.5) + 1; - - pj_log( ctx, PJ_LOG_DEBUG_MINOR, - "NTv1 %dx%d: LL=(%.9g,%.9g) UR=(%.9g,%.9g)", - ct->lim.lam, ct->lim.phi, - ct->ll.lam, ct->ll.phi, ur.lam, ur.phi ); - - ct->ll.lam *= DEG_TO_RAD; - ct->ll.phi *= DEG_TO_RAD; - ct->del.lam *= DEG_TO_RAD; - ct->del.phi *= DEG_TO_RAD; - ct->cvs = nullptr; - - gi->ct = ct; - gi->grid_offset = (long) sizeof(header); - gi->format = "ntv1"; - - return 1; -} - -/************************************************************************/ -/* pj_gridinfo_init_gtx() */ -/* */ -/* Load a NOAA .gtx vertical datum shift file. */ -/************************************************************************/ - -static int pj_gridinfo_init_gtx( projCtx ctx, PAFile fid, PJ_GRIDINFO *gi ) - -{ - unsigned char header[40]; - struct CTABLE *ct; - double xorigin,yorigin,xstep,ystep; - int rows, columns; - - /* cppcheck-suppress sizeofCalculation */ - STATIC_ASSERT( sizeof(pj_int32) == 4 ); - /* cppcheck-suppress sizeofCalculation */ - STATIC_ASSERT( sizeof(double) == 8 ); - -/* -------------------------------------------------------------------- */ -/* Read the header. */ -/* -------------------------------------------------------------------- */ - if( pj_ctx_fread( ctx, header, sizeof(header), 1, fid ) != 1 ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return 0; - } - -/* -------------------------------------------------------------------- */ -/* Regularize fields of interest and extract. */ -/* -------------------------------------------------------------------- */ - if( IS_LSB ) - { - swap_words( header+0, 8, 4 ); - swap_words( header+32, 4, 2 ); - } - - memcpy( &yorigin, header+0, 8 ); - memcpy( &xorigin, header+8, 8 ); - memcpy( &ystep, header+16, 8 ); - memcpy( &xstep, header+24, 8 ); - - memcpy( &rows, header+32, 4 ); - memcpy( &columns, header+36, 4 ); - - if( xorigin < -360 || xorigin > 360 - || yorigin < -90 || yorigin > 90 ) - { - pj_log( ctx, PJ_LOG_ERROR, - "gtx file header has invalid extents, corrupt?"); - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return 0; - } - -/* -------------------------------------------------------------------- */ -/* Fill in CTABLE structure. */ -/* -------------------------------------------------------------------- */ - ct = (struct CTABLE *) pj_malloc(sizeof(struct CTABLE)); - if (!ct) { - pj_ctx_set_errno(ctx, ENOMEM); - return 0; - } - strcpy( ct->id, "GTX Vertical Grid Shift File" ); - - ct->ll.lam = xorigin; - ct->ll.phi = yorigin; - ct->del.lam = xstep; - ct->del.phi = ystep; - ct->lim.lam = columns; - ct->lim.phi = rows; - - /* some GTX files come in 0-360 and we shift them back into the - expected -180 to 180 range if possible. This does not solve - problems with grids spanning the dateline. */ - if( ct->ll.lam >= 180.0 ) - ct->ll.lam -= 360.0; - - if( ct->ll.lam >= 0.0 && ct->ll.lam + ct->del.lam * ct->lim.lam > 180.0 ) - { - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "This GTX spans the dateline! This will cause problems." ); - } - - pj_log( ctx, PJ_LOG_DEBUG_MINOR, - "GTX %dx%d: LL=(%.9g,%.9g) UR=(%.9g,%.9g)", - ct->lim.lam, ct->lim.phi, - ct->ll.lam, ct->ll.phi, - ct->ll.lam + (columns-1)*xstep, ct->ll.phi + (rows-1)*ystep); - - ct->ll.lam *= DEG_TO_RAD; - ct->ll.phi *= DEG_TO_RAD; - ct->del.lam *= DEG_TO_RAD; - ct->del.phi *= DEG_TO_RAD; - ct->cvs = nullptr; - - gi->ct = ct; - gi->grid_offset = 40; - gi->format = "gtx"; - - return 1; -} - -/************************************************************************/ -/* pj_gridinfo_init() */ -/* */ -/* Open and parse header details from a datum gridshift file */ -/* returning a list of PJ_GRIDINFOs for the grids in that */ -/* file. This superceeds use of nad_init() for modern */ -/* applications. */ -/************************************************************************/ - -PJ_GRIDINFO *pj_gridinfo_init( projCtx ctx, const char *gridname ) - -{ - PJ_GRIDINFO *gilist; - PAFile fp; - char header[160]; - size_t header_size = 0; - - errno = pj_errno = 0; - ctx->last_errno = 0; - -/* -------------------------------------------------------------------- */ -/* Initialize a GRIDINFO with stub info we would use if it */ -/* cannot be loaded. */ -/* -------------------------------------------------------------------- */ - gilist = (PJ_GRIDINFO *) pj_calloc(1, sizeof(PJ_GRIDINFO)); - if (!gilist) { - pj_ctx_set_errno(ctx, ENOMEM); - return nullptr; - } - - gilist->gridname = pj_strdup( gridname ); - if (!gilist->gridname) { - pj_dalloc(gilist); - pj_ctx_set_errno(ctx, ENOMEM); - return nullptr; - } - gilist->filename = nullptr; - gilist->format = "missing"; - gilist->grid_offset = 0; - gilist->ct = nullptr; - gilist->next = nullptr; - -/* -------------------------------------------------------------------- */ -/* Open the file using the usual search rules. */ -/* -------------------------------------------------------------------- */ - if (!(fp = pj_open_lib(ctx, gridname, "rb"))) { - ctx->last_errno = 0; /* don't treat as a persistent error */ - return gilist; - } - - gilist->filename = pj_strdup(gridname); - if (!gilist->filename) { - pj_dalloc(gilist->gridname); - pj_dalloc(gilist); - pj_ctx_set_errno(ctx, ENOMEM); - return nullptr; - } - -/* -------------------------------------------------------------------- */ -/* Load a header, to determine the file type. */ -/* -------------------------------------------------------------------- */ - if( (header_size = pj_ctx_fread( ctx, header, 1, - sizeof(header), fp ) ) != sizeof(header) ) - { - /* some files may be smaller that sizeof(header), eg 160, so */ - ctx->last_errno = 0; /* don't treat as a persistent error */ - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "pj_gridinfo_init: short header read of %d bytes", - (int)header_size ); - } - - pj_ctx_fseek( ctx, fp, SEEK_SET, 0 ); - -/* -------------------------------------------------------------------- */ -/* Determine file type. */ -/* -------------------------------------------------------------------- */ - if( header_size >= 144 + 16 - && strncmp(header + 0, "HEADER", 6) == 0 - && strncmp(header + 96, "W GRID", 6) == 0 - && strncmp(header + 144, "TO NAD83 ", 16) == 0 ) - { - pj_gridinfo_init_ntv1( ctx, fp, gilist ); - } - - else if( header_size >= 48 + 7 - && strncmp(header + 0, "NUM_OREC", 8) == 0 - && strncmp(header + 48, "GS_TYPE", 7) == 0 ) - { - pj_gridinfo_init_ntv2( ctx, fp, gilist ); - } - - else if( strlen(gridname) > 4 - && (strcmp(gridname+strlen(gridname)-3,"gtx") == 0 - || strcmp(gridname+strlen(gridname)-3,"GTX") == 0) ) - { - pj_gridinfo_init_gtx( ctx, fp, gilist ); - } - - else if( header_size >= 9 && strncmp(header + 0,"CTABLE V2",9) == 0 ) - { - struct CTABLE *ct = nad_ctable2_init( ctx, (struct projFileAPI_t*)fp ); - - gilist->format = "ctable2"; - gilist->ct = ct; - - if (ct == nullptr) - { - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "CTABLE V2 ct is NULL."); - } - else - { - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "Ctable2 %s %dx%d: LL=(%.9g,%.9g) UR=(%.9g,%.9g)", - ct->id, - ct->lim.lam, ct->lim.phi, - ct->ll.lam * RAD_TO_DEG, ct->ll.phi * RAD_TO_DEG, - (ct->ll.lam + (ct->lim.lam-1)*ct->del.lam) * RAD_TO_DEG, - (ct->ll.phi + (ct->lim.phi-1)*ct->del.phi) * RAD_TO_DEG ); - } - } - - else - { - struct CTABLE *ct = nad_ctable_init( ctx, (struct projFileAPI_t*)fp ); - if (ct == nullptr) - { - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "CTABLE ct is NULL."); - } else - { - gilist->format = "ctable"; - gilist->ct = ct; - - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "Ctable %s %dx%d: LL=(%.9g,%.9g) UR=(%.9g,%.9g)", - ct->id, - ct->lim.lam, ct->lim.phi, - ct->ll.lam * RAD_TO_DEG, ct->ll.phi * RAD_TO_DEG, - (ct->ll.lam + (ct->lim.lam-1)*ct->del.lam) * RAD_TO_DEG, - (ct->ll.phi + (ct->lim.phi-1)*ct->del.phi) * RAD_TO_DEG ); - } - } - - pj_ctx_fclose(ctx, fp); - - return gilist; -} diff --git a/src/gridlist.cpp b/src/gridlist.cpp deleted file mode 100644 index c540b134..00000000 --- a/src/gridlist.cpp +++ /dev/null @@ -1,220 +0,0 @@ -/****************************************************************************** - * Project: PROJ.4 - * Purpose: Code to manage the list of currently loaded (cached) PJ_GRIDINFOs - * See pj_gridinfo.c for details of loading individual grids. - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ****************************************************************************** - * Copyright (c) 2000, Frank Warmerdam <warmerdam@pobox.com> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - *****************************************************************************/ - -#define PJ_LIB__ - -#include <errno.h> -#include <stddef.h> -#include <string.h> - -#include "proj.h" -#include "proj_internal.h" - -static PJ_GRIDINFO *grid_list = nullptr; -#define PJ_MAX_PATH_LENGTH 1024 - -/************************************************************************/ -/* pj_deallocate_grids() */ -/* */ -/* Deallocate all loaded grids. */ -/************************************************************************/ - -void pj_deallocate_grids() - -{ - while( grid_list != nullptr ) - { - PJ_GRIDINFO *item = grid_list; - grid_list = grid_list->next; - item->next = nullptr; - - pj_gridinfo_free( pj_get_default_ctx(), item ); - } -} - -/************************************************************************/ -/* pj_gridlist_merge_grid() */ -/* */ -/* Find/load the named gridfile and merge it into the */ -/* last_nadgrids_list. */ -/************************************************************************/ - -static int pj_gridlist_merge_gridfile( projCtx ctx, - const char *gridname, - PJ_GRIDINFO ***p_gridlist, - int *p_gridcount, - int *p_gridmax ) - -{ - int got_match=0; - PJ_GRIDINFO *this_grid, *tail = nullptr; - -/* -------------------------------------------------------------------- */ -/* Try to find in the existing list of loaded grids. Add all */ -/* matching grids as with NTv2 we can get many grids from one */ -/* file (one shared gridname). */ -/* -------------------------------------------------------------------- */ - for( this_grid = grid_list; this_grid != nullptr; this_grid = this_grid->next) - { - if( strcmp(this_grid->gridname,gridname) == 0 ) - { - got_match = 1; - - /* don't add to the list if it is invalid. */ - if( this_grid->ct == nullptr ) - return 0; - - /* do we need to grow the list? */ - if( *p_gridcount >= *p_gridmax - 2 ) - { - PJ_GRIDINFO **new_list; - int new_max = *p_gridmax + 20; - - new_list = (PJ_GRIDINFO **) pj_calloc(new_max, sizeof(void *)); - if (!new_list) { - pj_ctx_set_errno( ctx, ENOMEM ); - return 0; - } - if( *p_gridlist != nullptr ) - { - memcpy( new_list, *p_gridlist, - sizeof(void *) * (*p_gridmax) ); - pj_dalloc( *p_gridlist ); - } - - *p_gridlist = new_list; - *p_gridmax = new_max; - } - - /* add to the list */ - (*p_gridlist)[(*p_gridcount)++] = this_grid; - (*p_gridlist)[*p_gridcount] = nullptr; - } - - tail = this_grid; - } - - if( got_match ) - return 1; - -/* -------------------------------------------------------------------- */ -/* Try to load the named grid. */ -/* -------------------------------------------------------------------- */ - this_grid = pj_gridinfo_init( ctx, gridname ); - - if( this_grid == nullptr ) - { - return 0; - } - - if( tail != nullptr ) - tail->next = this_grid; - else - grid_list = this_grid; - -/* -------------------------------------------------------------------- */ -/* Recurse to add the grid now that it is loaded. */ -/* -------------------------------------------------------------------- */ - return pj_gridlist_merge_gridfile( ctx, gridname, p_gridlist, - p_gridcount, p_gridmax ); -} - -/************************************************************************/ -/* pj_gridlist_from_nadgrids() */ -/* */ -/* This functions loads the list of grids corresponding to a */ -/* particular nadgrids string into a list, and returns it. The */ -/* list is kept around till a request is made with a different */ -/* string in order to cut down on the string parsing cost, and */ -/* the cost of building the list of tables each time. */ -/************************************************************************/ - -PJ_GRIDINFO **pj_gridlist_from_nadgrids( projCtx ctx, const char *nadgrids, - int *grid_count) - -{ - const char *s; - PJ_GRIDINFO **gridlist = nullptr; - int grid_max = 0; - - pj_errno = 0; - *grid_count = 0; - - pj_acquire_lock(); - -/* -------------------------------------------------------------------- */ -/* Loop processing names out of nadgrids one at a time. */ -/* -------------------------------------------------------------------- */ - for( s = nadgrids; *s != '\0'; ) - { - size_t end_char; - int required = 1; - char name[PJ_MAX_PATH_LENGTH]; - - if( *s == '@' ) - { - required = 0; - s++; - } - - for( end_char = 0; - s[end_char] != '\0' && s[end_char] != ','; - end_char++ ) {} - - if( end_char >= sizeof(name) ) - { - pj_dalloc( gridlist ); - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_release_lock(); - return nullptr; - } - - strncpy( name, s, end_char ); - name[end_char] = '\0'; - - s += end_char; - if( *s == ',' ) - s++; - - if( !pj_gridlist_merge_gridfile( ctx, name, &gridlist, grid_count, - &grid_max) - && required ) - { - pj_dalloc( gridlist ); - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_release_lock(); - return nullptr; - } - else - pj_errno = 0; - } - - pj_release_lock(); - - return gridlist; -} diff --git a/src/grids.cpp b/src/grids.cpp new file mode 100644 index 00000000..68bf8600 --- /dev/null +++ b/src/grids.cpp @@ -0,0 +1,3402 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: Grid management + * Author: Even Rouault, <even.rouault at spatialys.com> + * + ****************************************************************************** + * Copyright (c) 2000, Frank Warmerdam <warmerdam@pobox.com> + * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef FROM_PROJ_CPP +#define FROM_PROJ_CPP +#endif +#define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS + +#include "grids.hpp" +#include "filemanager.hpp" +#include "proj/internal/internal.hpp" +#include "proj/internal/lru_cache.hpp" +#include "proj_internal.h" + +#ifdef TIFF_ENABLED +#include "tiffio.h" +#endif + +#include <algorithm> +#include <cmath> + +NS_PROJ_START + +using namespace internal; + +/************************************************************************/ +/* swap_words() */ +/* */ +/* Convert the byte order of the given word(s) in place. */ +/************************************************************************/ + +static const int byte_order_test = 1; +#define IS_LSB (1 == ((const unsigned char *)(&byte_order_test))[0]) + +static void swap_words(void *dataIn, size_t word_size, size_t word_count) + +{ + unsigned char *data = static_cast<unsigned char *>(dataIn); + for (size_t word = 0; word < word_count; word++) { + for (size_t i = 0; i < word_size / 2; i++) { + unsigned char t; + + t = data[i]; + data[i] = data[word_size - i - 1]; + data[word_size - i - 1] = t; + } + + data += word_size; + } +} + +// --------------------------------------------------------------------------- + +bool ExtentAndRes::fullWorldLongitude() const { + return eastLon - westLon + resLon >= 2 * M_PI - 1e-10; +} + +// --------------------------------------------------------------------------- + +bool ExtentAndRes::contains(const ExtentAndRes &other) const { + return other.westLon >= westLon && other.eastLon <= eastLon && + other.southLat >= southLat && other.northLat <= northLat; +} + +// --------------------------------------------------------------------------- + +bool ExtentAndRes::intersects(const ExtentAndRes &other) const { + return other.westLon < eastLon && westLon <= other.westLon && + other.southLat < northLat && southLat <= other.northLat; +} + +// --------------------------------------------------------------------------- + +Grid::Grid(const std::string &name, int widthIn, int heightIn, + const ExtentAndRes &extentIn) + : m_name(name), m_width(widthIn), m_height(heightIn), m_extent(extentIn) {} + +// --------------------------------------------------------------------------- + +Grid::~Grid() = default; + +// --------------------------------------------------------------------------- + +VerticalShiftGrid::VerticalShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : Grid(nameIn, widthIn, heightIn, extentIn) {} + +// --------------------------------------------------------------------------- + +VerticalShiftGrid::~VerticalShiftGrid() = default; + +// --------------------------------------------------------------------------- + +static ExtentAndRes globalExtent() { + ExtentAndRes extent; + extent.westLon = -M_PI; + extent.southLat = -M_PI / 2; + extent.eastLon = M_PI; + extent.northLat = M_PI / 2; + extent.resLon = M_PI; + extent.resLat = M_PI / 2; + return extent; +} + +// --------------------------------------------------------------------------- + +class NullVerticalShiftGrid : public VerticalShiftGrid { + + public: + NullVerticalShiftGrid() : VerticalShiftGrid("null", 3, 3, globalExtent()) {} + + bool isNullGrid() const override { return true; } + bool valueAt(int, int, float &out) const override; + bool isNodata(float, double) const override { return false; } + void reassign_context(PJ_CONTEXT *) override {} + bool hasChanged() const override { return false; } +}; + +// --------------------------------------------------------------------------- + +bool NullVerticalShiftGrid::valueAt(int, int, float &out) const { + out = 0.0f; + return true; +} + +// --------------------------------------------------------------------------- + +class GTXVerticalShiftGrid : public VerticalShiftGrid { + PJ_CONTEXT *m_ctx; + std::unique_ptr<File> m_fp; + + GTXVerticalShiftGrid(const GTXVerticalShiftGrid &) = delete; + GTXVerticalShiftGrid &operator=(const GTXVerticalShiftGrid &) = delete; + + public: + explicit GTXVerticalShiftGrid(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp, + const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : VerticalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), + m_fp(std::move(fp)) {} + + ~GTXVerticalShiftGrid() override; + + bool valueAt(int x, int y, float &out) const override; + bool isNodata(float val, double multiplier) const override; + + static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &name); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } + + bool hasChanged() const override { return m_fp->hasChanged(); } +}; + +// --------------------------------------------------------------------------- + +GTXVerticalShiftGrid::~GTXVerticalShiftGrid() = default; + +// --------------------------------------------------------------------------- + +GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, + std::unique_ptr<File> fp, + const std::string &name) { + unsigned char header[40]; + + /* -------------------------------------------------------------------- */ + /* Read the header. */ + /* -------------------------------------------------------------------- */ + if (fp->read(header, sizeof(header)) != sizeof(header)) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + + /* -------------------------------------------------------------------- */ + /* Regularize fields of interest and extract. */ + /* -------------------------------------------------------------------- */ + if (IS_LSB) { + swap_words(header + 0, 8, 4); + swap_words(header + 32, 4, 2); + } + + double xorigin, yorigin, xstep, ystep; + int rows, columns; + + memcpy(&yorigin, header + 0, 8); + memcpy(&xorigin, header + 8, 8); + memcpy(&ystep, header + 16, 8); + memcpy(&xstep, header + 24, 8); + + memcpy(&rows, header + 32, 4); + memcpy(&columns, header + 36, 4); + + if (xorigin < -360 || xorigin > 360 || yorigin < -90 || yorigin > 90) { + pj_log(ctx, PJ_LOG_ERROR, + "gtx file header has invalid extents, corrupt?"); + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + + /* some GTX files come in 0-360 and we shift them back into the + expected -180 to 180 range if possible. This does not solve + problems with grids spanning the dateline. */ + if (xorigin >= 180.0) + xorigin -= 360.0; + + if (xorigin >= 0.0 && xorigin + xstep * columns > 180.0) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "This GTX spans the dateline! This will cause problems."); + } + + ExtentAndRes extent; + extent.westLon = xorigin * DEG_TO_RAD; + extent.southLat = yorigin * DEG_TO_RAD; + extent.resLon = xstep * DEG_TO_RAD; + extent.resLat = ystep * DEG_TO_RAD; + extent.eastLon = (xorigin + xstep * (columns - 1)) * DEG_TO_RAD; + extent.northLat = (yorigin + ystep * (rows - 1)) * DEG_TO_RAD; + + return new GTXVerticalShiftGrid(ctx, std::move(fp), name, columns, rows, + extent); +} + +// --------------------------------------------------------------------------- + +bool GTXVerticalShiftGrid::valueAt(int x, int y, float &out) const { + assert(x >= 0 && y >= 0 && x < m_width && y < m_height); + + m_fp->seek(40 + sizeof(float) * (y * m_width + x)); + if (m_fp->read(&out, sizeof(out)) != sizeof(out)) { + pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return false; + } + if (IS_LSB) { + swap_words(&out, sizeof(float), 1); + } + return true; +} + +// --------------------------------------------------------------------------- + +bool GTXVerticalShiftGrid::isNodata(float val, double multiplier) const { + /* nodata? */ + /* GTX official nodata value if -88.88880f, but some grids also */ + /* use other big values for nodata (e.g naptrans2008.gtx has */ + /* nodata values like -2147479936), so test them too */ + return val * multiplier > 1000 || val * multiplier < -1000 || + val == -88.88880f; +} + +// --------------------------------------------------------------------------- + +VerticalShiftGridSet::VerticalShiftGridSet() = default; + +// --------------------------------------------------------------------------- + +VerticalShiftGridSet::~VerticalShiftGridSet() = default; + +// --------------------------------------------------------------------------- + +static bool IsTIFF(size_t header_size, const unsigned char *header) { + // Test combinations of signature for ClassicTIFF/BigTIFF little/big endian + return header_size >= 4 && (((header[0] == 'I' && header[1] == 'I') || + (header[0] == 'M' && header[1] == 'M')) && + ((header[2] == 0x2A && header[3] == 0) || + (header[3] == 0x2A && header[2] == 0) || + (header[2] == 0x2B && header[3] == 0) || + (header[3] == 0x2B && header[2] == 0))); +} + +#ifdef TIFF_ENABLED + +// --------------------------------------------------------------------------- + +enum class TIFFDataType { Int16, UInt16, Int32, UInt32, Float32, Float64 }; + +// --------------------------------------------------------------------------- + +constexpr uint16 TIFFTAG_GEOPIXELSCALE = 33550; +constexpr uint16 TIFFTAG_GEOTIEPOINTS = 33922; +constexpr uint16 TIFFTAG_GEOTRANSMATRIX = 34264; +constexpr uint16 TIFFTAG_GEOKEYDIRECTORY = 34735; +constexpr uint16 TIFFTAG_GEODOUBLEPARAMS = 34736; +constexpr uint16 TIFFTAG_GEOASCIIPARAMS = 34737; +constexpr uint16 TIFFTAG_GDAL_METADATA = 42112; +constexpr uint16 TIFFTAG_GDAL_NODATA = 42113; + +// --------------------------------------------------------------------------- + +class BlockCache { + public: + void insert(uint32 ifdIdx, uint32 blockNumber, + const std::vector<unsigned char> &data); + std::shared_ptr<std::vector<unsigned char>> get(uint32 ifdIdx, + uint32 blockNumber); + + private: + struct Key { + uint32 ifdIdx; + uint32 blockNumber; + + Key(uint32 ifdIdxIn, uint32 blockNumberIn) + : ifdIdx(ifdIdxIn), blockNumber(blockNumberIn) {} + bool operator==(const Key &other) const { + return ifdIdx == other.ifdIdx && blockNumber == other.blockNumber; + } + }; + + struct KeyHasher { + std::size_t operator()(const Key &k) const { + return k.ifdIdx ^ (k.blockNumber << 16) ^ (k.blockNumber >> 16); + } + }; + + static constexpr int NUM_BLOCKS_AT_CROSSING_TILES = 4; + static constexpr int MAX_SAMPLE_COUNT = 3; + lru11::Cache< + Key, std::shared_ptr<std::vector<unsigned char>>, lru11::NullLock, + std::unordered_map< + Key, + typename std::list<lru11::KeyValuePair< + Key, std::shared_ptr<std::vector<unsigned char>>>>::iterator, + KeyHasher>> + cache_{NUM_BLOCKS_AT_CROSSING_TILES * MAX_SAMPLE_COUNT}; +}; + +// --------------------------------------------------------------------------- + +void BlockCache::insert(uint32 ifdIdx, uint32 blockNumber, + const std::vector<unsigned char> &data) { + cache_.insert(Key(ifdIdx, blockNumber), + std::make_shared<std::vector<unsigned char>>(data)); +} + +// --------------------------------------------------------------------------- + +std::shared_ptr<std::vector<unsigned char>> +BlockCache::get(uint32 ifdIdx, uint32 blockNumber) { + std::shared_ptr<std::vector<unsigned char>> ret; + cache_.tryGet(Key(ifdIdx, blockNumber), ret); + return ret; +} + +// --------------------------------------------------------------------------- + +class GTiffGrid : public Grid { + PJ_CONTEXT *m_ctx; // owned by the belonging GTiffDataset + TIFF *m_hTIFF; // owned by the belonging GTiffDataset + BlockCache &m_cache; // owned by the belonging GTiffDataset + File *m_fp; // owned by the belonging GTiffDataset + uint32 m_ifdIdx; + TIFFDataType m_dt; + uint16 m_samplesPerPixel; + uint16 m_planarConfig; + bool m_bottomUp; + toff_t m_dirOffset; + bool m_tiled; + uint32 m_blockWidth = 0; + uint32 m_blockHeight = 0; + mutable std::vector<unsigned char> m_buffer{}; + unsigned m_blocksPerRow = 0; + unsigned m_blocksPerCol = 0; + std::map<int, double> m_mapOffset{}; + std::map<int, double> m_mapScale{}; + std::map<std::pair<int, std::string>, std::string> m_metadata{}; + bool m_hasNodata = false; + float m_noData = 0.0f; + uint32 m_subfileType = 0; + + GTiffGrid(const GTiffGrid &) = delete; + GTiffGrid &operator=(const GTiffGrid &) = delete; + + void getScaleOffset(double &scale, double &offset, uint16 sample) const; + + template <class T> + float readValue(const std::vector<unsigned char> &buffer, + uint32 offsetInBlock, uint16 sample) const; + + public: + GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, File *fp, + uint32 ifdIdx, const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn, TIFFDataType dtIn, + uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn); + + ~GTiffGrid() override; + + uint16 samplesPerPixel() const { return m_samplesPerPixel; } + + bool valueAt(uint16 sample, int x, int y, float &out) const; + + bool isNodata(float val) const; + + std::string metadataItem(const std::string &key, int sample = -1) const; + + uint32 subfileType() const { return m_subfileType; } + + void reassign_context(PJ_CONTEXT *ctx) { m_ctx = ctx; } + + bool hasChanged() const override { return m_fp->hasChanged(); } +}; + +// --------------------------------------------------------------------------- + +GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, File *fp, + uint32 ifdIdx, const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn, + TIFFDataType dtIn, uint16 samplesPerPixelIn, + uint16 planarConfig, bool bottomUpIn) + : Grid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), + m_cache(cache), m_fp(fp), m_ifdIdx(ifdIdx), m_dt(dtIn), + m_samplesPerPixel(samplesPerPixelIn), m_planarConfig(planarConfig), + m_bottomUp(bottomUpIn), m_dirOffset(TIFFCurrentDirOffset(hTIFF)), + m_tiled(TIFFIsTiled(hTIFF) != 0) { + + if (m_tiled) { + TIFFGetField(m_hTIFF, TIFFTAG_TILEWIDTH, &m_blockWidth); + TIFFGetField(m_hTIFF, TIFFTAG_TILELENGTH, &m_blockHeight); + } else { + m_blockWidth = widthIn; + TIFFGetField(m_hTIFF, TIFFTAG_ROWSPERSTRIP, &m_blockHeight); + if (m_blockHeight > static_cast<unsigned>(m_height)) + m_blockHeight = m_height; + } + + TIFFGetField(m_hTIFF, TIFFTAG_SUBFILETYPE, &m_subfileType); + + m_blocksPerRow = (m_width + m_blockWidth - 1) / m_blockWidth; + m_blocksPerCol = (m_height + m_blockHeight - 1) / m_blockHeight; + + const char *text = nullptr; + // Poor-man XML parsing of TIFFTAG_GDAL_METADATA tag. Hopefully good + // enough for our purposes. + if (TIFFGetField(m_hTIFF, TIFFTAG_GDAL_METADATA, &text)) { + const char *ptr = text; + while (true) { + ptr = strstr(ptr, "<Item "); + if (ptr == nullptr) + break; + const char *endTag = strchr(ptr, '>'); + if (endTag == nullptr) + break; + const char *endValue = strchr(endTag, '<'); + if (endValue == nullptr) + break; + + std::string tag; + tag.append(ptr, endTag - ptr); + + std::string value; + value.append(endTag + 1, endValue - (endTag + 1)); + + std::string name; + auto namePos = tag.find("name=\""); + if (namePos == std::string::npos) + break; + { + namePos += strlen("name=\""); + const auto endQuote = tag.find('"', namePos); + if (endQuote == std::string::npos) + break; + name = tag.substr(namePos, endQuote - namePos); + } + + const auto samplePos = tag.find("sample=\""); + int sample = -1; + if (samplePos != std::string::npos) { + sample = atoi(tag.c_str() + samplePos + strlen("sample=\"")); + } + + m_metadata[std::pair<int, std::string>(sample, name)] = value; + + auto rolePos = tag.find("role=\""); + if (rolePos != std::string::npos) { + rolePos += strlen("role=\""); + const auto endQuote = tag.find('"', rolePos); + if (endQuote == std::string::npos) + break; + const auto role = tag.substr(rolePos, endQuote - rolePos); + if (role == "offset") { + if (sample >= 0) { + try { + m_mapOffset[sample] = c_locale_stod(value); + } catch (const std::exception &) { + } + } + } else if (role == "scale") { + if (sample >= 0) { + try { + m_mapScale[sample] = c_locale_stod(value); + } catch (const std::exception &) { + } + } + } + } + + ptr = endValue + 1; + } + } + + if (TIFFGetField(m_hTIFF, TIFFTAG_GDAL_NODATA, &text)) { + try { + m_noData = static_cast<float>(c_locale_stod(text)); + m_hasNodata = true; + } catch (const std::exception &) { + } + } + + auto oIter = m_metadata.find(std::pair<int, std::string>(-1, "grid_name")); + if (oIter != m_metadata.end()) { + m_name += ", " + oIter->second; + } +} + +// --------------------------------------------------------------------------- + +GTiffGrid::~GTiffGrid() = default; + +// --------------------------------------------------------------------------- + +void GTiffGrid::getScaleOffset(double &scale, double &offset, + uint16 sample) const { + { + auto iter = m_mapScale.find(sample); + if (iter != m_mapScale.end()) + scale = iter->second; + } + + { + auto iter = m_mapOffset.find(sample); + if (iter != m_mapOffset.end()) + offset = iter->second; + } +} + +// --------------------------------------------------------------------------- + +template <class T> +float GTiffGrid::readValue(const std::vector<unsigned char> &buffer, + uint32 offsetInBlock, uint16 sample) const { + const auto ptr = reinterpret_cast<const T *>(buffer.data()); + assert(offsetInBlock < buffer.size() / sizeof(T)); + const auto val = ptr[offsetInBlock]; + if (!m_hasNodata || static_cast<float>(val) != m_noData) { + double scale = 1; + double offset = 0; + getScaleOffset(scale, offset, sample); + return static_cast<float>(val * scale + offset); + } else { + return static_cast<float>(val); + } +} + +// --------------------------------------------------------------------------- + +bool GTiffGrid::valueAt(uint16 sample, int x, int yFromBottom, + float &out) const { + assert(x >= 0 && yFromBottom >= 0 && x < m_width && yFromBottom < m_height); + assert(sample < m_samplesPerPixel); + + const int blockX = x / m_blockWidth; + + // All non-TIFF grids have the first rows in the file being the one + // corresponding to the southern-most row. In GeoTIFF, the convention is + // *generally* different (when m_bottomUp == false), TIFF being an + // image-oriented image. If m_bottomUp == true, then we had GeoTIFF hints + // that the first row of the image is the southern-most. + const int yTIFF = m_bottomUp ? yFromBottom : m_height - 1 - yFromBottom; + const int blockY = yTIFF / m_blockHeight; + + uint32 blockId = blockY * m_blocksPerRow + blockX; + if (m_planarConfig == PLANARCONFIG_SEPARATE) { + blockId += sample * m_blocksPerCol * m_blocksPerRow; + } + + auto cachedBuffer = m_cache.get(m_ifdIdx, blockId); + std::vector<unsigned char> *pBuffer = &m_buffer; + if (cachedBuffer != nullptr) { + // Safe as we don't access the cache before pBuffer is used + pBuffer = cachedBuffer.get(); + } else { + if (TIFFCurrentDirOffset(m_hTIFF) != m_dirOffset && + !TIFFSetSubDirectory(m_hTIFF, m_dirOffset)) { + return false; + } + if (m_buffer.empty()) { + const auto blockSize = static_cast<size_t>( + m_tiled ? TIFFTileSize64(m_hTIFF) : TIFFStripSize64(m_hTIFF)); + try { + m_buffer.resize(blockSize); + } catch (const std::exception &e) { + pj_log(m_ctx, PJ_LOG_ERROR, "Exception %s", e.what()); + return false; + } + } + + if (m_tiled) { + if (TIFFReadEncodedTile(m_hTIFF, blockId, m_buffer.data(), + m_buffer.size()) == -1) { + return false; + } + } else { + if (TIFFReadEncodedStrip(m_hTIFF, blockId, m_buffer.data(), + m_buffer.size()) == -1) { + return false; + } + } + + try { + m_cache.insert(m_ifdIdx, blockId, m_buffer); + } catch (const std::exception &e) { + // Should normally not happen + pj_log(m_ctx, PJ_LOG_ERROR, "Exception %s", e.what()); + } + } + + uint32 offsetInBlock = + (x % m_blockWidth) + (yTIFF % m_blockHeight) * m_blockWidth; + if (m_planarConfig == PLANARCONFIG_CONTIG) + offsetInBlock = offsetInBlock * m_samplesPerPixel + sample; + + switch (m_dt) { + case TIFFDataType::Int16: + out = readValue<short>(*pBuffer, offsetInBlock, sample); + break; + + case TIFFDataType::UInt16: + out = readValue<unsigned short>(*pBuffer, offsetInBlock, sample); + break; + + case TIFFDataType::Int32: + out = readValue<int>(*pBuffer, offsetInBlock, sample); + break; + + case TIFFDataType::UInt32: + out = readValue<unsigned int>(*pBuffer, offsetInBlock, sample); + break; + + case TIFFDataType::Float32: + out = readValue<float>(*pBuffer, offsetInBlock, sample); + break; + + case TIFFDataType::Float64: + out = readValue<double>(*pBuffer, offsetInBlock, sample); + break; + } + + return true; +} + +// --------------------------------------------------------------------------- + +bool GTiffGrid::isNodata(float val) const { + return (m_hasNodata && val == m_noData) || std::isnan(val); +} + +// --------------------------------------------------------------------------- + +std::string GTiffGrid::metadataItem(const std::string &key, int sample) const { + auto iter = m_metadata.find(std::pair<int, std::string>(sample, key)); + if (iter == m_metadata.end()) { + return std::string(); + } + return iter->second; +} + +// --------------------------------------------------------------------------- + +class GTiffDataset { + PJ_CONTEXT *m_ctx; + std::unique_ptr<File> m_fp; + TIFF *m_hTIFF = nullptr; + bool m_hasNextGrid = false; + uint32 m_ifdIdx = 0; + toff_t m_nextDirOffset = 0; + std::string m_filename{}; + BlockCache m_cache{}; + + GTiffDataset(const GTiffDataset &) = delete; + GTiffDataset &operator=(const GTiffDataset &) = delete; + + // libtiff I/O routines + static tsize_t tiffReadProc(thandle_t fd, tdata_t buf, tsize_t size) { + GTiffDataset *self = static_cast<GTiffDataset *>(fd); + return self->m_fp->read(buf, size); + } + + static tsize_t tiffWriteProc(thandle_t, tdata_t, tsize_t) { + assert(false); + return 0; + } + + static toff_t tiffSeekProc(thandle_t fd, toff_t off, int whence) { + GTiffDataset *self = static_cast<GTiffDataset *>(fd); + if (self->m_fp->seek(off, whence)) + return static_cast<toff_t>(self->m_fp->tell()); + else + return static_cast<toff_t>(-1); + } + + static int tiffCloseProc(thandle_t) { + // done in destructor + return 0; + } + + static toff_t tiffSizeProc(thandle_t fd) { + GTiffDataset *self = static_cast<GTiffDataset *>(fd); + const auto old_off = self->m_fp->tell(); + self->m_fp->seek(0, SEEK_END); + const auto file_size = static_cast<toff_t>(self->m_fp->tell()); + self->m_fp->seek(old_off); + return file_size; + } + + static int tiffMapProc(thandle_t, tdata_t *, toff_t *) { return (0); } + + static void tiffUnmapProc(thandle_t, tdata_t, toff_t) {} + + public: + GTiffDataset(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp) + : m_ctx(ctx), m_fp(std::move(fp)) {} + virtual ~GTiffDataset(); + + bool openTIFF(const std::string &filename); + + std::unique_ptr<GTiffGrid> nextGrid(); + + void reassign_context(PJ_CONTEXT *ctx) { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } +}; + +// --------------------------------------------------------------------------- + +GTiffDataset::~GTiffDataset() { + if (m_hTIFF) + TIFFClose(m_hTIFF); +} + +// --------------------------------------------------------------------------- +class OneTimeTIFFTagInit { + + static TIFFExtendProc ParentExtender; + + // Function called by libtiff when initializing a TIFF directory + static void GTiffTagExtender(TIFF *tif) { + static const TIFFFieldInfo xtiffFieldInfo[] = { + // GeoTIFF tags + {TIFFTAG_GEOPIXELSCALE, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast<char *>("GeoPixelScale")}, + {TIFFTAG_GEOTIEPOINTS, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast<char *>("GeoTiePoints")}, + {TIFFTAG_GEOTRANSMATRIX, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast<char *>("GeoTransformationMatrix")}, + + {TIFFTAG_GEOKEYDIRECTORY, -1, -1, TIFF_SHORT, FIELD_CUSTOM, TRUE, + TRUE, const_cast<char *>("GeoKeyDirectory")}, + {TIFFTAG_GEODOUBLEPARAMS, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast<char *>("GeoDoubleParams")}, + {TIFFTAG_GEOASCIIPARAMS, -1, -1, TIFF_ASCII, FIELD_CUSTOM, TRUE, + FALSE, const_cast<char *>("GeoASCIIParams")}, + + // GDAL tags + {TIFFTAG_GDAL_METADATA, -1, -1, TIFF_ASCII, FIELD_CUSTOM, TRUE, + FALSE, const_cast<char *>("GDALMetadata")}, + {TIFFTAG_GDAL_NODATA, -1, -1, TIFF_ASCII, FIELD_CUSTOM, TRUE, FALSE, + const_cast<char *>("GDALNoDataValue")}, + + }; + + if (ParentExtender) + (*ParentExtender)(tif); + + TIFFMergeFieldInfo(tif, xtiffFieldInfo, + sizeof(xtiffFieldInfo) / sizeof(xtiffFieldInfo[0])); + } + + public: + OneTimeTIFFTagInit() { + assert(ParentExtender == nullptr); + // Install our TIFF tag extender + ParentExtender = TIFFSetTagExtender(GTiffTagExtender); + } +}; + +TIFFExtendProc OneTimeTIFFTagInit::ParentExtender = nullptr; + +// --------------------------------------------------------------------------- + +bool GTiffDataset::openTIFF(const std::string &filename) { + static OneTimeTIFFTagInit oneTimeTIFFTagInit; + m_hTIFF = + TIFFClientOpen(filename.c_str(), "r", static_cast<thandle_t>(this), + GTiffDataset::tiffReadProc, GTiffDataset::tiffWriteProc, + GTiffDataset::tiffSeekProc, GTiffDataset::tiffCloseProc, + GTiffDataset::tiffSizeProc, GTiffDataset::tiffMapProc, + GTiffDataset::tiffUnmapProc); + + m_filename = filename; + m_hasNextGrid = true; + return m_hTIFF != nullptr; +} +// --------------------------------------------------------------------------- + +std::unique_ptr<GTiffGrid> GTiffDataset::nextGrid() { + if (!m_hasNextGrid) + return nullptr; + if (m_nextDirOffset) { + TIFFSetSubDirectory(m_hTIFF, m_nextDirOffset); + } + + uint32 width = 0; + uint32 height = 0; + TIFFGetField(m_hTIFF, TIFFTAG_IMAGEWIDTH, &width); + TIFFGetField(m_hTIFF, TIFFTAG_IMAGELENGTH, &height); + if (width == 0 || height == 0 || width > INT_MAX || height > INT_MAX) { + pj_log(m_ctx, PJ_LOG_ERROR, "Invalid image size"); + return nullptr; + } + + uint16 samplesPerPixel = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_SAMPLESPERPIXEL, &samplesPerPixel)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing SamplesPerPixel tag"); + return nullptr; + } + if (samplesPerPixel == 0) { + pj_log(m_ctx, PJ_LOG_ERROR, "Invalid SamplesPerPixel value"); + return nullptr; + } + + uint16 bitsPerSample = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_BITSPERSAMPLE, &bitsPerSample)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing BitsPerSample tag"); + return nullptr; + } + + uint16 planarConfig = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_PLANARCONFIG, &planarConfig)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing PlanarConfig tag"); + return nullptr; + } + + uint16 sampleFormat = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_SAMPLEFORMAT, &sampleFormat)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing SampleFormat tag"); + return nullptr; + } + + TIFFDataType dt; + if (sampleFormat == SAMPLEFORMAT_INT && bitsPerSample == 16) + dt = TIFFDataType::Int16; + else if (sampleFormat == SAMPLEFORMAT_UINT && bitsPerSample == 16) + dt = TIFFDataType::UInt16; + else if (sampleFormat == SAMPLEFORMAT_INT && bitsPerSample == 32) + dt = TIFFDataType::Int32; + else if (sampleFormat == SAMPLEFORMAT_UINT && bitsPerSample == 32) + dt = TIFFDataType::UInt32; + else if (sampleFormat == SAMPLEFORMAT_IEEEFP && bitsPerSample == 32) + dt = TIFFDataType::Float32; + else if (sampleFormat == SAMPLEFORMAT_IEEEFP && bitsPerSample == 64) + dt = TIFFDataType::Float64; + else { + pj_log( + m_ctx, PJ_LOG_ERROR, + "Unsupported combination of SampleFormat and BitsPerSample values"); + return nullptr; + } + + uint16 photometric = PHOTOMETRIC_MINISBLACK; + if (!TIFFGetField(m_hTIFF, TIFFTAG_PHOTOMETRIC, &photometric)) + photometric = PHOTOMETRIC_MINISBLACK; + if (photometric != PHOTOMETRIC_MINISBLACK) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported Photometric value"); + return nullptr; + } + + uint16 compression = COMPRESSION_NONE; + if (!TIFFGetField(m_hTIFF, TIFFTAG_COMPRESSION, &compression)) + compression = COMPRESSION_NONE; + + if (compression != COMPRESSION_NONE && + !TIFFIsCODECConfigured(compression)) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Cannot open TIFF file due to missing codec."); + return nullptr; + } + // We really don't want to try dealing with old-JPEG images + if (compression == COMPRESSION_OJPEG) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported compression method."); + return nullptr; + } + + const auto blockSize = TIFFIsTiled(m_hTIFF) ? TIFFTileSize64(m_hTIFF) + : TIFFStripSize64(m_hTIFF); + if (blockSize == 0 || blockSize > 64 * 1024 * 2014) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported block size."); + return nullptr; + } + + unsigned short count = 0; + unsigned short *geokeys = nullptr; + bool pixelIsArea = false; + if (!TIFFGetField(m_hTIFF, TIFFTAG_GEOKEYDIRECTORY, &count, &geokeys)) { + pj_log(m_ctx, PJ_LOG_DEBUG_MINOR, "No GeoKeys tag"); + } else { + if (count < 4 || (count % 4) != 0) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Wrong number of values in GeoKeys tag"); + return nullptr; + } + + if (geokeys[0] != 1) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported GeoTIFF major version"); + return nullptr; + } + // We only know that we support GeoTIFF 1.0 and 1.1 at that time + if (geokeys[1] != 1 || geokeys[2] > 1) { + pj_log(m_ctx, PJ_LOG_DEBUG_MINOR, + "GeoTIFF %d.%d possibly not handled", geokeys[1], + geokeys[2]); + } + + for (unsigned int i = 4; i + 3 < count; i += 4) { + constexpr unsigned short GTModelTypeGeoKey = 1024; + constexpr unsigned short ModelTypeGeographic = 2; + + constexpr unsigned short GTRasterTypeGeoKey = 1025; + constexpr unsigned short RasterPixelIsArea = 1; + // constexpr unsigned short RasterPixelIsPoint = 2; + + if (geokeys[i] == GTModelTypeGeoKey) { + if (geokeys[i + 3] != ModelTypeGeographic) { + pj_log(m_ctx, PJ_LOG_ERROR, "Only GTModelTypeGeoKey = " + "ModelTypeGeographic is " + "supported"); + return nullptr; + } + } else if (geokeys[i] == GTRasterTypeGeoKey) { + if (geokeys[i + 3] == RasterPixelIsArea) { + pixelIsArea = true; + } + } + } + } + + double hRes = 0; + double vRes = 0; + double westLon = 0; + double northLat = 0; + + double *matrix = nullptr; + if (TIFFGetField(m_hTIFF, TIFFTAG_GEOTRANSMATRIX, &count, &matrix) && + count == 16) { + // If using GDAL to produce a bottom-up georeferencing, it will produce + // a GeoTransformationMatrix, since negative values in GeoPixelScale + // have historically been implementation bugs. + if (matrix[1] != 0 || matrix[4] != 0) { + pj_log(m_ctx, PJ_LOG_ERROR, "Rotational terms not supported in " + "GeoTransformationMatrix tag"); + return nullptr; + } + + westLon = matrix[3]; + hRes = matrix[0]; + northLat = matrix[7]; + vRes = -matrix[5]; // negation to simulate GeoPixelScale convention + } else { + double *geopixelscale = nullptr; + if (TIFFGetField(m_hTIFF, TIFFTAG_GEOPIXELSCALE, &count, + &geopixelscale) != 1) { + pj_log(m_ctx, PJ_LOG_ERROR, "No GeoPixelScale tag"); + return nullptr; + } + if (count != 3) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Wrong number of values in GeoPixelScale tag"); + return nullptr; + } + hRes = geopixelscale[0]; + vRes = geopixelscale[1]; + + double *geotiepoints = nullptr; + if (TIFFGetField(m_hTIFF, TIFFTAG_GEOTIEPOINTS, &count, + &geotiepoints) != 1) { + pj_log(m_ctx, PJ_LOG_ERROR, "No GeoTiePoints tag"); + return nullptr; + } + if (count != 6) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Wrong number of values in GeoTiePoints tag"); + return nullptr; + } + + westLon = geotiepoints[3] - geotiepoints[0] * hRes; + northLat = geotiepoints[4] + geotiepoints[1] * vRes; + } + + if (pixelIsArea) { + westLon += 0.5 * hRes; + northLat -= 0.5 * vRes; + } + + ExtentAndRes extent; + extent.westLon = westLon * DEG_TO_RAD; + extent.northLat = northLat * DEG_TO_RAD; + extent.resLon = hRes * DEG_TO_RAD; + extent.resLat = fabs(vRes) * DEG_TO_RAD; + extent.eastLon = (westLon + hRes * (width - 1)) * DEG_TO_RAD; + extent.southLat = (northLat - vRes * (height - 1)) * DEG_TO_RAD; + + if (vRes < 0) { + std::swap(extent.northLat, extent.southLat); + } + + if (!(fabs(extent.westLon) <= 4 * M_PI && + fabs(extent.eastLon) <= 4 * M_PI && + fabs(extent.northLat) <= M_PI + 1e-5 && + fabs(extent.southLat) <= M_PI + 1e-5 && + extent.westLon < extent.eastLon && + extent.southLat < extent.northLat && extent.resLon > 1e-10 && + extent.resLat > 1e-10)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Inconsistent georeferencing for %s", + m_filename.c_str()); + return nullptr; + } + + auto ret = std::unique_ptr<GTiffGrid>(new GTiffGrid( + m_ctx, m_hTIFF, m_cache, m_fp.get(), m_ifdIdx, m_filename, width, + height, extent, dt, samplesPerPixel, planarConfig, vRes < 0)); + m_ifdIdx++; + m_hasNextGrid = TIFFReadDirectory(m_hTIFF) != 0; + m_nextDirOffset = TIFFCurrentDirOffset(m_hTIFF); + return ret; +} + +// --------------------------------------------------------------------------- + +class GTiffVGridShiftSet : public VerticalShiftGridSet { + + std::unique_ptr<GTiffDataset> m_GTiffDataset; + + GTiffVGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp) + : m_GTiffDataset(new GTiffDataset(ctx, std::move(fp))) {} + + public: + ~GTiffVGridShiftSet() override; + + static std::unique_ptr<GTiffVGridShiftSet> + open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + VerticalShiftGridSet::reassign_context(ctx); + if (m_GTiffDataset) { + m_GTiffDataset->reassign_context(ctx); + } + } + + bool reopen(PJ_CONTEXT *ctx) override { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + m_grids.clear(); + m_GTiffDataset.reset(); + auto fp = FileManager::open_resource_file(ctx, m_name.c_str()); + if (!fp) { + return false; + } + auto newGS = open(ctx, std::move(fp), m_name); + if (newGS) { + m_grids = std::move(newGS->m_grids); + m_GTiffDataset = std::move(newGS->m_GTiffDataset); + } + return !m_grids.empty(); + } +}; + +#endif // TIFF_ENABLED + +// --------------------------------------------------------------------------- + +template <class GridType, class GenericGridType> +static void +insertIntoHierarchy(PJ_CONTEXT *ctx, std::unique_ptr<GridType> &&grid, + const std::string &gridName, const std::string &parentName, + std::vector<std::unique_ptr<GenericGridType>> &topGrids, + std::map<std::string, GridType *> &mapGrids) { + const auto &extent = grid->extentAndRes(); + + // If we have one or both of grid_name and parent_grid_name, try to use + // the names to recreate the hiearchy + if (!gridName.empty()) { + if (mapGrids.find(gridName) != mapGrids.end()) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Several grids called %s found!", + gridName.c_str()); + } + mapGrids[gridName] = grid.get(); + } + bool gridInserted = false; + if (!parentName.empty()) { + auto iter = mapGrids.find(parentName); + if (iter == mapGrids.end()) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Grid %s refers to non-existing parent %s. " + "Using bounding-box method.", + gridName.c_str(), parentName.c_str()); + } else { + if (iter->second->extentAndRes().contains(extent)) { + iter->second->m_children.emplace_back(std::move(grid)); + gridInserted = true; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Grid %s refers to parent %s, but its extent is " + "not included in it. Using bounding-box method.", + gridName.c_str(), parentName.c_str()); + } + } + } else if (!gridName.empty()) { + topGrids.emplace_back(std::move(grid)); + gridInserted = true; + } + + // Fallback to analyzing spatial extents + if (!gridInserted) { + for (const auto &candidateParent : topGrids) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast<GridType *>(candidateParent.get()) + ->insertGrid(ctx, std::move(grid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + topGrids.emplace_back(std::move(grid)); + } + } +} + +#ifdef TIFF_ENABLED +// --------------------------------------------------------------------------- + +class GTiffVGrid : public VerticalShiftGrid { + friend void insertIntoHierarchy<GTiffVGrid, VerticalShiftGrid>( + PJ_CONTEXT *ctx, std::unique_ptr<GTiffVGrid> &&grid, + const std::string &gridName, const std::string &parentName, + std::vector<std::unique_ptr<VerticalShiftGrid>> &topGrids, + std::map<std::string, GTiffVGrid *> &mapGrids); + + std::unique_ptr<GTiffGrid> m_grid; + uint16 m_idxSample; + + public: + GTiffVGrid(std::unique_ptr<GTiffGrid> &&grid, uint16 idxSample); + + ~GTiffVGrid() override; + + bool valueAt(int x, int y, float &out) const override { + return m_grid->valueAt(m_idxSample, x, y, out); + } + + bool isNodata(float val, double /* multiplier */) const override { + return m_grid->isNodata(val); + } + + void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr<GTiffVGrid> &&subgrid); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_grid->reassign_context(ctx); + } + + bool hasChanged() const override { return m_grid->hasChanged(); } +}; + +// --------------------------------------------------------------------------- + +GTiffVGridShiftSet::~GTiffVGridShiftSet() = default; + +// --------------------------------------------------------------------------- + +GTiffVGrid::GTiffVGrid(std::unique_ptr<GTiffGrid> &&grid, uint16 idxSample) + : VerticalShiftGrid(grid->name(), grid->width(), grid->height(), + grid->extentAndRes()), + m_grid(std::move(grid)), m_idxSample(idxSample) {} + +// --------------------------------------------------------------------------- + +GTiffVGrid::~GTiffVGrid() = default; + +// --------------------------------------------------------------------------- + +void GTiffVGrid::insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr<GTiffVGrid> &&subgrid) { + bool gridInserted = false; + const auto &extent = subgrid->extentAndRes(); + for (const auto &candidateParent : m_children) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast<GTiffVGrid *>(candidateParent.get()) + ->insertGrid(ctx, std::move(subgrid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + m_children.emplace_back(std::move(subgrid)); + } +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<GTiffVGridShiftSet> +GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename) { + auto set = std::unique_ptr<GTiffVGridShiftSet>( + new GTiffVGridShiftSet(ctx, std::move(fp))); + set->m_name = filename; + set->m_format = "gtiff"; + if (!set->m_GTiffDataset->openTIFF(filename)) { + return nullptr; + } + uint16 idxSample = 0; + + std::map<std::string, GTiffVGrid *> mapGrids; + for (int ifd = 0;; ++ifd) { + auto grid = set->m_GTiffDataset->nextGrid(); + if (!grid) { + if (ifd == 0) { + return nullptr; + } + break; + } + + const auto subfileType = grid->subfileType(); + if (subfileType != 0 && subfileType != FILETYPE_PAGE) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid subfileType"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has a unsupported subfileType", + ifd); + continue; + } + } + + // Identify the index of the geoid_undulation/vertical_offset + bool foundDescriptionForAtLeastOneSample = false; + bool foundDescriptionForShift = false; + for (int i = 0; i < static_cast<int>(grid->samplesPerPixel()); ++i) { + const auto desc = grid->metadataItem("DESCRIPTION", i); + if (!desc.empty()) { + foundDescriptionForAtLeastOneSample = true; + } + if (desc == "geoid_undulation" || desc == "vertical_offset") { + idxSample = static_cast<uint16>(i); + foundDescriptionForShift = true; + } + } + + if (foundDescriptionForAtLeastOneSample) { + if (!foundDescriptionForShift) { + if (ifd > 0) { + // Assuming that extra IFD without our channel of interest + // can be ignored + // One could imagine to put the accuracy values in separate + // IFD for example + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has no " + "geoid_undulation/vertical_offset channel", + ifd); + continue; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "IFD 0 has channel descriptions, but no " + "geoid_undulation/vertical_offset channel"); + return nullptr; + } + } + } + + if (idxSample >= grid->samplesPerPixel()) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid sample index"); + return nullptr; + } + + const std::string gridName = grid->metadataItem("grid_name"); + const std::string parentName = grid->metadataItem("parent_grid_name"); + + auto vgrid = + internal::make_unique<GTiffVGrid>(std::move(grid), idxSample); + + insertIntoHierarchy(ctx, std::move(vgrid), gridName, parentName, + set->m_grids, mapGrids); + } + return set; +} +#endif // TIFF_ENABLED + +// --------------------------------------------------------------------------- + +std::unique_ptr<VerticalShiftGridSet> +VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { + if (filename == "null") { + auto set = + std::unique_ptr<VerticalShiftGridSet>(new VerticalShiftGridSet()); + set->m_name = filename; + set->m_format = "null"; + set->m_grids.push_back(std::unique_ptr<NullVerticalShiftGrid>( + new NullVerticalShiftGrid())); + return set; + } + + auto fp = FileManager::open_resource_file(ctx, filename.c_str()); + if (!fp) { + return nullptr; + } + const auto actualName(fp->name()); + if (ends_with(actualName, "gtx") || ends_with(actualName, "GTX")) { + auto grid = GTXVerticalShiftGrid::open(ctx, std::move(fp), actualName); + if (!grid) { + return nullptr; + } + auto set = + std::unique_ptr<VerticalShiftGridSet>(new VerticalShiftGridSet()); + set->m_name = actualName; + set->m_format = "gtx"; + set->m_grids.push_back(std::unique_ptr<VerticalShiftGrid>(grid)); + return set; + } + + /* -------------------------------------------------------------------- */ + /* Load a header, to determine the file type. */ + /* -------------------------------------------------------------------- */ + unsigned char header[4]; + size_t header_size = fp->read(header, sizeof(header)); + if (header_size != sizeof(header)) { + return nullptr; + } + fp->seek(0); + + if (IsTIFF(header_size, header)) { +#ifdef TIFF_ENABLED + auto set = GTiffVGridShiftSet::open(ctx, std::move(fp), actualName); + if (!set) + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return set; +#else + pj_log(ctx, PJ_LOG_ERROR, + "TIFF grid, but TIFF support disabled in this build"); + return nullptr; +#endif + } + + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized vertical grid format"); + return nullptr; +} + +// --------------------------------------------------------------------------- + +bool VerticalShiftGridSet::reopen(PJ_CONTEXT *ctx) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + auto newGS = open(ctx, m_name); + m_grids.clear(); + if (newGS) { + m_grids = std::move(newGS->m_grids); + } + return !m_grids.empty(); +} + +// --------------------------------------------------------------------------- + +const VerticalShiftGrid *VerticalShiftGrid::gridAt(double lon, + double lat) const { + for (const auto &child : m_children) { + const auto &extentChild = child->extentAndRes(); + if ((extentChild.fullWorldLongitude() || + (lon >= extentChild.westLon && lon <= extentChild.eastLon)) && + lat >= extentChild.southLat && lat <= extentChild.northLat) { + return child->gridAt(lon, lat); + } + } + return this; +} +// --------------------------------------------------------------------------- + +const VerticalShiftGrid *VerticalShiftGridSet::gridAt(double lon, + double lat) const { + for (const auto &grid : m_grids) { + if (dynamic_cast<NullVerticalShiftGrid *>(grid.get())) { + return grid.get(); + } + const auto &extent = grid->extentAndRes(); + if ((extent.fullWorldLongitude() || + (lon >= extent.westLon && lon <= extent.eastLon)) && + lat >= extent.southLat && lat <= extent.northLat) { + return grid->gridAt(lon, lat); + } + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +void VerticalShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { + for (const auto &grid : m_grids) { + grid->reassign_context(ctx); + } +} + +// --------------------------------------------------------------------------- + +HorizontalShiftGrid::HorizontalShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, + const ExtentAndRes &extentIn) + : Grid(nameIn, widthIn, heightIn, extentIn) {} + +// --------------------------------------------------------------------------- + +HorizontalShiftGrid::~HorizontalShiftGrid() = default; + +// --------------------------------------------------------------------------- + +HorizontalShiftGridSet::HorizontalShiftGridSet() = default; + +// --------------------------------------------------------------------------- + +HorizontalShiftGridSet::~HorizontalShiftGridSet() = default; + +// --------------------------------------------------------------------------- + +class NullHorizontalShiftGrid : public HorizontalShiftGrid { + + public: + NullHorizontalShiftGrid() + : HorizontalShiftGrid("null", 3, 3, globalExtent()) {} + + bool isNullGrid() const override { return true; } + + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; + + void reassign_context(PJ_CONTEXT *) override {} + + bool hasChanged() const override { return false; } +}; + +// --------------------------------------------------------------------------- + +bool NullHorizontalShiftGrid::valueAt(int, int, bool, float &lonShift, + float &latShift) const { + lonShift = 0.0f; + latShift = 0.0f; + return true; +} + +// --------------------------------------------------------------------------- + +static double to_double(const void *data) { + double d; + memcpy(&d, data, sizeof(d)); + return d; +} + +// --------------------------------------------------------------------------- + +class NTv1Grid : public HorizontalShiftGrid { + PJ_CONTEXT *m_ctx; + std::unique_ptr<File> m_fp; + + NTv1Grid(const NTv1Grid &) = delete; + NTv1Grid &operator=(const NTv1Grid &) = delete; + + public: + explicit NTv1Grid(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp, + const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn) + : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), + m_fp(std::move(fp)) {} + + ~NTv1Grid() override; + + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; + + static NTv1Grid *open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } + + bool hasChanged() const override { return m_fp->hasChanged(); } +}; + +// --------------------------------------------------------------------------- + +NTv1Grid::~NTv1Grid() = default; + +// --------------------------------------------------------------------------- + +NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename) { + unsigned char header[192]; + + /* -------------------------------------------------------------------- */ + /* Read the header. */ + /* -------------------------------------------------------------------- */ + if (fp->read(header, sizeof(header)) != sizeof(header)) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + + /* -------------------------------------------------------------------- */ + /* Regularize fields of interest. */ + /* -------------------------------------------------------------------- */ + if (IS_LSB) { + swap_words(header + 8, sizeof(int), 1); + swap_words(header + 24, sizeof(double), 1); + swap_words(header + 40, sizeof(double), 1); + swap_words(header + 56, sizeof(double), 1); + swap_words(header + 72, sizeof(double), 1); + swap_words(header + 88, sizeof(double), 1); + swap_words(header + 104, sizeof(double), 1); + } + + if (*((int *)(header + 8)) != 12) { + pj_log(ctx, PJ_LOG_ERROR, + "NTv1 grid shift file has wrong record count, corrupt?"); + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + + ExtentAndRes extent; + extent.westLon = -to_double(header + 72) * DEG_TO_RAD; + extent.southLat = to_double(header + 24) * DEG_TO_RAD; + extent.eastLon = -to_double(header + 56) * DEG_TO_RAD; + extent.northLat = to_double(header + 40) * DEG_TO_RAD; + extent.resLon = to_double(header + 104) * DEG_TO_RAD; + extent.resLat = to_double(header + 88) * DEG_TO_RAD; + if (!(fabs(extent.westLon) <= 4 * M_PI && + fabs(extent.eastLon) <= 4 * M_PI && + fabs(extent.northLat) <= M_PI + 1e-5 && + fabs(extent.southLat) <= M_PI + 1e-5 && + extent.westLon < extent.eastLon && + extent.southLat < extent.northLat && extent.resLon > 1e-10 && + extent.resLat > 1e-10)) { + pj_log(ctx, PJ_LOG_ERROR, "Inconsistent georeferencing for %s", + filename.c_str()); + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + const int columns = static_cast<int>( + fabs((extent.eastLon - extent.westLon) / extent.resLon + 0.5) + 1); + const int rows = static_cast<int>( + fabs((extent.northLat - extent.southLat) / extent.resLat + 0.5) + 1); + + return new NTv1Grid(ctx, std::move(fp), filename, columns, rows, extent); +} + +// --------------------------------------------------------------------------- + +bool NTv1Grid::valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const { + assert(x >= 0 && y >= 0 && x < m_width && y < m_height); + + double two_doubles[2]; + // NTv1 is organized from east to west ! + m_fp->seek(192 + 2 * sizeof(double) * (y * m_width + m_width - 1 - x)); + if (m_fp->read(&two_doubles[0], sizeof(two_doubles)) != + sizeof(two_doubles)) { + pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return false; + } + if (IS_LSB) { + swap_words(&two_doubles[0], sizeof(double), 2); + } + /* convert seconds to radians */ + latShift = static_cast<float>(two_doubles[0] * ((M_PI / 180.0) / 3600.0)); + // west longitude positive convention ! + lonShift = (compensateNTConvention ? -1 : 1) * + static_cast<float>(two_doubles[1] * ((M_PI / 180.0) / 3600.0)); + + return true; +} + +// --------------------------------------------------------------------------- + +class CTable2Grid : public HorizontalShiftGrid { + PJ_CONTEXT *m_ctx; + std::unique_ptr<File> m_fp; + + CTable2Grid(const CTable2Grid &) = delete; + CTable2Grid &operator=(const CTable2Grid &) = delete; + + public: + CTable2Grid(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn) + : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), + m_fp(std::move(fp)) {} + + ~CTable2Grid() override; + + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; + + static CTable2Grid *open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } + + bool hasChanged() const override { return m_fp->hasChanged(); } +}; + +// --------------------------------------------------------------------------- + +CTable2Grid::~CTable2Grid() = default; + +// --------------------------------------------------------------------------- + +CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename) { + unsigned char header[160]; + + /* -------------------------------------------------------------------- */ + /* Read the header. */ + /* -------------------------------------------------------------------- */ + if (fp->read(header, sizeof(header)) != sizeof(header)) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + + /* -------------------------------------------------------------------- */ + /* Regularize fields of interest. */ + /* -------------------------------------------------------------------- */ + if (!IS_LSB) { + swap_words(header + 96, sizeof(double), 4); + swap_words(header + 128, sizeof(int), 2); + } + + ExtentAndRes extent; + static_assert(sizeof(extent.westLon) == 8, "wrong sizeof"); + static_assert(sizeof(extent.southLat) == 8, "wrong sizeof"); + static_assert(sizeof(extent.resLon) == 8, "wrong sizeof"); + static_assert(sizeof(extent.resLat) == 8, "wrong sizeof"); + memcpy(&extent.westLon, header + 96, 8); + memcpy(&extent.southLat, header + 104, 8); + memcpy(&extent.resLon, header + 112, 8); + memcpy(&extent.resLat, header + 120, 8); + if (!(fabs(extent.westLon) <= 4 * M_PI && + fabs(extent.southLat) <= M_PI + 1e-5 && extent.resLon > 1e-10 && + extent.resLat > 1e-10)) { + pj_log(ctx, PJ_LOG_ERROR, "Inconsistent georeferencing for %s", + filename.c_str()); + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + int width; + int height; + memcpy(&width, header + 128, 4); + memcpy(&height, header + 132, 4); + if (width <= 0 || height <= 0) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + extent.eastLon = extent.westLon + (width - 1) * extent.resLon; + extent.northLat = extent.southLat + (height - 1) * extent.resLon; + + return new CTable2Grid(ctx, std::move(fp), filename, width, height, extent); +} + +// --------------------------------------------------------------------------- + +bool CTable2Grid::valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const { + assert(x >= 0 && y >= 0 && x < m_width && y < m_height); + + float two_floats[2]; + m_fp->seek(160 + 2 * sizeof(float) * (y * m_width + x)); + if (m_fp->read(&two_floats[0], sizeof(two_floats)) != sizeof(two_floats)) { + pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return false; + } + if (!IS_LSB) { + swap_words(&two_floats[0], sizeof(float), 2); + } + + latShift = two_floats[1]; + // west longitude positive convention ! + lonShift = (compensateNTConvention ? -1 : 1) * two_floats[0]; + + return true; +} + +// --------------------------------------------------------------------------- + +class NTv2GridSet : public HorizontalShiftGridSet { + std::unique_ptr<File> m_fp; + + NTv2GridSet(const NTv2GridSet &) = delete; + NTv2GridSet &operator=(const NTv2GridSet &) = delete; + + explicit NTv2GridSet(std::unique_ptr<File> &&fp) : m_fp(std::move(fp)) {} + + public: + ~NTv2GridSet() override; + + static std::unique_ptr<NTv2GridSet> open(PJ_CONTEXT *ctx, + std::unique_ptr<File> fp, + const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + HorizontalShiftGridSet::reassign_context(ctx); + m_fp->reassign_context(ctx); + } +}; + +// --------------------------------------------------------------------------- + +class NTv2Grid : public HorizontalShiftGrid { + friend class NTv2GridSet; + + std::string m_name; + PJ_CONTEXT *m_ctx; // owned by the parent NTv2GridSet + File *m_fp; // owned by the parent NTv2GridSet + unsigned long long m_offset; + bool m_mustSwap; + + NTv2Grid(const NTv2Grid &) = delete; + NTv2Grid &operator=(const NTv2Grid &) = delete; + + public: + NTv2Grid(const std::string &nameIn, PJ_CONTEXT *ctx, File *fp, + unsigned long long offsetIn, bool mustSwapIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), + m_name(nameIn), m_ctx(ctx), m_fp(fp), m_offset(offsetIn), + m_mustSwap(mustSwapIn) {} + + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } + + bool hasChanged() const override { return m_fp->hasChanged(); } +}; + +// --------------------------------------------------------------------------- + +bool NTv2Grid::valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const { + assert(x >= 0 && y >= 0 && x < m_width && y < m_height); + + float two_float[2]; + // NTv2 is organized from east to west ! + // there are 4 components: lat shift, lon shift, lat error, lon error + m_fp->seek( + m_offset + + 4 * sizeof(float) * + (static_cast<unsigned long long>(y) * m_width + m_width - 1 - x)); + if (m_fp->read(&two_float[0], sizeof(two_float)) != sizeof(two_float)) { + pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return false; + } + if (m_mustSwap) { + swap_words(&two_float[0], sizeof(float), 2); + } + /* convert seconds to radians */ + latShift = static_cast<float>(two_float[0] * ((M_PI / 180.0) / 3600.0)); + // west longitude positive convention ! + lonShift = (compensateNTConvention ? -1 : 1) * + static_cast<float>(two_float[1] * ((M_PI / 180.0) / 3600.0)); + return true; +} + +// --------------------------------------------------------------------------- + +NTv2GridSet::~NTv2GridSet() = default; + +// --------------------------------------------------------------------------- + +std::unique_ptr<NTv2GridSet> NTv2GridSet::open(PJ_CONTEXT *ctx, + std::unique_ptr<File> fp, + const std::string &filename) { + File *fpRaw = fp.get(); + auto set = std::unique_ptr<NTv2GridSet>(new NTv2GridSet(std::move(fp))); + set->m_name = filename; + set->m_format = "ntv2"; + + char header[11 * 16]; + + /* -------------------------------------------------------------------- */ + /* Read the header. */ + /* -------------------------------------------------------------------- */ + if (fpRaw->read(header, sizeof(header)) != sizeof(header)) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + + constexpr int OFFSET_GS_TYPE = 56; + if (memcmp(header + OFFSET_GS_TYPE, "SECONDS", 7) != 0) { + pj_log(ctx, PJ_LOG_ERROR, "Only GS_TYPE=SECONDS is supported"); + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + + const bool must_swap = (header[8] == 11) ? !IS_LSB : IS_LSB; + constexpr int OFFSET_NUM_SUBFILES = 8 + 32; + if (must_swap) { + // swap_words( header+8, 4, 1 ); + // swap_words( header+8+16, 4, 1 ); + swap_words(header + OFFSET_NUM_SUBFILES, 4, 1); + // swap_words( header+8+7*16, 8, 1 ); + // swap_words( header+8+8*16, 8, 1 ); + // swap_words( header+8+9*16, 8, 1 ); + // swap_words( header+8+10*16, 8, 1 ); + } + + /* -------------------------------------------------------------------- */ + /* Get the subfile count out ... all we really use for now. */ + /* -------------------------------------------------------------------- */ + unsigned int num_subfiles; + memcpy(&num_subfiles, header + OFFSET_NUM_SUBFILES, 4); + + std::map<std::string, NTv2Grid *> mapGrids; + + /* ==================================================================== */ + /* Step through the subfiles, creating a grid for each. */ + /* ==================================================================== */ + for (unsigned subfile = 0; subfile < num_subfiles; subfile++) { + // Read header + if (fpRaw->read(header, sizeof(header)) != sizeof(header)) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + + if (strncmp(header, "SUB_NAME", 8) != 0) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + + // Byte swap interesting fields if needed. + constexpr int OFFSET_GS_COUNT = 8 + 16 * 10; + constexpr int OFFSET_SOUTH_LAT = 8 + 16 * 4; + if (must_swap) { + // 6 double values: southLat, northLat, eastLon, westLon, resLat, + // resLon + swap_words(header + OFFSET_SOUTH_LAT, sizeof(double), 6); + swap_words(header + OFFSET_GS_COUNT, sizeof(int), 1); + } + + std::string gridName; + gridName.append(header + 8, 8); + + ExtentAndRes extent; + extent.southLat = to_double(header + OFFSET_SOUTH_LAT) * DEG_TO_RAD / + 3600.0; /* S_LAT */ + extent.northLat = to_double(header + OFFSET_SOUTH_LAT + 16) * + DEG_TO_RAD / 3600.0; /* N_LAT */ + extent.eastLon = -to_double(header + OFFSET_SOUTH_LAT + 16 * 2) * + DEG_TO_RAD / 3600.0; /* E_LONG */ + extent.westLon = -to_double(header + OFFSET_SOUTH_LAT + 16 * 3) * + DEG_TO_RAD / 3600.0; /* W_LONG */ + extent.resLat = + to_double(header + OFFSET_SOUTH_LAT + 16 * 4) * DEG_TO_RAD / 3600.0; + extent.resLon = + to_double(header + OFFSET_SOUTH_LAT + 16 * 5) * DEG_TO_RAD / 3600.0; + + if (!(fabs(extent.westLon) <= 4 * M_PI && + fabs(extent.eastLon) <= 4 * M_PI && + fabs(extent.northLat) <= M_PI + 1e-5 && + fabs(extent.southLat) <= M_PI + 1e-5 && + extent.westLon < extent.eastLon && + extent.southLat < extent.northLat && extent.resLon > 1e-10 && + extent.resLat > 1e-10)) { + pj_log(ctx, PJ_LOG_ERROR, "Inconsistent georeferencing for %s", + filename.c_str()); + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + const int columns = static_cast<int>( + fabs((extent.eastLon - extent.westLon) / extent.resLon + 0.5) + 1); + const int rows = static_cast<int>( + fabs((extent.northLat - extent.southLat) / extent.resLat + 0.5) + + 1); + + pj_log(ctx, PJ_LOG_DEBUG_MINOR, + "NTv2 %s %dx%d: LL=(%.9g,%.9g) UR=(%.9g,%.9g)", gridName.c_str(), + columns, rows, extent.westLon * RAD_TO_DEG, + extent.southLat * RAD_TO_DEG, extent.eastLon * RAD_TO_DEG, + extent.northLat * RAD_TO_DEG); + + unsigned int gs_count; + memcpy(&gs_count, header + OFFSET_GS_COUNT, 4); + if (gs_count / columns != static_cast<unsigned>(rows)) { + pj_log(ctx, PJ_LOG_ERROR, + "GS_COUNT(%u) does not match expected cells (%dx%d)", + gs_count, columns, rows); + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } + + const auto offset = fpRaw->tell(); + auto grid = std::unique_ptr<NTv2Grid>( + new NTv2Grid(filename + ", " + gridName, ctx, fpRaw, offset, + must_swap, columns, rows, extent)); + std::string parentName; + parentName.assign(header + 24, 8); + auto iter = mapGrids.find(parentName); + auto gridPtr = grid.get(); + if (iter == mapGrids.end()) { + set->m_grids.emplace_back(std::move(grid)); + } else { + iter->second->m_children.emplace_back(std::move(grid)); + } + mapGrids[gridName] = gridPtr; + + // Skip grid data. 4 components of size float + fpRaw->seek(static_cast<unsigned long long>(gs_count) * 4 * 4, + SEEK_CUR); + } + return set; +} + +#ifdef TIFF_ENABLED + +// --------------------------------------------------------------------------- + +class GTiffHGridShiftSet : public HorizontalShiftGridSet { + + std::unique_ptr<GTiffDataset> m_GTiffDataset; + + GTiffHGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp) + : m_GTiffDataset(new GTiffDataset(ctx, std::move(fp))) {} + + public: + ~GTiffHGridShiftSet() override; + + static std::unique_ptr<GTiffHGridShiftSet> + open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + HorizontalShiftGridSet::reassign_context(ctx); + if (m_GTiffDataset) { + m_GTiffDataset->reassign_context(ctx); + } + } + + bool reopen(PJ_CONTEXT *ctx) override { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + m_grids.clear(); + m_GTiffDataset.reset(); + auto fp = FileManager::open_resource_file(ctx, m_name.c_str()); + if (!fp) { + return false; + } + auto newGS = open(ctx, std::move(fp), m_name); + if (newGS) { + m_grids = std::move(newGS->m_grids); + m_GTiffDataset = std::move(newGS->m_GTiffDataset); + } + return !m_grids.empty(); + } +}; + +// --------------------------------------------------------------------------- + +class GTiffHGrid : public HorizontalShiftGrid { + friend void insertIntoHierarchy<GTiffHGrid, HorizontalShiftGrid>( + PJ_CONTEXT *ctx, std::unique_ptr<GTiffHGrid> &&grid, + const std::string &gridName, const std::string &parentName, + std::vector<std::unique_ptr<HorizontalShiftGrid>> &topGrids, + std::map<std::string, GTiffHGrid *> &mapGrids); + + std::unique_ptr<GTiffGrid> m_grid; + uint16 m_idxLatShift; + uint16 m_idxLonShift; + double m_convFactorToRadian; + bool m_positiveEast; + + public: + GTiffHGrid(std::unique_ptr<GTiffGrid> &&grid, uint16 idxLatShift, + uint16 idxLonShift, double convFactorToRadian, + bool positiveEast); + + ~GTiffHGrid() override; + + bool valueAt(int x, int y, bool, float &lonShift, + float &latShift) const override; + + void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr<GTiffHGrid> &&subgrid); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_grid->reassign_context(ctx); + } + + bool hasChanged() const override { return m_grid->hasChanged(); } +}; + +// --------------------------------------------------------------------------- + +GTiffHGridShiftSet::~GTiffHGridShiftSet() = default; + +// --------------------------------------------------------------------------- + +GTiffHGrid::GTiffHGrid(std::unique_ptr<GTiffGrid> &&grid, uint16 idxLatShift, + uint16 idxLonShift, double convFactorToRadian, + bool positiveEast) + : HorizontalShiftGrid(grid->name(), grid->width(), grid->height(), + grid->extentAndRes()), + m_grid(std::move(grid)), m_idxLatShift(idxLatShift), + m_idxLonShift(idxLonShift), m_convFactorToRadian(convFactorToRadian), + m_positiveEast(positiveEast) {} + +// --------------------------------------------------------------------------- + +GTiffHGrid::~GTiffHGrid() = default; + +// --------------------------------------------------------------------------- + +bool GTiffHGrid::valueAt(int x, int y, bool, float &lonShift, + float &latShift) const { + if (!m_grid->valueAt(m_idxLatShift, x, y, latShift) || + !m_grid->valueAt(m_idxLonShift, x, y, lonShift)) { + return false; + } + // From arc-seconds to radians + latShift = static_cast<float>(latShift * m_convFactorToRadian); + lonShift = static_cast<float>(lonShift * m_convFactorToRadian); + if (!m_positiveEast) { + lonShift = -lonShift; + } + return true; +} + +// --------------------------------------------------------------------------- + +void GTiffHGrid::insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr<GTiffHGrid> &&subgrid) { + bool gridInserted = false; + const auto &extent = subgrid->extentAndRes(); + for (const auto &candidateParent : m_children) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast<GTiffHGrid *>(candidateParent.get()) + ->insertGrid(ctx, std::move(subgrid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + m_children.emplace_back(std::move(subgrid)); + } +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<GTiffHGridShiftSet> +GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename) { + auto set = std::unique_ptr<GTiffHGridShiftSet>( + new GTiffHGridShiftSet(ctx, std::move(fp))); + set->m_name = filename; + set->m_format = "gtiff"; + if (!set->m_GTiffDataset->openTIFF(filename)) { + return nullptr; + } + + // Defaults inspired from NTv2 + uint16 idxLatShift = 0; + uint16 idxLonShift = 1; + constexpr double ARC_SECOND_TO_RADIAN = (M_PI / 180.0) / 3600.0; + double convFactorToRadian = ARC_SECOND_TO_RADIAN; + bool positiveEast = true; + + std::map<std::string, GTiffHGrid *> mapGrids; + for (int ifd = 0;; ++ifd) { + auto grid = set->m_GTiffDataset->nextGrid(); + if (!grid) { + if (ifd == 0) { + return nullptr; + } + break; + } + + const auto subfileType = grid->subfileType(); + if (subfileType != 0 && subfileType != FILETYPE_PAGE) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid subfileType"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has a unsupported subfileType", + ifd); + continue; + } + } + + if (grid->samplesPerPixel() < 2) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, + "At least 2 samples per pixel needed"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has not at least 2 samples", ifd); + continue; + } + } + + // Identify the index of the latitude and longitude offset channels + bool foundDescriptionForAtLeastOneSample = false; + bool foundDescriptionForLatOffset = false; + bool foundDescriptionForLonOffset = false; + for (int i = 0; i < static_cast<int>(grid->samplesPerPixel()); ++i) { + const auto desc = grid->metadataItem("DESCRIPTION", i); + if (!desc.empty()) { + foundDescriptionForAtLeastOneSample = true; + } + if (desc == "latitude_offset") { + idxLatShift = static_cast<uint16>(i); + foundDescriptionForLatOffset = true; + } else if (desc == "longitude_offset") { + idxLonShift = static_cast<uint16>(i); + foundDescriptionForLonOffset = true; + } + } + + if (foundDescriptionForAtLeastOneSample) { + if (!foundDescriptionForLonOffset && + !foundDescriptionForLatOffset) { + if (ifd > 0) { + // Assuming that extra IFD without + // longitude_offset/latitude_offset can be ignored + // One could imagine to put the accuracy values in separate + // IFD for example + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has no " + "longitude_offset/latitude_offset channel", + ifd); + continue; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "IFD 0 has channel descriptions, but no " + "longitude_offset/latitude_offset channel"); + return nullptr; + } + } + } + if (foundDescriptionForLatOffset && !foundDescriptionForLonOffset) { + pj_log(ctx, PJ_LOG_ERROR, + "Found latitude_offset channel, but not longitude_offset"); + return nullptr; + } else if (foundDescriptionForLonOffset && + !foundDescriptionForLatOffset) { + pj_log(ctx, PJ_LOG_ERROR, + "Found longitude_offset channel, but not latitude_offset"); + return nullptr; + } + + if (idxLatShift >= grid->samplesPerPixel() || + idxLonShift >= grid->samplesPerPixel()) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid sample index"); + return nullptr; + } + + if (foundDescriptionForLonOffset) { + const std::string positiveValue = + grid->metadataItem("positive_value", idxLonShift); + if (!positiveValue.empty()) { + if (positiveValue == "west") { + positiveEast = false; + } else if (positiveValue == "east") { + positiveEast = true; + } else { + pj_log(ctx, PJ_LOG_ERROR, + "Unsupported value %s for 'positive_value'", + positiveValue.c_str()); + return nullptr; + } + } + } + + // Identify their unit + { + const auto unitLatShift = + grid->metadataItem("UNITTYPE", idxLatShift); + const auto unitLonShift = + grid->metadataItem("UNITTYPE", idxLonShift); + if (unitLatShift != unitLonShift) { + pj_log(ctx, PJ_LOG_ERROR, + "Different unit for longitude and latitude offset"); + return nullptr; + } + if (!unitLatShift.empty()) { + if (unitLatShift == "arc-second") { + convFactorToRadian = ARC_SECOND_TO_RADIAN; + } else if (unitLatShift == "radian") { + convFactorToRadian = 1.0; + } else if (unitLatShift == "degree") { + convFactorToRadian = M_PI / 180.0; + } else { + pj_log(ctx, PJ_LOG_ERROR, "Unsupported unit %s", + unitLatShift.c_str()); + return nullptr; + } + } + } + + const std::string gridName = grid->metadataItem("grid_name"); + const std::string parentName = grid->metadataItem("parent_grid_name"); + + auto hgrid = internal::make_unique<GTiffHGrid>( + std::move(grid), idxLatShift, idxLonShift, convFactorToRadian, + positiveEast); + + insertIntoHierarchy(ctx, std::move(hgrid), gridName, parentName, + set->m_grids, mapGrids); + } + return set; +} +#endif // TIFF_ENABLED + +// --------------------------------------------------------------------------- + +std::unique_ptr<HorizontalShiftGridSet> +HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { + if (filename == "null") { + auto set = std::unique_ptr<HorizontalShiftGridSet>( + new HorizontalShiftGridSet()); + set->m_name = filename; + set->m_format = "null"; + set->m_grids.push_back(std::unique_ptr<NullHorizontalShiftGrid>( + new NullHorizontalShiftGrid())); + return set; + } + + auto fp = FileManager::open_resource_file(ctx, filename.c_str()); + if (!fp) { + return nullptr; + } + const auto actualName(fp->name()); + + char header[160]; + /* -------------------------------------------------------------------- */ + /* Load a header, to determine the file type. */ + /* -------------------------------------------------------------------- */ + size_t header_size = fp->read(header, sizeof(header)); + if (header_size != sizeof(header)) { + /* some files may be smaller that sizeof(header), eg 160, so */ + ctx->last_errno = 0; /* don't treat as a persistent error */ + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "pj_gridinfo_init: short header read of %d bytes", + (int)header_size); + } + fp->seek(0); + + /* -------------------------------------------------------------------- */ + /* Determine file type. */ + /* -------------------------------------------------------------------- */ + if (header_size >= 144 + 16 && strncmp(header + 0, "HEADER", 6) == 0 && + strncmp(header + 96, "W GRID", 6) == 0 && + strncmp(header + 144, "TO NAD83 ", 16) == 0) { + auto grid = NTv1Grid::open(ctx, std::move(fp), actualName); + if (!grid) { + return nullptr; + } + auto set = std::unique_ptr<HorizontalShiftGridSet>( + new HorizontalShiftGridSet()); + set->m_name = actualName; + set->m_format = "ntv1"; + set->m_grids.push_back(std::unique_ptr<HorizontalShiftGrid>(grid)); + return set; + } else if (header_size >= 9 && strncmp(header + 0, "CTABLE V2", 9) == 0) { + auto grid = CTable2Grid::open(ctx, std::move(fp), actualName); + if (!grid) { + return nullptr; + } + auto set = std::unique_ptr<HorizontalShiftGridSet>( + new HorizontalShiftGridSet()); + set->m_name = actualName; + set->m_format = "ctable2"; + set->m_grids.push_back(std::unique_ptr<HorizontalShiftGrid>(grid)); + return set; + } else if (header_size >= 48 + 7 && + strncmp(header + 0, "NUM_OREC", 8) == 0 && + strncmp(header + 48, "GS_TYPE", 7) == 0) { + return NTv2GridSet::open(ctx, std::move(fp), actualName); + } else if (IsTIFF(header_size, + reinterpret_cast<const unsigned char *>(header))) { +#ifdef TIFF_ENABLED + auto set = GTiffHGridShiftSet::open(ctx, std::move(fp), actualName); + if (!set) + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return set; +#else + pj_log(ctx, PJ_LOG_ERROR, + "TIFF grid, but TIFF support disabled in this build"); + return nullptr; +#endif + } + + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized horizontal grid format"); + return nullptr; +} + +// --------------------------------------------------------------------------- + +bool HorizontalShiftGridSet::reopen(PJ_CONTEXT *ctx) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + auto newGS = open(ctx, m_name); + m_grids.clear(); + if (newGS) { + m_grids = std::move(newGS->m_grids); + } + return !m_grids.empty(); +} + +// --------------------------------------------------------------------------- + +#define REL_TOLERANCE_HGRIDSHIFT 1e-5 + +const HorizontalShiftGrid *HorizontalShiftGrid::gridAt(double lon, + double lat) const { + for (const auto &child : m_children) { + const auto &extentChild = child->extentAndRes(); + const double epsilon = (extentChild.resLon + extentChild.resLat) * + REL_TOLERANCE_HGRIDSHIFT; + if ((extentChild.fullWorldLongitude() || + (lon + epsilon >= extentChild.westLon && + lon - epsilon <= extentChild.eastLon)) && + lat + epsilon >= extentChild.southLat && + lat - epsilon <= extentChild.northLat) { + return child->gridAt(lon, lat); + } + } + return this; +} +// --------------------------------------------------------------------------- + +const HorizontalShiftGrid *HorizontalShiftGridSet::gridAt(double lon, + double lat) const { + for (const auto &grid : m_grids) { + if (dynamic_cast<NullHorizontalShiftGrid *>(grid.get())) { + return grid.get(); + } + const auto &extent = grid->extentAndRes(); + const double epsilon = + (extent.resLon + extent.resLat) * REL_TOLERANCE_HGRIDSHIFT; + if ((extent.fullWorldLongitude() || + (lon + epsilon >= extent.westLon && + lon - epsilon <= extent.eastLon)) && + lat + epsilon >= extent.southLat && + lat - epsilon <= extent.northLat) { + return grid->gridAt(lon, lat); + } + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +void HorizontalShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { + for (const auto &grid : m_grids) { + grid->reassign_context(ctx); + } +} + +#ifdef TIFF_ENABLED +// --------------------------------------------------------------------------- + +class GTiffGenericGridShiftSet : public GenericShiftGridSet { + + std::unique_ptr<GTiffDataset> m_GTiffDataset; + + GTiffGenericGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp) + : m_GTiffDataset(new GTiffDataset(ctx, std::move(fp))) {} + + public: + ~GTiffGenericGridShiftSet() override; + + static std::unique_ptr<GTiffGenericGridShiftSet> + open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + GenericShiftGridSet::reassign_context(ctx); + if (m_GTiffDataset) { + m_GTiffDataset->reassign_context(ctx); + } + } + + bool reopen(PJ_CONTEXT *ctx) override { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + m_grids.clear(); + m_GTiffDataset.reset(); + auto fp = FileManager::open_resource_file(ctx, m_name.c_str()); + if (!fp) { + return false; + } + auto newGS = open(ctx, std::move(fp), m_name); + if (newGS) { + m_grids = std::move(newGS->m_grids); + m_GTiffDataset = std::move(newGS->m_GTiffDataset); + } + return !m_grids.empty(); + } +}; + +// --------------------------------------------------------------------------- + +class GTiffGenericGrid : public GenericShiftGrid { + friend void insertIntoHierarchy<GTiffGenericGrid, GenericShiftGrid>( + PJ_CONTEXT *ctx, std::unique_ptr<GTiffGenericGrid> &&grid, + const std::string &gridName, const std::string &parentName, + std::vector<std::unique_ptr<GenericShiftGrid>> &topGrids, + std::map<std::string, GTiffGenericGrid *> &mapGrids); + + std::unique_ptr<GTiffGrid> m_grid; + + public: + GTiffGenericGrid(std::unique_ptr<GTiffGrid> &&grid); + + ~GTiffGenericGrid() override; + + bool valueAt(int x, int y, int sample, float &out) const override; + + int samplesPerPixel() const override { return m_grid->samplesPerPixel(); } + + std::string unit(int sample) const override { + return m_grid->metadataItem("UNITTYPE", sample); + } + + std::string description(int sample) const override { + return m_grid->metadataItem("DESCRIPTION", sample); + } + + std::string metadataItem(const std::string &key, + int sample = -1) const override { + return m_grid->metadataItem(key, sample); + } + + void insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr<GTiffGenericGrid> &&subgrid); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_grid->reassign_context(ctx); + } + + bool hasChanged() const override { return m_grid->hasChanged(); } +}; + +// --------------------------------------------------------------------------- + +GTiffGenericGridShiftSet::~GTiffGenericGridShiftSet() = default; + +// --------------------------------------------------------------------------- + +GTiffGenericGrid::GTiffGenericGrid(std::unique_ptr<GTiffGrid> &&grid) + : GenericShiftGrid(grid->name(), grid->width(), grid->height(), + grid->extentAndRes()), + m_grid(std::move(grid)) {} + +// --------------------------------------------------------------------------- + +GTiffGenericGrid::~GTiffGenericGrid() = default; + +// --------------------------------------------------------------------------- + +bool GTiffGenericGrid::valueAt(int x, int y, int sample, float &out) const { + if (sample < 0 || + static_cast<unsigned>(sample) >= m_grid->samplesPerPixel()) + return false; + return m_grid->valueAt(static_cast<uint16>(sample), x, y, out); +} + +// --------------------------------------------------------------------------- + +void GTiffGenericGrid::insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr<GTiffGenericGrid> &&subgrid) { + bool gridInserted = false; + const auto &extent = subgrid->extentAndRes(); + for (const auto &candidateParent : m_children) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast<GTiffGenericGrid *>(candidateParent.get()) + ->insertGrid(ctx, std::move(subgrid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + m_children.emplace_back(std::move(subgrid)); + } +} +#endif // TIFF_ENABLED + +// --------------------------------------------------------------------------- + +class NullGenericShiftGrid : public GenericShiftGrid { + + public: + NullGenericShiftGrid() : GenericShiftGrid("null", 3, 3, globalExtent()) {} + + bool isNullGrid() const override { return true; } + bool valueAt(int, int, int, float &out) const override; + + int samplesPerPixel() const override { return 0; } + + std::string unit(int) const override { return std::string(); } + + std::string description(int) const override { return std::string(); } + + std::string metadataItem(const std::string &, int) const override { + return std::string(); + } + + void reassign_context(PJ_CONTEXT *) override {} + + bool hasChanged() const override { return false; } +}; + +// --------------------------------------------------------------------------- + +bool NullGenericShiftGrid::valueAt(int, int, int, float &out) const { + out = 0.0f; + return true; +} + +// --------------------------------------------------------------------------- + +#ifdef TIFF_ENABLED + +std::unique_ptr<GTiffGenericGridShiftSet> +GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename) { + auto set = std::unique_ptr<GTiffGenericGridShiftSet>( + new GTiffGenericGridShiftSet(ctx, std::move(fp))); + set->m_name = filename; + set->m_format = "gtiff"; + if (!set->m_GTiffDataset->openTIFF(filename)) { + return nullptr; + } + + std::map<std::string, GTiffGenericGrid *> mapGrids; + for (int ifd = 0;; ++ifd) { + auto grid = set->m_GTiffDataset->nextGrid(); + if (!grid) { + if (ifd == 0) { + return nullptr; + } + break; + } + + const auto subfileType = grid->subfileType(); + if (subfileType != 0 && subfileType != FILETYPE_PAGE) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid subfileType"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has a unsupported subfileType", + ifd); + continue; + } + } + + const std::string gridName = grid->metadataItem("grid_name"); + const std::string parentName = grid->metadataItem("parent_grid_name"); + + auto hgrid = internal::make_unique<GTiffGenericGrid>(std::move(grid)); + + insertIntoHierarchy(ctx, std::move(hgrid), gridName, parentName, + set->m_grids, mapGrids); + } + return set; +} +#endif // TIFF_ENABLED + +// --------------------------------------------------------------------------- + +GenericShiftGrid::GenericShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : Grid(nameIn, widthIn, heightIn, extentIn) {} + +// --------------------------------------------------------------------------- + +GenericShiftGrid::~GenericShiftGrid() = default; + +// --------------------------------------------------------------------------- + +GenericShiftGridSet::GenericShiftGridSet() = default; + +// --------------------------------------------------------------------------- + +GenericShiftGridSet::~GenericShiftGridSet() = default; + +// --------------------------------------------------------------------------- + +std::unique_ptr<GenericShiftGridSet> +GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { + if (filename == "null") { + auto set = + std::unique_ptr<GenericShiftGridSet>(new GenericShiftGridSet()); + set->m_name = filename; + set->m_format = "null"; + set->m_grids.push_back( + std::unique_ptr<NullGenericShiftGrid>(new NullGenericShiftGrid())); + return set; + } + + auto fp = FileManager::open_resource_file(ctx, filename.c_str()); + if (!fp) { + return nullptr; + } + const auto actualName(fp->name()); + + /* -------------------------------------------------------------------- */ + /* Load a header, to determine the file type. */ + /* -------------------------------------------------------------------- */ + unsigned char header[4]; + size_t header_size = fp->read(header, sizeof(header)); + if (header_size != sizeof(header)) { + return nullptr; + } + fp->seek(0); + + if (IsTIFF(header_size, header)) { +#ifdef TIFF_ENABLED + auto set = + GTiffGenericGridShiftSet::open(ctx, std::move(fp), actualName); + if (!set) + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return set; +#else + pj_log(ctx, PJ_LOG_ERROR, + "TIFF grid, but TIFF support disabled in this build"); + return nullptr; +#endif + } + + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized generic grid format"); + return nullptr; +} + +// --------------------------------------------------------------------------- + +bool GenericShiftGridSet::reopen(PJ_CONTEXT *ctx) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + auto newGS = open(ctx, m_name); + m_grids.clear(); + if (newGS) { + m_grids = std::move(newGS->m_grids); + } + return !m_grids.empty(); +} + +// --------------------------------------------------------------------------- + +const GenericShiftGrid *GenericShiftGrid::gridAt(double lon, double lat) const { + for (const auto &child : m_children) { + const auto &extentChild = child->extentAndRes(); + if ((extentChild.fullWorldLongitude() || + (lon >= extentChild.westLon && lon <= extentChild.eastLon)) && + lat >= extentChild.southLat && lat <= extentChild.northLat) { + return child->gridAt(lon, lat); + } + } + return this; +} + +// --------------------------------------------------------------------------- + +const GenericShiftGrid *GenericShiftGridSet::gridAt(double lon, + double lat) const { + for (const auto &grid : m_grids) { + if (dynamic_cast<NullGenericShiftGrid *>(grid.get())) { + return grid.get(); + } + const auto &extent = grid->extentAndRes(); + if ((extent.fullWorldLongitude() || + (lon >= extent.westLon && lon <= extent.eastLon)) && + lat >= extent.southLat && lat <= extent.northLat) { + return grid->gridAt(lon, lat); + } + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +void GenericShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { + for (const auto &grid : m_grids) { + grid->reassign_context(ctx); + } +} + +// --------------------------------------------------------------------------- + +ListOfGenericGrids pj_generic_grid_init(PJ *P, const char *gridkey) { + std::string key("s"); + key += gridkey; + const char *gridnames = pj_param(P->ctx, P->params, key.c_str()).s; + if (gridnames == nullptr) + return {}; + + auto listOfGridNames = internal::split(std::string(gridnames), ','); + ListOfGenericGrids grids; + for (const auto &gridnameStr : listOfGridNames) { + const char *gridname = gridnameStr.c_str(); + bool canFail = false; + if (gridname[0] == '@') { + canFail = true; + gridname++; + } + auto gridSet = GenericShiftGridSet::open(P->ctx, gridname); + if (!gridSet) { + if (!canFail) { + if (proj_context_errno(P->ctx) != PJD_ERR_NETWORK_ERROR) { + pj_ctx_set_errno(P->ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + } + return {}; + } + pj_ctx_set_errno(P->ctx, 0); // don't treat as a persistent error + } else { + grids.emplace_back(std::move(gridSet)); + } + } + + return grids; +} + +// --------------------------------------------------------------------------- + +static const HorizontalShiftGrid * +findGrid(const ListOfHGrids &grids, const PJ_LP &input, + HorizontalShiftGridSet *&gridSetOut) { + for (const auto &gridset : grids) { + auto grid = gridset->gridAt(input.lam, input.phi); + if (grid) { + gridSetOut = gridset.get(); + return grid; + } + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +static ListOfHGrids getListOfGridSets(PJ_CONTEXT *ctx, const char *grids) { + ListOfHGrids list; + auto listOfGrids = internal::split(std::string(grids), ','); + for (const auto &grid : listOfGrids) { + const char *gridname = grid.c_str(); + bool canFail = false; + if (gridname[0] == '@') { + canFail = true; + gridname++; + } + auto gridSet = HorizontalShiftGridSet::open(ctx, gridname); + if (!gridSet) { + if (!canFail) { + if (proj_context_errno(ctx) != PJD_ERR_NETWORK_ERROR) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + } + return {}; + } + pj_ctx_set_errno(ctx, 0); // don't treat as a persistent error + } else { + list.emplace_back(std::move(gridSet)); + } + } + return list; +} + +/**********************************************/ +ListOfHGrids pj_hgrid_init(PJ *P, const char *gridkey) { + /********************************************** + + Initizalize and populate list of horizontal + grids. + + Takes a PJ-object and the plus-parameter + name that is used in the proj-string to + specify the grids to load, e.g. "+grids". + The + should be left out here. + + Returns the number of loaded grids. + + ***********************************************/ + + std::string key("s"); + key += gridkey; + const char *grids = pj_param(P->ctx, P->params, key.c_str()).s; + if (grids == nullptr) + return {}; + + return getListOfGridSets(P->ctx, grids); +} + +// --------------------------------------------------------------------------- + +typedef struct { pj_int32 lam, phi; } ILP; + +// Apply bilinear interpolation for horizontal shift grids +static PJ_LP pj_hgrid_interpolate(PJ_LP t, const HorizontalShiftGrid *grid, + bool compensateNTConvention) { + PJ_LP val, frct; + ILP indx; + int in; + + const auto &extent = grid->extentAndRes(); + t.lam /= extent.resLon; + indx.lam = std::isnan(t.lam) ? 0 : (pj_int32)lround(floor(t.lam)); + t.phi /= extent.resLat; + indx.phi = std::isnan(t.phi) ? 0 : (pj_int32)lround(floor(t.phi)); + + frct.lam = t.lam - indx.lam; + frct.phi = t.phi - indx.phi; + val.lam = val.phi = HUGE_VAL; + if (indx.lam < 0) { + if (indx.lam == -1 && frct.lam > 1 - 10 * REL_TOLERANCE_HGRIDSHIFT) { + ++indx.lam; + frct.lam = 0.; + } else + return val; + } else if ((in = indx.lam + 1) >= grid->width()) { + if (in == grid->width() && frct.lam < 10 * REL_TOLERANCE_HGRIDSHIFT) { + --indx.lam; + frct.lam = 1.; + } else + return val; + } + if (indx.phi < 0) { + if (indx.phi == -1 && frct.phi > 1 - 10 * REL_TOLERANCE_HGRIDSHIFT) { + ++indx.phi; + frct.phi = 0.; + } else + return val; + } else if ((in = indx.phi + 1) >= grid->height()) { + if (in == grid->height() && frct.phi < 10 * REL_TOLERANCE_HGRIDSHIFT) { + --indx.phi; + frct.phi = 1.; + } else + return val; + } + + float f00Lon = 0, f00Lat = 0; + float f10Lon = 0, f10Lat = 0; + float f01Lon = 0, f01Lat = 0; + float f11Lon = 0, f11Lat = 0; + if (!grid->valueAt(indx.lam, indx.phi, compensateNTConvention, f00Lon, + f00Lat) || + !grid->valueAt(indx.lam + 1, indx.phi, compensateNTConvention, f10Lon, + f10Lat) || + !grid->valueAt(indx.lam, indx.phi + 1, compensateNTConvention, f01Lon, + f01Lat) || + !grid->valueAt(indx.lam + 1, indx.phi + 1, compensateNTConvention, + f11Lon, f11Lat)) { + return val; + } + + double m10 = frct.lam; + double m11 = m10; + double m01 = 1. - frct.lam; + double m00 = m01; + m11 *= frct.phi; + m01 *= frct.phi; + frct.phi = 1. - frct.phi; + m00 *= frct.phi; + m10 *= frct.phi; + val.lam = m00 * f00Lon + m10 * f10Lon + m01 * f01Lon + m11 * f11Lon; + val.phi = m00 * f00Lat + m10 * f10Lat + m01 * f01Lat + m11 * f11Lat; + return val; +} + +// --------------------------------------------------------------------------- + +#define MAX_ITERATIONS 10 +#define TOL 1e-12 + +static PJ_LP pj_hgrid_apply_internal(projCtx ctx, PJ_LP in, + PJ_DIRECTION direction, + const HorizontalShiftGrid *grid, + HorizontalShiftGridSet *gridset, + const ListOfHGrids &grids, + bool &shouldRetry) { + PJ_LP t, tb, del, dif; + int i = MAX_ITERATIONS; + const double toltol = TOL * TOL; + + shouldRetry = false; + if (in.lam == HUGE_VAL) + return in; + + /* normalize input to ll origin */ + tb = in; + const auto *extent = &(grid->extentAndRes()); + tb.lam -= extent->westLon; + tb.phi -= extent->southLat; + + t = pj_hgrid_interpolate(tb, grid, true); + if (grid->hasChanged()) { + shouldRetry = gridset->reopen(ctx); + return t; + } + if (t.lam == HUGE_VAL) + return t; + + if (direction == PJ_FWD) { + in.lam += t.lam; + in.phi += t.phi; + return in; + } + + t.lam = tb.lam - t.lam; + t.phi = tb.phi - t.phi; + + do { + del = pj_hgrid_interpolate(t, grid, true); + if (grid->hasChanged()) { + shouldRetry = gridset->reopen(ctx); + return t; + } + + /* We can possibly go outside of the initial guessed grid, so try */ + /* to fetch a new grid into which iterate... */ + if (del.lam == HUGE_VAL) { + PJ_LP lp; + lp.lam = t.lam + extent->westLon; + lp.phi = t.phi + extent->southLat; + auto newGrid = findGrid(grids, lp, gridset); + if (newGrid == nullptr || newGrid == grid || newGrid->isNullGrid()) + break; + pj_log(ctx, PJ_LOG_DEBUG_MINOR, "Switching from grid %s to grid %s", + grid->name().c_str(), newGrid->name().c_str()); + grid = newGrid; + extent = &(grid->extentAndRes()); + t.lam = lp.lam - extent->westLon; + t.phi = lp.phi - extent->southLat; + tb = in; + tb.lam -= extent->westLon; + tb.phi -= extent->southLat; + dif.lam = std::numeric_limits<double>::max(); + dif.phi = std::numeric_limits<double>::max(); + continue; + } + + dif.lam = t.lam + del.lam - tb.lam; + dif.phi = t.phi + del.phi - tb.phi; + t.lam -= dif.lam; + t.phi -= dif.phi; + + } while (--i && (dif.lam * dif.lam + dif.phi * dif.phi > + toltol)); /* prob. slightly faster than hypot() */ + + if (i == 0) { + /* If we had access to a context, this should go through pj_log, and we + * should set ctx->errno */ + if (getenv("PROJ_DEBUG")) + fprintf(stderr, + "Inverse grid shift iterator failed to converge.\n"); + t.lam = t.phi = HUGE_VAL; + return t; + } + + /* and again: pj_log and ctx->errno */ + if (del.lam == HUGE_VAL && getenv("PROJ_DEBUG")) + fprintf(stderr, "Inverse grid shift iteration failed, presumably at " + "grid edge.\nUsing first approximation.\n"); + + in.lam = adjlon(t.lam + extent->westLon); + in.phi = t.phi + extent->southLat; + return in; +} + +// --------------------------------------------------------------------------- + +PJ_LP pj_hgrid_apply(PJ_CONTEXT *ctx, const ListOfHGrids &grids, PJ_LP lp, + PJ_DIRECTION direction) { + PJ_LP out; + + out.lam = HUGE_VAL; + out.phi = HUGE_VAL; + + while (true) { + HorizontalShiftGridSet *gridset = nullptr; + const auto grid = findGrid(grids, lp, gridset); + if (!grid) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return out; + } + if (grid->isNullGrid()) { + return lp; + } + + bool shouldRetry = false; + out = pj_hgrid_apply_internal(ctx, lp, direction, grid, gridset, grids, + shouldRetry); + if (!shouldRetry) { + break; + } + } + + if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) + pj_ctx_set_errno(ctx, PJD_ERR_GRID_AREA); + + return out; +} + +/********************************************/ +/* proj_hgrid_value() */ +/* */ +/* Return coordinate offset in grid */ +/********************************************/ +PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp) { + PJ_LP out = proj_coord_error().lp; + + HorizontalShiftGridSet *gridset = nullptr; + const auto grid = findGrid(grids, lp, gridset); + if (!grid) { + pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); + return out; + } + + /* normalize input to ll origin */ + const auto &extent = grid->extentAndRes(); + lp.lam -= extent.westLon; + lp.phi -= extent.southLat; + + lp.lam = adjlon(lp.lam - M_PI) + M_PI; + + out = pj_hgrid_interpolate(lp, grid, false); + if (grid->hasChanged()) { + if (gridset->reopen(P->ctx)) { + return pj_hgrid_value(P, grids, lp); + } + out.lam = HUGE_VAL; + out.phi = HUGE_VAL; + } + + if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) { + pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); + } + + return out; +} + +// --------------------------------------------------------------------------- + +static double read_vgrid_value(PJ_CONTEXT *ctx, const ListOfVGrids &grids, + const PJ_LP &input, const double vmultiplier) { + + /* do not deal with NaN coordinates */ + /* cppcheck-suppress duplicateExpression */ + if (std::isnan(input.phi) || std::isnan(input.lam)) { + return HUGE_VAL; + } + + VerticalShiftGridSet *curGridset = nullptr; + const VerticalShiftGrid *grid = nullptr; + for (const auto &gridset : grids) { + grid = gridset->gridAt(input.lam, input.phi); + if (grid) { + curGridset = gridset.get(); + break; + } + } + if (!grid) { + return HUGE_VAL; + } + + const auto &extent = grid->extentAndRes(); + + /* Interpolation a location within the grid */ + double grid_x = (input.lam - extent.westLon) / extent.resLon; + if (extent.fullWorldLongitude()) { + // The first fmod goes to ]-lim, lim[ range + // So we add lim again to be in ]0, 2*lim[ and fmod again + grid_x = + fmod(fmod(grid_x + grid->width(), grid->width()) + grid->width(), + grid->width()); + } + double grid_y = (input.phi - extent.southLat) / extent.resLat; + int grid_ix = static_cast<int>(lround(floor(grid_x))); + assert(grid_ix >= 0 && grid_ix < grid->width()); + int grid_iy = static_cast<int>(lround(floor(grid_y))); + assert(grid_iy >= 0 && grid_iy < grid->height()); + grid_x -= grid_ix; + grid_y -= grid_iy; + + int grid_ix2 = grid_ix + 1; + if (grid_ix2 >= grid->width()) { + if (extent.fullWorldLongitude()) { + grid_ix2 = 0; + } else { + grid_ix2 = grid->width() - 1; + } + } + int grid_iy2 = grid_iy + 1; + if (grid_iy2 >= grid->height()) + grid_iy2 = grid->height() - 1; + + float value_a = 0; + float value_b = 0; + float value_c = 0; + float value_d = 0; + bool error = (!grid->valueAt(grid_ix, grid_iy, value_a) || + !grid->valueAt(grid_ix2, grid_iy, value_b) || + !grid->valueAt(grid_ix, grid_iy2, value_c) || + !grid->valueAt(grid_ix2, grid_iy2, value_d)); + if (grid->hasChanged()) { + if (curGridset->reopen(ctx)) { + return read_vgrid_value(ctx, grids, input, vmultiplier); + } + error = true; + } + + if (error) { + return HUGE_VAL; + } + + double total_weight = 0.0; + int n_weights = 0; + double value = 0.0f; + + if (!grid->isNodata(value_a, vmultiplier)) { + double weight = (1.0 - grid_x) * (1.0 - grid_y); + value += value_a * weight; + total_weight += weight; + n_weights++; + } + if (!grid->isNodata(value_b, vmultiplier)) { + double weight = (grid_x) * (1.0 - grid_y); + value += value_b * weight; + total_weight += weight; + n_weights++; + } + if (!grid->isNodata(value_c, vmultiplier)) { + double weight = (1.0 - grid_x) * (grid_y); + value += value_c * weight; + total_weight += weight; + n_weights++; + } + if (!grid->isNodata(value_d, vmultiplier)) { + double weight = (grid_x) * (grid_y); + value += value_d * weight; + total_weight += weight; + n_weights++; + } + if (n_weights == 0) + value = HUGE_VAL; + else if (n_weights != 4) + value /= total_weight; + + return value * vmultiplier; +} + +/**********************************************/ +ListOfVGrids pj_vgrid_init(PJ *P, const char *gridkey) { + /********************************************** + + Initizalize and populate gridlist. + + Takes a PJ-object and the plus-parameter + name that is used in the proj-string to + specify the grids to load, e.g. "+grids". + The + should be left out here. + + Returns the number of loaded grids. + + ***********************************************/ + + std::string key("s"); + key += gridkey; + const char *gridnames = pj_param(P->ctx, P->params, key.c_str()).s; + if (gridnames == nullptr) + return {}; + + auto listOfGridNames = internal::split(std::string(gridnames), ','); + ListOfVGrids grids; + for (const auto &gridnameStr : listOfGridNames) { + const char *gridname = gridnameStr.c_str(); + bool canFail = false; + if (gridname[0] == '@') { + canFail = true; + gridname++; + } + auto gridSet = VerticalShiftGridSet::open(P->ctx, gridname); + if (!gridSet) { + if (!canFail) { + if (proj_context_errno(P->ctx) != PJD_ERR_NETWORK_ERROR) { + pj_ctx_set_errno(P->ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + } + return {}; + } + pj_ctx_set_errno(P->ctx, 0); // don't treat as a persistent error + } else { + grids.emplace_back(std::move(gridSet)); + } + } + + return grids; +} + +/***********************************************/ +double pj_vgrid_value(PJ *P, const ListOfVGrids &grids, PJ_LP lp, + double vmultiplier) { + /*********************************************** + + Read grid value at position lp in grids loaded + with proj_grid_init. + + Returns the grid value of the given coordinate. + + ************************************************/ + + double value; + + value = read_vgrid_value(P->ctx, grids, lp, vmultiplier); + proj_log_trace(P, "proj_vgrid_value: (%f, %f) = %f", lp.lam * RAD_TO_DEG, + lp.phi * RAD_TO_DEG, value); + + return value; +} + +// --------------------------------------------------------------------------- + +const GenericShiftGrid *pj_find_generic_grid(const ListOfGenericGrids &grids, + const PJ_LP &input, + GenericShiftGridSet *&gridSetOut) { + for (const auto &gridset : grids) { + auto grid = gridset->gridAt(input.lam, input.phi); + if (grid) { + gridSetOut = gridset.get(); + return grid; + } + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +// Used by +proj=deformation and +proj=xyzgridshift to do bilinear interpolation +// on 3 sample values per node. +bool pj_bilinear_interpolation_three_samples(const GenericShiftGrid *grid, + const PJ_LP &lp, int idx1, + int idx2, int idx3, double &v1, + double &v2, double &v3, + bool &must_retry) { + must_retry = false; + + const auto &extent = grid->extentAndRes(); + double grid_x = (lp.lam - extent.westLon) / extent.resLon; + double grid_y = (lp.phi - extent.southLat) / extent.resLat; + int ix = static_cast<int>(grid_x); + int iy = static_cast<int>(grid_y); + int ix2 = std::min(ix + 1, grid->width() - 1); + int iy2 = std::min(iy + 1, grid->height() - 1); + + float dx1 = 0.0f, dy1 = 0.0f, dz1 = 0.0f; + float dx2 = 0.0f, dy2 = 0.0f, dz2 = 0.0f; + float dx3 = 0.0f, dy3 = 0.0f, dz3 = 0.0f; + float dx4 = 0.0f, dy4 = 0.0f, dz4 = 0.0f; + bool error = (!grid->valueAt(ix, iy, idx1, dx1) || + !grid->valueAt(ix, iy, idx2, dy1) || + !grid->valueAt(ix, iy, idx3, dz1) || + !grid->valueAt(ix2, iy, idx1, dx2) || + !grid->valueAt(ix2, iy, idx2, dy2) || + !grid->valueAt(ix2, iy, idx3, dz2) || + !grid->valueAt(ix, iy2, idx1, dx3) || + !grid->valueAt(ix, iy2, idx2, dy3) || + !grid->valueAt(ix, iy2, idx3, dz3) || + !grid->valueAt(ix2, iy2, idx1, dx4) || + !grid->valueAt(ix2, iy2, idx2, dy4) || + !grid->valueAt(ix2, iy2, idx3, dz4)); + if (grid->hasChanged()) { + must_retry = true; + return false; + } + if (error) { + return false; + } + + double frct_lam = grid_x - ix; + double frct_phi = grid_y - iy; + double m10 = frct_lam; + double m11 = m10; + double m01 = 1. - frct_lam; + double m00 = m01; + m11 *= frct_phi; + m01 *= frct_phi; + frct_phi = 1. - frct_phi; + m00 *= frct_phi; + m10 *= frct_phi; + + v1 = m00 * dx1 + m10 * dx2 + m01 * dx3 + m11 * dx4; + v2 = m00 * dy1 + m10 * dy2 + m01 * dy3 + m11 * dy4; + v3 = m00 * dz1 + m10 * dz2 + m01 * dz3 + m11 * dz4; + return true; +} + +NS_PROJ_END + +/************************************************************************/ +/* pj_apply_gridshift() */ +/* */ +/* This is the externally callable interface - part of the */ +/* public API - though it is not used internally any more and I */ +/* doubt it is used by any other applications. But we preserve */ +/* it to honour our public api. */ +/************************************************************************/ + +int pj_apply_gridshift(projCtx ctx, const char *nadgrids, int inverse, + long point_count, int point_offset, double *x, double *y, + double * /*z */) + +{ + auto hgrids = NS_PROJ::getListOfGridSets(ctx, nadgrids); + if (hgrids.empty()) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return 1; + } + + for (long i = 0; i < point_count; i++) { + PJ_LP input; + + long io = i * point_offset; + input.phi = y[io]; + input.lam = x[io]; + + auto output = + pj_hgrid_apply(ctx, hgrids, input, inverse ? PJ_INV : PJ_FWD); + + if (output.lam != HUGE_VAL) { + y[io] = output.phi; + x[io] = output.lam; + } else { + if (ctx->debug_level >= PJ_LOG_DEBUG_MAJOR) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "pj_apply_gridshift(): failed to find a grid shift " + "table for\n" + " location (%.7fdW,%.7fdN)", + x[io] * RAD_TO_DEG, y[io] * RAD_TO_DEG); + } + } + } + + return 0; +} diff --git a/src/grids.hpp b/src/grids.hpp new file mode 100644 index 00000000..0fd1b7b0 --- /dev/null +++ b/src/grids.hpp @@ -0,0 +1,264 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: Grid management + * Author: Even Rouault, <even.rouault at spatialys.com> + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef GRIDS_HPP_INCLUDED +#define GRIDS_HPP_INCLUDED + +#include <memory> +#include <vector> + +#include "proj.h" +#include "proj/util.hpp" + +NS_PROJ_START + +struct ExtentAndRes { + double westLon; // in radian + double southLat; // in radian + double eastLon; // in radian + double northLat; // in radian + double resLon; // in radian + double resLat; // in radian + + bool fullWorldLongitude() const; + bool contains(const ExtentAndRes &other) const; + bool intersects(const ExtentAndRes &other) const; +}; + +// --------------------------------------------------------------------------- + +class PROJ_GCC_DLL Grid { + protected: + std::string m_name; + int m_width; + int m_height; + ExtentAndRes m_extent; + + Grid(const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn); + + public: + PROJ_FOR_TEST virtual ~Grid(); + + PROJ_FOR_TEST int width() const { return m_width; } + PROJ_FOR_TEST int height() const { return m_height; } + PROJ_FOR_TEST const ExtentAndRes &extentAndRes() const { return m_extent; } + PROJ_FOR_TEST const std::string &name() const { return m_name; } + + PROJ_FOR_TEST virtual bool isNullGrid() const { return false; } + PROJ_FOR_TEST virtual bool hasChanged() const = 0; +}; + +// --------------------------------------------------------------------------- + +class PROJ_GCC_DLL VerticalShiftGrid : public Grid { + protected: + std::vector<std::unique_ptr<VerticalShiftGrid>> m_children{}; + + public: + PROJ_FOR_TEST VerticalShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn); + PROJ_FOR_TEST ~VerticalShiftGrid() override; + + PROJ_FOR_TEST const VerticalShiftGrid *gridAt(double lon, double lat) const; + + PROJ_FOR_TEST virtual bool isNodata(float /*val*/, + double /* multiplier */) const = 0; + + // x = 0 is western-most column, y = 0 is southern-most line + PROJ_FOR_TEST virtual bool valueAt(int x, int y, float &out) const = 0; + + PROJ_FOR_TEST virtual void reassign_context(PJ_CONTEXT *ctx) = 0; +}; + +// --------------------------------------------------------------------------- + +class PROJ_GCC_DLL VerticalShiftGridSet { + protected: + std::string m_name{}; + std::string m_format{}; + std::vector<std::unique_ptr<VerticalShiftGrid>> m_grids{}; + + VerticalShiftGridSet(); + + public: + PROJ_FOR_TEST virtual ~VerticalShiftGridSet(); + + PROJ_FOR_TEST static std::unique_ptr<VerticalShiftGridSet> + open(PJ_CONTEXT *ctx, const std::string &filename); + + PROJ_FOR_TEST const std::string &name() const { return m_name; } + PROJ_FOR_TEST const std::string &format() const { return m_format; } + PROJ_FOR_TEST const std::vector<std::unique_ptr<VerticalShiftGrid>> & + grids() const { + return m_grids; + } + PROJ_FOR_TEST const VerticalShiftGrid *gridAt(double lon, double lat) const; + + PROJ_FOR_TEST virtual void reassign_context(PJ_CONTEXT *ctx); + PROJ_FOR_TEST virtual bool reopen(PJ_CONTEXT *ctx); +}; + +// --------------------------------------------------------------------------- + +class PROJ_GCC_DLL HorizontalShiftGrid : public Grid { + protected: + std::vector<std::unique_ptr<HorizontalShiftGrid>> m_children{}; + + public: + PROJ_FOR_TEST HorizontalShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, + const ExtentAndRes &extentIn); + PROJ_FOR_TEST ~HorizontalShiftGrid() override; + + PROJ_FOR_TEST const HorizontalShiftGrid *gridAt(double lon, + double lat) const; + + // x = 0 is western-most column, y = 0 is southern-most line + PROJ_FOR_TEST virtual bool valueAt(int x, int y, + bool compensateNTConvention, + float &lonShift, + float &latShift) const = 0; + + PROJ_FOR_TEST virtual void reassign_context(PJ_CONTEXT *ctx) = 0; +}; + +// --------------------------------------------------------------------------- + +class PROJ_GCC_DLL HorizontalShiftGridSet { + protected: + std::string m_name{}; + std::string m_format{}; + std::vector<std::unique_ptr<HorizontalShiftGrid>> m_grids{}; + + HorizontalShiftGridSet(); + + public: + PROJ_FOR_TEST virtual ~HorizontalShiftGridSet(); + + PROJ_FOR_TEST static std::unique_ptr<HorizontalShiftGridSet> + open(PJ_CONTEXT *ctx, const std::string &filename); + + PROJ_FOR_TEST const std::string &name() const { return m_name; } + PROJ_FOR_TEST const std::string &format() const { return m_format; } + PROJ_FOR_TEST const std::vector<std::unique_ptr<HorizontalShiftGrid>> & + grids() const { + return m_grids; + } + PROJ_FOR_TEST const HorizontalShiftGrid *gridAt(double lon, + double lat) const; + + PROJ_FOR_TEST virtual void reassign_context(PJ_CONTEXT *ctx); + PROJ_FOR_TEST virtual bool reopen(PJ_CONTEXT *ctx); +}; + +// --------------------------------------------------------------------------- + +class PROJ_GCC_DLL GenericShiftGrid : public Grid { + protected: + std::vector<std::unique_ptr<GenericShiftGrid>> m_children{}; + + public: + PROJ_FOR_TEST GenericShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn); + + PROJ_FOR_TEST ~GenericShiftGrid() override; + + PROJ_FOR_TEST const GenericShiftGrid *gridAt(double lon, double lat) const; + + PROJ_FOR_TEST virtual std::string unit(int sample) const = 0; + + PROJ_FOR_TEST virtual std::string description(int sample) const = 0; + + PROJ_FOR_TEST virtual std::string metadataItem(const std::string &key, + int sample = -1) const = 0; + + PROJ_FOR_TEST virtual int samplesPerPixel() const = 0; + + // x = 0 is western-most column, y = 0 is southern-most line + PROJ_FOR_TEST virtual bool valueAt(int x, int y, int sample, + float &out) const = 0; + + PROJ_FOR_TEST virtual void reassign_context(PJ_CONTEXT *ctx) = 0; +}; + +// --------------------------------------------------------------------------- + +class PROJ_GCC_DLL GenericShiftGridSet { + protected: + std::string m_name{}; + std::string m_format{}; + std::vector<std::unique_ptr<GenericShiftGrid>> m_grids{}; + + GenericShiftGridSet(); + + public: + PROJ_FOR_TEST virtual ~GenericShiftGridSet(); + + PROJ_FOR_TEST static std::unique_ptr<GenericShiftGridSet> + open(PJ_CONTEXT *ctx, const std::string &filename); + + PROJ_FOR_TEST const std::string &name() const { return m_name; } + PROJ_FOR_TEST const std::string &format() const { return m_format; } + PROJ_FOR_TEST const std::vector<std::unique_ptr<GenericShiftGrid>> & + grids() const { + return m_grids; + } + PROJ_FOR_TEST const GenericShiftGrid *gridAt(double lon, double lat) const; + + PROJ_FOR_TEST virtual void reassign_context(PJ_CONTEXT *ctx); + PROJ_FOR_TEST virtual bool reopen(PJ_CONTEXT *ctx); +}; + +// --------------------------------------------------------------------------- + +typedef std::vector<std::unique_ptr<HorizontalShiftGridSet>> ListOfHGrids; +typedef std::vector<std::unique_ptr<VerticalShiftGridSet>> ListOfVGrids; +typedef std::vector<std::unique_ptr<GenericShiftGridSet>> ListOfGenericGrids; + +ListOfVGrids pj_vgrid_init(PJ *P, const char *grids); +ListOfHGrids pj_hgrid_init(PJ *P, const char *grids); +ListOfGenericGrids pj_generic_grid_init(PJ *P, const char *grids); + +PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp); +double pj_vgrid_value(PJ *P, const ListOfVGrids &, PJ_LP lp, + double vmultiplier); +PJ_LP pj_hgrid_apply(PJ_CONTEXT *ctx, const ListOfHGrids &grids, PJ_LP lp, + PJ_DIRECTION direction); + +const GenericShiftGrid *pj_find_generic_grid(const ListOfGenericGrids &grids, + const PJ_LP &input, + GenericShiftGridSet *&gridSetOut); +bool pj_bilinear_interpolation_three_samples(const GenericShiftGrid *grid, + const PJ_LP &lp, int idx1, + int idx2, int idx3, double &v1, + double &v2, double &v3, + bool &must_retry); + +NS_PROJ_END + +#endif // GRIDS_HPP_INCLUDED diff --git a/src/init.cpp b/src/init.cpp index 1ed46e5a..19fcf47b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -646,12 +646,6 @@ pj_init_ctx_with_allow_init_epsg(projCtx ctx, int argc, char **argv, int allow_i PIN->long_wrap_center = 0.0; strcpy( PIN->axis, "enu" ); - PIN->gridlist = nullptr; - PIN->gridlist_count = 0; - - PIN->vgridlist_geoid = nullptr; - PIN->vgridlist_geoid_count = 0; - /* Set datum parameters. Similarly to +init parameters we want to expand */ /* +datum parameters as late as possible when dealing with pipelines. */ /* otherwise only the first occurrence of +datum will be expanded and that */ diff --git a/src/iso19111/c_api.cpp b/src/iso19111/c_api.cpp index 9f73f8e9..4c98af3d 100644 --- a/src/iso19111/c_api.cpp +++ b/src/iso19111/c_api.cpp @@ -177,7 +177,11 @@ static PJ *pj_obj_create(PJ_CONTEXT *ctx, const IdentifiedObjectNNPtr &objIn) { auto formatter = PROJStringFormatter::create( PROJStringFormatter::Convention::PROJ_5, dbContext); auto projString = coordop->exportToPROJString(formatter.get()); + if (pj_context_is_network_enabled(ctx)) { + ctx->defer_grid_opening = true; + } auto pj = pj_create_internal(ctx, projString.c_str()); + ctx->defer_grid_opening = false; if (pj) { pj->iso_obj = objIn; if (ctx->cpp_context) { @@ -766,7 +770,7 @@ int PROJ_DLL proj_grid_get_info_from_database( bool open_license; bool available; if (!db_context->lookForGridInfo( - grid_name, ctx->cpp_context->lastGridFullName_, + grid_name, false, ctx->cpp_context->lastGridFullName_, ctx->cpp_context->lastGridPackageName_, ctx->cpp_context->lastGridUrl_, direct_download, open_license, available)) { @@ -6615,7 +6619,10 @@ int proj_coordoperation_is_instantiable(PJ_CONTEXT *ctx, } auto dbContext = getDBcontextNoException(ctx, __FUNCTION__); try { - auto ret = op->isPROJInstantiable(dbContext) ? 1 : 0; + auto ret = op->isPROJInstantiable(dbContext, + pj_context_is_network_enabled(ctx)) + ? 1 + : 0; if (ctx->cpp_context) { ctx->cpp_context->autoCloseDbIfNeeded(); } @@ -6927,7 +6934,8 @@ int proj_coordoperation_get_grid_used_count(PJ_CONTEXT *ctx, try { if (!coordoperation->gridsNeededAsked) { coordoperation->gridsNeededAsked = true; - const auto gridsNeeded = co->gridsNeeded(dbContext); + const auto gridsNeeded = + co->gridsNeeded(dbContext, pj_context_is_network_enabled(ctx)); for (const auto &gridDesc : gridsNeeded) { coordoperation->gridsNeeded.emplace_back(gridDesc); } @@ -7264,6 +7272,12 @@ void PROJ_DLL proj_operation_factory_context_set_grid_availability_use( CoordinateOperationContext::GridAvailabilityUse:: IGNORE_GRID_AVAILABILITY); break; + + case PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE: + factory_ctx->operationContext->setGridAvailabilityUse( + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE); + break; } } catch (const std::exception &e) { proj_log_error(ctx, __FUNCTION__, e.what()); diff --git a/src/iso19111/coordinateoperation.cpp b/src/iso19111/coordinateoperation.cpp index aaa0c413..a1650046 100644 --- a/src/iso19111/coordinateoperation.cpp +++ b/src/iso19111/coordinateoperation.cpp @@ -788,13 +788,15 @@ void CoordinateOperation::setAccuracies( * available. */ bool CoordinateOperation::isPROJInstantiable( - const io::DatabaseContextPtr &databaseContext) const { + const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { try { exportToPROJString(io::PROJStringFormatter::create().get()); } catch (const std::exception &) { return false; } - for (const auto &gridDesc : gridsNeeded(databaseContext)) { + for (const auto &gridDesc : + gridsNeeded(databaseContext, considerKnownGridsAsAvailable)) { if (!gridDesc.available) { return false; } @@ -2020,8 +2022,9 @@ bool SingleOperation::_isEquivalentTo(const util::IComparable *other, // --------------------------------------------------------------------------- -std::set<GridDescription> SingleOperation::gridsNeeded( - const io::DatabaseContextPtr &databaseContext) const { +std::set<GridDescription> +SingleOperation::gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { std::set<GridDescription> res; for (const auto &genOpParamvalue : parameterValues()) { auto opParamvalue = dynamic_cast<const OperationParameterValue *>( @@ -2033,9 +2036,9 @@ std::set<GridDescription> SingleOperation::gridsNeeded( desc.shortName = value->valueFile(); if (databaseContext) { databaseContext->lookForGridInfo( - desc.shortName, desc.fullName, desc.packageName, - desc.url, desc.directDownload, desc.openLicense, - desc.available); + desc.shortName, considerKnownGridsAsAvailable, + desc.fullName, desc.packageName, desc.url, + desc.directDownload, desc.openLicense, desc.available); } res.insert(desc); } @@ -10233,10 +10236,12 @@ bool ConcatenatedOperation::_isEquivalentTo( // --------------------------------------------------------------------------- std::set<GridDescription> ConcatenatedOperation::gridsNeeded( - const io::DatabaseContextPtr &databaseContext) const { + const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { std::set<GridDescription> res; for (const auto &operation : operations()) { - const auto l_gridsNeeded = operation->gridsNeeded(databaseContext); + const auto l_gridsNeeded = operation->gridsNeeded( + databaseContext, considerKnownGridsAsAvailable); for (const auto &gridDesc : l_gridsNeeded) { res.insert(gridDesc); } @@ -11156,7 +11161,10 @@ struct FilterResults { bool gridsKnown = true; if (context->getAuthorityFactory()) { const auto gridsNeeded = op->gridsNeeded( - context->getAuthorityFactory()->databaseContext()); + context->getAuthorityFactory()->databaseContext(), + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { hasGrids = true; if (gridAvailabilityUse == @@ -11278,6 +11286,7 @@ struct FilterResults { CoordinateOperationPtr lastOp; bool first = true; + const auto gridAvailabilityUse = context->getGridAvailabilityUse(); for (const auto &op : res) { const auto curAccuracy = getAccuracy(op); bool dummy = false; @@ -11290,7 +11299,10 @@ struct FilterResults { if (context->getAuthorityFactory()) { const auto gridsNeeded = op->gridsNeeded( - context->getAuthorityFactory()->databaseContext()); + context->getAuthorityFactory()->databaseContext(), + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { curHasGrids = true; curSetOfGrids.insert(gridDesc.shortName); @@ -11603,6 +11615,7 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirect( buildCRSIds(sourceCRS, context, sourceIds); buildCRSIds(targetCRS, context, targetIds); + const auto gridAvailabilityUse = context.context->getGridAvailabilityUse(); for (const auto &idSrc : sourceIds) { const auto &srcAuthName = idSrc.first; const auto &srcCode = idSrc.second; @@ -11622,9 +11635,16 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirect( tmpAuthFactory->createFromCoordinateReferenceSystemCodes( srcAuthName, srcCode, targetAuthName, targetCode, context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, + gridAvailabilityUse == CoordinateOperationContext::GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, + KNOWN_AVAILABLE, context.context->getDiscardSuperseded(), true, false, context.extent1, context.extent2); res.insert(res.end(), resTmp.begin(), resTmp.end()); @@ -11667,6 +11687,7 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirectTo( std::list<std::pair<std::string, std::string>> ids; buildCRSIds(targetCRS, context, ids); + const auto gridAvailabilityUse = context.context->getGridAvailabilityUse(); for (const auto &id : ids) { const auto &targetAuthName = id.first; const auto &targetCode = id.second; @@ -11680,9 +11701,15 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirectTo( auto res = tmpAuthFactory->createFromCoordinateReferenceSystemCodes( std::string(), std::string(), targetAuthName, targetCode, context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == - CoordinateOperationContext::GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, + + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE, + gridAvailabilityUse == CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, context.context->getDiscardSuperseded(), true, true, context.extent1, context.extent2); if (!res.empty()) { @@ -11730,6 +11757,7 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate( buildCRSIds(sourceCRS, context, sourceIds); buildCRSIds(targetCRS, context, targetIds); + const auto gridAvailabilityUse = context.context->getGridAvailabilityUse(); for (const auto &idSrc : sourceIds) { const auto &srcAuthName = idSrc.first; const auto &srcCode = idSrc.second; @@ -11749,21 +11777,28 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate( std::vector<CoordinateOperationNNPtr> res; if (useCreateBetweenGeodeticCRSWithDatumBasedIntermediates) { - res = tmpAuthFactory - ->createBetweenGeodeticCRSWithDatumBasedIntermediates( - sourceCRS, srcAuthName, srcCode, targetCRS, - targetAuthName, targetCode, - context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == - CoordinateOperationContext:: - GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, - context.context->getDiscardSuperseded(), - authFactory->getAuthority() != "any" && - authorities.size() > 1 - ? authorities - : std::vector<std::string>(), - context.extent1, context.extent2); + res = + tmpAuthFactory + ->createBetweenGeodeticCRSWithDatumBasedIntermediates( + sourceCRS, srcAuthName, srcCode, targetCRS, + targetAuthName, targetCode, + context.context->getUsePROJAlternativeGridNames(), + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, + context.context->getDiscardSuperseded(), + authFactory->getAuthority() != "any" && + authorities.size() > 1 + ? authorities + : std::vector<std::string>(), + context.extent1, context.extent2); } else { io::AuthorityFactory::ObjectType intermediateObjectType = io::AuthorityFactory::ObjectType::CRS; @@ -11781,9 +11816,15 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate( res = tmpAuthFactory->createFromCRSCodesWithIntermediates( srcAuthName, srcCode, targetAuthName, targetCode, context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE, + gridAvailabilityUse == CoordinateOperationContext::GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, + KNOWN_AVAILABLE, context.context->getDiscardSuperseded(), context.context->getIntermediateCRS(), intermediateObjectType, @@ -14211,15 +14252,21 @@ void CoordinateOperationFactory::Private::createOperationsCompoundToGeog( componentsSrc[1], targetCRS->promoteTo3D(std::string(), dbContext), context); bool foundRegisteredTransformWithAllGridsAvailable = false; + const auto gridAvailabilityUse = + context.context->getGridAvailabilityUse(); const bool ignoreMissingGrids = - context.context->getGridAvailabilityUse() == + gridAvailabilityUse == CoordinateOperationContext::GridAvailabilityUse:: IGNORE_GRID_AVAILABILITY; for (const auto &op : verticalTransforms) { if (hasIdentifiers(op) && dbContext) { bool missingGrid = false; if (!ignoreMissingGrids) { - const auto gridsNeeded = op->gridsNeeded(dbContext); + const auto gridsNeeded = op->gridsNeeded( + dbContext, + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { if (!gridDesc.available) { missingGrid = true; @@ -14248,7 +14295,11 @@ void CoordinateOperationFactory::Private::createOperationsCompoundToGeog( if (hasIdentifiers(op) && dbContext) { bool missingGrid = false; if (!ignoreMissingGrids) { - const auto gridsNeeded = op->gridsNeeded(dbContext); + const auto gridsNeeded = op->gridsNeeded( + dbContext, + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { if (!gridDesc.available) { missingGrid = true; @@ -15079,8 +15130,9 @@ CoordinateOperationNNPtr PROJBasedOperation::_shallowClone() const { // --------------------------------------------------------------------------- -std::set<GridDescription> PROJBasedOperation::gridsNeeded( - const io::DatabaseContextPtr &databaseContext) const { +std::set<GridDescription> +PROJBasedOperation::gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { std::set<GridDescription> res; try { @@ -15093,7 +15145,8 @@ std::set<GridDescription> PROJBasedOperation::gridsNeeded( desc.shortName = shortName; if (databaseContext) { databaseContext->lookForGridInfo( - desc.shortName, desc.fullName, desc.packageName, desc.url, + desc.shortName, considerKnownGridsAsAvailable, + desc.fullName, desc.packageName, desc.url, desc.directDownload, desc.openLicense, desc.available); } res.insert(desc); diff --git a/src/iso19111/factory.cpp b/src/iso19111/factory.cpp index 0c7692ad..7d15172e 100644 --- a/src/iso19111/factory.cpp +++ b/src/iso19111/factory.cpp @@ -45,6 +45,8 @@ #include "proj/internal/lru_cache.hpp" #include "proj/internal/tracing.hpp" +#include "sqlite3_utils.hpp" + #include <cmath> #include <cstdlib> #include <cstring> @@ -278,9 +280,7 @@ struct DatabaseContext::Private { void registerFunctions(); #ifdef ENABLE_CUSTOM_LOCKLESS_VFS - std::string thisNamePtr_{}; - sqlite3_vfs *vfs_{}; - bool createCustomVFS(); + std::unique_ptr<SQLite3VFS> vfs_{}; #endif Private(const Private &) = delete; @@ -297,13 +297,6 @@ DatabaseContext::Private::~Private() { assert(recLevel_ == 0); closeDB(); - -#ifdef ENABLE_CUSTOM_LOCKLESS_VFS - if (vfs_) { - sqlite3_vfs_unregister(vfs_); - delete vfs_; - } -#endif } // --------------------------------------------------------------------------- @@ -499,104 +492,12 @@ void DatabaseContext::Private::cache(const std::string &code, // --------------------------------------------------------------------------- -#ifdef ENABLE_CUSTOM_LOCKLESS_VFS - -typedef int (*ClosePtr)(sqlite3_file *); - -static int VFSClose(sqlite3_file *file) { - sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr); - assert(defaultVFS); - ClosePtr defaultClosePtr; - std::memcpy(&defaultClosePtr, - reinterpret_cast<char *>(file) + defaultVFS->szOsFile, - sizeof(ClosePtr)); - void *methods = const_cast<sqlite3_io_methods *>(file->pMethods); - int ret = defaultClosePtr(file); - std::free(methods); - return ret; -} - -// No-lock implementation -static int VSFLock(sqlite3_file *, int) { return SQLITE_OK; } - -static int VSFUnlock(sqlite3_file *, int) { return SQLITE_OK; } - -static int VFSOpen(sqlite3_vfs *vfs, const char *name, sqlite3_file *file, - int flags, int *outFlags) { - sqlite3_vfs *defaultVFS = static_cast<sqlite3_vfs *>(vfs->pAppData); - int ret = defaultVFS->xOpen(defaultVFS, name, file, flags, outFlags); - if (ret == SQLITE_OK) { - ClosePtr defaultClosePtr = file->pMethods->xClose; - assert(defaultClosePtr); - sqlite3_io_methods *methods = static_cast<sqlite3_io_methods *>( - std::malloc(sizeof(sqlite3_io_methods))); - if (!methods) { - file->pMethods->xClose(file); - return SQLITE_NOMEM; - } - memcpy(methods, file->pMethods, sizeof(sqlite3_io_methods)); - methods->xClose = VFSClose; - methods->xLock = VSFLock; - methods->xUnlock = VSFUnlock; - file->pMethods = methods; - // Save original xClose pointer at end of file structure - std::memcpy(reinterpret_cast<char *>(file) + defaultVFS->szOsFile, - &defaultClosePtr, sizeof(ClosePtr)); - } - return ret; -} - -static int VFSAccess(sqlite3_vfs *vfs, const char *zName, int flags, - int *pResOut) { - sqlite3_vfs *defaultVFS = static_cast<sqlite3_vfs *>(vfs->pAppData); - // Do not bother stat'ing for journal or wal files - if (std::strstr(zName, "-journal") || std::strstr(zName, "-wal")) { - *pResOut = false; - return SQLITE_OK; - } - return defaultVFS->xAccess(defaultVFS, zName, flags, pResOut); -} - -// --------------------------------------------------------------------------- - -bool DatabaseContext::Private::createCustomVFS() { - - sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr); - assert(defaultVFS); - - std::ostringstream buffer; - buffer << this; - thisNamePtr_ = buffer.str(); - - vfs_ = new sqlite3_vfs(); - vfs_->iVersion = 1; - vfs_->szOsFile = defaultVFS->szOsFile + sizeof(ClosePtr); - vfs_->mxPathname = defaultVFS->mxPathname; - vfs_->zName = thisNamePtr_.c_str(); - vfs_->pAppData = defaultVFS; - vfs_->xOpen = VFSOpen; - vfs_->xDelete = defaultVFS->xDelete; - vfs_->xAccess = VFSAccess; - vfs_->xFullPathname = defaultVFS->xFullPathname; - vfs_->xDlOpen = defaultVFS->xDlOpen; - vfs_->xDlError = defaultVFS->xDlError; - vfs_->xDlSym = defaultVFS->xDlSym; - vfs_->xDlClose = defaultVFS->xDlClose; - vfs_->xRandomness = defaultVFS->xRandomness; - vfs_->xSleep = defaultVFS->xSleep; - vfs_->xCurrentTime = defaultVFS->xCurrentTime; - vfs_->xGetLastError = defaultVFS->xGetLastError; - vfs_->xCurrentTimeInt64 = defaultVFS->xCurrentTimeInt64; - return sqlite3_vfs_register(vfs_, false) == SQLITE_OK; -} - -#endif // ENABLE_CUSTOM_LOCKLESS_VFS - -// --------------------------------------------------------------------------- - void DatabaseContext::Private::open(const std::string &databasePath, PJ_CONTEXT *ctx) { - setPjCtxt(ctx ? ctx : pj_get_default_ctx()); + if (!ctx) { + ctx = pj_get_default_ctx(); + } + setPjCtxt(ctx); std::string path(databasePath); if (path.empty()) { path.resize(2048); @@ -608,18 +509,23 @@ void DatabaseContext::Private::open(const std::string &databasePath, } } - if ( + std::string vfsName; #ifdef ENABLE_CUSTOM_LOCKLESS_VFS - !createCustomVFS() || + if (ctx->custom_sqlite3_vfs_name.empty()) { + vfs_ = SQLite3VFS::create(false, true, true); + if (vfs_ == nullptr) { + throw FactoryException("Open of " + path + " failed"); + } + vfsName = vfs_->name(); + } else #endif - sqlite3_open_v2(path.c_str(), &sqlite_handle_, + { + vfsName = ctx->custom_sqlite3_vfs_name; + } + if (sqlite3_open_v2(path.c_str(), &sqlite_handle_, SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX, -#ifdef ENABLE_CUSTOM_LOCKLESS_VFS - thisNamePtr_.c_str() -#else - nullptr -#endif - ) != SQLITE_OK || + vfsName.empty() ? nullptr : vfsName.c_str()) != + SQLITE_OK || !sqlite_handle_) { throw FactoryException("Open of " + path + " failed"); } @@ -1021,14 +927,14 @@ bool DatabaseContext::lookForGridAlternative(const std::string &officialName, // --------------------------------------------------------------------------- -bool DatabaseContext::lookForGridInfo(const std::string &projFilename, - std::string &fullFilename, - std::string &packageName, - std::string &url, bool &directDownload, - bool &openLicense, - bool &gridAvailable) const { +bool DatabaseContext::lookForGridInfo( + const std::string &projFilename, bool considerKnownGridsAsAvailable, + std::string &fullFilename, std::string &packageName, std::string &url, + bool &directDownload, bool &openLicense, bool &gridAvailable) const { Private::GridInfoCache info; - if (d->getGridInfoFromCache(projFilename, info)) { + const std::string key(projFilename + + (considerKnownGridsAsAvailable ? "true" : "false")); + if (d->getGridInfoFromCache(key, info)) { fullFilename = info.fullFilename; packageName = info.packageName; url = info.url; @@ -1044,16 +950,20 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename, openLicense = false; directDownload = false; - fullFilename.resize(2048); - if (d->pjCtxt() == nullptr) { - d->setPjCtxt(pj_get_default_ctx()); + if (considerKnownGridsAsAvailable) { + fullFilename = projFilename; + } else { + fullFilename.resize(2048); + if (d->pjCtxt() == nullptr) { + d->setPjCtxt(pj_get_default_ctx()); + } + int errno_before = proj_context_errno(d->pjCtxt()); + gridAvailable = + pj_find_file(d->pjCtxt(), projFilename.c_str(), &fullFilename[0], + fullFilename.size() - 1) != 0; + proj_context_errno_set(d->pjCtxt(), errno_before); + fullFilename.resize(strlen(fullFilename.c_str())); } - int errno_before = proj_context_errno(d->pjCtxt()); - gridAvailable = - pj_find_file(d->pjCtxt(), projFilename.c_str(), &fullFilename[0], - fullFilename.size() - 1) != 0; - proj_context_errno_set(d->pjCtxt(), errno_before); - fullFilename.resize(strlen(fullFilename.c_str())); auto res = d->run("SELECT " @@ -1077,6 +987,10 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename, openLicense = (row[3].empty() ? row[4] : row[3]) == "1"; directDownload = (row[5].empty() ? row[6] : row[5]) == "1"; + if (considerKnownGridsAsAvailable && !packageName.empty()) { + gridAvailable = true; + } + info.fullFilename = fullFilename; info.packageName = packageName; info.url = url; @@ -1085,7 +999,7 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename, } info.gridAvailable = gridAvailable; info.found = ret; - d->cache(projFilename, info); + d->cache(key, info); return ret; } @@ -1324,8 +1238,8 @@ struct AuthorityFactory::Private { return AuthorityFactory::create(context_, auth_name); } - bool - rejectOpDueToMissingGrid(const operation::CoordinateOperationNNPtr &op); + bool rejectOpDueToMissingGrid(const operation::CoordinateOperationNNPtr &op, + bool considerKnownGridsAsAvailable); UnitOfMeasure createUnitOfMeasure(const std::string &auth_name, const std::string &code); @@ -1452,8 +1366,10 @@ util::PropertyMap AuthorityFactory::Private::createProperties( // --------------------------------------------------------------------------- bool AuthorityFactory::Private::rejectOpDueToMissingGrid( - const operation::CoordinateOperationNNPtr &op) { - for (const auto &gridDesc : op->gridsNeeded(context())) { + const operation::CoordinateOperationNNPtr &op, + bool considerKnownGridsAsAvailable) { + for (const auto &gridDesc : + op->gridsNeeded(context(), considerKnownGridsAsAvailable)) { if (!gridDesc.available) { return true; } @@ -3449,7 +3365,7 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( const std::string &sourceCRSCode, const std::string &targetCRSCode) const { return createFromCoordinateReferenceSystemCodes( d->authority(), sourceCRSCode, d->authority(), targetCRSCode, false, - false, false); + false, false, false); } // --------------------------------------------------------------------------- @@ -3478,6 +3394,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( * should be substituted to the official grid names. * @param discardIfMissingGrid Whether coordinate operations that reference * missing grids should be removed from the result set. + * @param considerKnownGridsAsAvailable Whether known grids should be considered + * as available (typically when network is enabled). * @param discardSuperseded Whether cordinate operations that are superseded * (but not deprecated) should be removed from the result set. * @param tryReverseOrder whether to search in the reverse order too (and thus @@ -3498,8 +3416,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, bool tryReverseOrder, - bool reportOnlyIntersectingTransformations, + bool considerKnownGridsAsAvailable, bool discardSuperseded, + bool tryReverseOrder, bool reportOnlyIntersectingTransformations, const metadata::ExtentPtr &intersectingExtent1, const metadata::ExtentPtr &intersectingExtent2) const { @@ -3510,6 +3428,7 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( cacheKey += targetCRSCode; cacheKey += (usePROJAlternativeGridNames ? '1' : '0'); cacheKey += (discardIfMissingGrid ? '1' : '0'); + cacheKey += (considerKnownGridsAsAvailable ? '1' : '0'); cacheKey += (discardSuperseded ? '1' : '0'); cacheKey += (tryReverseOrder ? '1' : '0'); cacheKey += (reportOnlyIntersectingTransformations ? '1' : '0'); @@ -3748,7 +3667,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( target_crs_code != targetCRSCode))) { op = op->inverse(); } - if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) { + if (!discardIfMissingGrid || + !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) { list.emplace_back(op); } } @@ -3813,6 +3733,8 @@ static bool useIrrelevantPivot(const operation::CoordinateOperationNNPtr &op, * should be substituted to the official grid names. * @param discardIfMissingGrid Whether coordinate operations that reference * missing grids should be removed from the result set. + * @param considerKnownGridsAsAvailable Whether known grids should be considered + * as available (typically when network is enabled). * @param discardSuperseded Whether cordinate operations that are superseded * (but not deprecated) should be removed from the result set. * @param intermediateCRSAuthCodes List of (auth_name, code) of CRS that can be @@ -3841,7 +3763,7 @@ AuthorityFactory::createFromCRSCodesWithIntermediates( const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, + bool considerKnownGridsAsAvailable, bool discardSuperseded, const std::vector<std::pair<std::string, std::string>> &intermediateCRSAuthCodes, ObjectType allowedIntermediateObjectType, @@ -4289,7 +4211,8 @@ AuthorityFactory::createFromCRSCodesWithIntermediates( std::vector<operation::CoordinateOperationNNPtr> list; for (const auto &op : listTmp) { - if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) { + if (!discardIfMissingGrid || + !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) { list.emplace_back(op); } } @@ -4307,7 +4230,8 @@ AuthorityFactory::createBetweenGeodeticCRSWithDatumBasedIntermediates( const std::string &sourceCRSCode, const crs::CRSNNPtr &targetCRS, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, const std::vector<std::string> &allowedAuthorities, + bool considerKnownGridsAsAvailable, bool discardSuperseded, + const std::vector<std::string> &allowedAuthorities, const metadata::ExtentPtr &intersectingExtent1, const metadata::ExtentPtr &intersectingExtent2) const { @@ -4890,7 +4814,8 @@ AuthorityFactory::createBetweenGeodeticCRSWithDatumBasedIntermediates( std::vector<operation::CoordinateOperationNNPtr> list; for (const auto &op : listTmp) { - if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) { + if (!discardIfMissingGrid || + !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) { list.emplace_back(op); } } diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index 2e470683..704ece3d 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -198,6 +198,7 @@ set(SRC_LIBPROJ_TRANSFORMATIONS transformations/horner.cpp transformations/molodensky.cpp transformations/vgridshift.cpp + transformations/xyzgridshift.cpp ) set(SRC_LIBPROJ_ISO19111 @@ -219,8 +220,6 @@ set(SRC_LIBPROJ_CORE 4D_api.cpp aasincos.cpp adjlon.cpp - apply_gridshift.cpp - apply_vgridshift.cpp auth.cpp ctx.cpp datum_set.cpp @@ -234,13 +233,9 @@ set(SRC_LIBPROJ_CORE fileapi.cpp fwd.cpp gauss.cpp - gc_reader.cpp geocent.cpp geocent.h geodesic.c - gridcatalog.cpp - gridinfo.cpp - gridlist.cpp init.cpp initcache.cpp internal.cpp @@ -251,10 +246,6 @@ set(SRC_LIBPROJ_CORE mlfn.cpp msfn.cpp mutex.cpp - nad_cvt.cpp - nad_init.cpp - nad_intr.cpp - open_lib.cpp param.cpp phi2.cpp pipeline.cpp @@ -285,6 +276,13 @@ set(SRC_LIBPROJ_CORE proj_json_streaming_writer.hpp proj_json_streaming_writer.cpp tracing.cpp + grids.hpp + grids.cpp + filemanager.hpp + filemanager.cpp + networkfilemanager.cpp + sqlite3_utils.hpp + sqlite3_utils.cpp ${CMAKE_CURRENT_BINARY_DIR}/proj_config.h ) @@ -348,6 +346,10 @@ target_compile_options(${PROJ_CORE_TARGET} PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${PROJ_CXX_WARN_FLAGS}> ) +if(MSVC OR MINGW) + target_compile_definitions(${PROJ_CORE_TARGET} PRIVATE -DNOMINMAX) +endif() + # Tell Intel compiler to do arithmetic accurately. This is needed to stop the # compiler from ignoring parentheses in expressions like (a + b) + c and from # simplifying 0.0 + x to x (which is wrong if x = -0.0). @@ -419,9 +421,19 @@ if(USE_THREAD AND Threads_FOUND AND CMAKE_USE_PTHREADS_INIT) target_link_libraries(${PROJ_CORE_TARGET} ${CMAKE_THREAD_LIBS_INIT}) endif() -include_directories(${SQLITE3_INCLUDE_DIR}) +target_include_directories(${PROJ_CORE_TARGET} PRIVATE ${SQLITE3_INCLUDE_DIR}) target_link_libraries(${PROJ_CORE_TARGET} ${SQLITE3_LIBRARY}) +if(NOT DISABLE_TIFF) + target_include_directories(${PROJ_CORE_TARGET} PRIVATE ${TIFF_INCLUDE_DIR}) + target_link_libraries(${PROJ_CORE_TARGET} ${TIFF_LIBRARY}) +endif() + +if(CURL_FOUND) + target_include_directories(${PROJ_CORE_TARGET} PRIVATE ${CURL_INCLUDE_DIR}) + target_link_libraries(${PROJ_CORE_TARGET} ${CURL_LIBRARY}) +endif() + if(MSVC AND BUILD_LIBPROJ_SHARED) target_compile_definitions(${PROJ_CORE_TARGET} PRIVATE PROJ_MSVC_DLL_EXPORT=1) diff --git a/src/malloc.cpp b/src/malloc.cpp index 393437e3..9ac28546 100644 --- a/src/malloc.cpp +++ b/src/malloc.cpp @@ -48,6 +48,10 @@ #include "proj.h" #include "proj_internal.h" +#include "grids.hpp" +#include "filemanager.hpp" + +using namespace NS_PROJ; /**********************************************************************/ void *pj_malloc(size_t size) { @@ -225,10 +229,8 @@ PJ *pj_default_destructor (PJ *P, int errlev) { /* Destructor */ pj_dealloc(P->def_spherification); pj_dealloc(P->def_ellps); - /* free grid lists */ - pj_dealloc( P->gridlist ); - pj_dealloc( P->vgridlist_geoid ); - pj_dealloc( P->catalog_name ); + delete static_cast<ListOfHGrids*>(P->hgrids_legacy); + delete static_cast<ListOfVGrids*>(P->vgrids_legacy); /* We used to call pj_dalloc( P->catalog ), but this will leak */ /* memory. The safe way to clear catalog and grid is to call */ @@ -261,4 +263,5 @@ void proj_cleanup() { /*****************************************************************************/ pj_clear_initcache(); pj_deallocate_grids(); + FileManager::clearMemoryCache(); } diff --git a/src/nad_cvt.cpp b/src/nad_cvt.cpp deleted file mode 100644 index e8b8e9b7..00000000 --- a/src/nad_cvt.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#define PJ_LIB__ - -#include <stdio.h> -#include <stdlib.h> - -#include "proj.h" -#include "proj_internal.h" -#include <math.h> - -#include <limits> - -#define MAX_ITERATIONS 10 -#define TOL 1e-12 - -PJ_LP nad_cvt(projCtx ctx, PJ_LP in, int inverse, struct CTABLE *ct, int grid_count, PJ_GRIDINFO **tables) { - PJ_LP t, tb,del, dif; - int i = MAX_ITERATIONS; - const double toltol = TOL*TOL; - - if (in.lam == HUGE_VAL) - return in; - - /* normalize input to ll origin */ - tb = in; - tb.lam -= ct->ll.lam; - tb.phi -= ct->ll.phi; - tb.lam = adjlon (tb.lam - M_PI) + M_PI; - - t = nad_intr (tb, ct); - if (t.lam == HUGE_VAL) - return t; - - if (!inverse) { - in.lam -= t.lam; - in.phi += t.phi; - return in; - } - - t.lam = tb.lam + t.lam; - t.phi = tb.phi - t.phi; - - do { - del = nad_intr(t, ct); - - /* In case we are (or have switched) on the null grid, exit now */ - if( del.lam == 0 && del.phi == 0 ) - break; - - /* We can possibly go outside of the initial guessed grid, so try */ - /* to fetch a new grid into which iterate... */ - if (del.lam == HUGE_VAL) - { - if( grid_count == 0 ) - break; - PJ_LP lp; - lp.lam = t.lam + ct->ll.lam; - lp.phi = t.phi + ct->ll.phi; - auto newCt = find_ctable(ctx, lp, grid_count, tables); - if( newCt == nullptr || newCt == ct ) - break; - pj_log(ctx, PJ_LOG_DEBUG_MINOR, "Switching from grid %s to grid %s", - ct->id, - newCt->id); - ct = newCt; - t.lam = lp.lam - ct->ll.lam; - t.phi = lp.phi - ct->ll.phi; - tb = in; - tb.lam -= ct->ll.lam; - tb.phi -= ct->ll.phi; - tb.lam = adjlon (tb.lam - M_PI) + M_PI; - dif.lam = std::numeric_limits<double>::max(); - dif.phi = std::numeric_limits<double>::max(); - continue; - } - - dif.lam = t.lam - del.lam - tb.lam; - dif.phi = t.phi + del.phi - tb.phi; - t.lam -= dif.lam; - t.phi -= dif.phi; - - } while (--i && (dif.lam*dif.lam + dif.phi*dif.phi > toltol)); /* prob. slightly faster than hypot() */ - - if (i==0) { - /* If we had access to a context, this should go through pj_log, and we should set ctx->errno */ - if (getenv ("PROJ_DEBUG")) - fprintf( stderr, "Inverse grid shift iterator failed to converge.\n" ); - t.lam = t.phi = HUGE_VAL; - return t; - } - - /* and again: pj_log and ctx->errno */ - if (del.lam==HUGE_VAL && getenv ("PROJ_DEBUG")) - fprintf (stderr, "Inverse grid shift iteration failed, presumably at grid edge.\nUsing first approximation.\n"); - - in.lam = adjlon (t.lam + ct->ll.lam); - in.phi = t.phi + ct->ll.phi; - return in; -} diff --git a/src/nad_init.cpp b/src/nad_init.cpp deleted file mode 100644 index 315318be..00000000 --- a/src/nad_init.cpp +++ /dev/null @@ -1,308 +0,0 @@ -/****************************************************************************** - * Project: PROJ.4 - * Purpose: Load datum shift files into memory. - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ****************************************************************************** - * Copyright (c) 2000, Frank Warmerdam - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - *****************************************************************************/ - -#define PJ_LIB__ - -#include <errno.h> -#include <stddef.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include "proj.h" -#include "proj_internal.h" - -/************************************************************************/ -/* swap_words() */ -/* */ -/* Convert the byte order of the given word(s) in place. */ -/************************************************************************/ - -static const int byte_order_test = 1; -#define IS_LSB (((const unsigned char *) (&byte_order_test))[0] == 1) - -static void swap_words( void *data_in, int word_size, int word_count ) - -{ - int word; - unsigned char *data = (unsigned char *) data_in; - - for( word = 0; word < word_count; word++ ) - { - int i; - - for( i = 0; i < word_size/2; i++ ) - { - unsigned char t; - - t = data[i]; - data[i] = data[word_size-i-1]; - data[word_size-i-1] = t; - } - - data += word_size; - } -} - -/************************************************************************/ -/* nad_ctable_load() */ -/* */ -/* Load the data portion of a ctable formatted grid. */ -/************************************************************************/ - -int nad_ctable_load( projCtx ctx, struct CTABLE *ct, struct projFileAPI_t* fileapi ) - -{ - PAFile fid = (PAFile)fileapi; - size_t a_size; - - pj_ctx_fseek( ctx, fid, sizeof(struct CTABLE), SEEK_SET ); - - /* read all the actual shift values */ - a_size = ct->lim.lam * ct->lim.phi; - ct->cvs = (FLP *) pj_malloc(sizeof(FLP) * a_size); - if( ct->cvs == nullptr - || pj_ctx_fread(ctx, ct->cvs, sizeof(FLP), a_size, fid) != a_size ) - { - pj_dalloc( ct->cvs ); - ct->cvs = nullptr; - - pj_log( ctx, PJ_LOG_ERROR, - "ctable loading failed on fread() - binary incompatible?" ); - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return 0; - } - - return 1; -} - -/************************************************************************/ -/* nad_ctable_init() */ -/* */ -/* Read the header portion of a "ctable" format grid. */ -/************************************************************************/ - -struct CTABLE *nad_ctable_init( projCtx ctx, struct projFileAPI_t* fileapi ) -{ - PAFile fid = (PAFile)fileapi; - struct CTABLE *ct; - int id_end; - - /* read the table header */ - ct = (struct CTABLE *) pj_malloc(sizeof(struct CTABLE)); - if( ct == nullptr - || pj_ctx_fread( ctx, ct, sizeof(struct CTABLE), 1, fid ) != 1 ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_dalloc( ct ); - return nullptr; - } - - /* do some minimal validation to ensure the structure isn't corrupt */ - if( ct->lim.lam < 1 || ct->lim.lam > 100000 - || ct->lim.phi < 1 || ct->lim.phi > 100000 ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_dalloc( ct ); - return nullptr; - } - - /* trim white space and newlines off id */ - for( id_end = (int)strlen(ct->id)-1; id_end > 0; id_end-- ) - { - if( ct->id[id_end] == '\n' || ct->id[id_end] == ' ' ) - ct->id[id_end] = '\0'; - else - break; - } - - ct->cvs = nullptr; - - return ct; -} - -/************************************************************************/ -/* nad_ctable2_load() */ -/* */ -/* Load the data portion of a ctable2 formatted grid. */ -/************************************************************************/ - -int nad_ctable2_load( projCtx ctx, struct CTABLE *ct, struct projFileAPI_t* fileapi ) - -{ - PAFile fid = (PAFile)fileapi; - size_t a_size; - - pj_ctx_fseek( ctx, fid, 160, SEEK_SET ); - - /* read all the actual shift values */ - a_size = ct->lim.lam * ct->lim.phi; - ct->cvs = (FLP *) pj_malloc(sizeof(FLP) * a_size); - if( ct->cvs == nullptr - || pj_ctx_fread(ctx, ct->cvs, sizeof(FLP), a_size, fid) != a_size ) - { - pj_dalloc( ct->cvs ); - ct->cvs = nullptr; - - if( getenv("PROJ_DEBUG") != nullptr ) - { - fprintf( stderr, - "ctable2 loading failed on fread() - binary incompatible?\n" ); - } - - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return 0; - } - - if( !IS_LSB ) - { - swap_words( ct->cvs, 4, (int)a_size * 2 ); - } - - return 1; -} - -/************************************************************************/ -/* nad_ctable2_init() */ -/* */ -/* Read the header portion of a "ctable2" format grid. */ -/************************************************************************/ - -struct CTABLE *nad_ctable2_init( projCtx ctx, struct projFileAPI_t* fileapi ) -{ - PAFile fid = (PAFile)fileapi; - struct CTABLE *ct; - int id_end; - char header[160]; - - if( pj_ctx_fread( ctx, header, sizeof(header), 1, fid ) != 1 ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return nullptr; - } - - if( !IS_LSB ) - { - swap_words( header + 96, 8, 4 ); - swap_words( header + 128, 4, 2 ); - } - - if( strncmp(header,"CTABLE V2",9) != 0 ) - { - pj_log( ctx, PJ_LOG_ERROR, "ctable2 - wrong header!" ); - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return nullptr; - } - - /* read the table header */ - ct = (struct CTABLE *) pj_malloc(sizeof(struct CTABLE)); - if( ct == nullptr ) - { - pj_ctx_set_errno( ctx, ENOMEM ); - return nullptr; - } - - memcpy( ct->id, header + 16, 80 ); - memcpy( &ct->ll.lam, header + 96, 8 ); - memcpy( &ct->ll.phi, header + 104, 8 ); - memcpy( &ct->del.lam, header + 112, 8 ); - memcpy( &ct->del.phi, header + 120, 8 ); - memcpy( &ct->lim.lam, header + 128, 4 ); - memcpy( &ct->lim.phi, header + 132, 4 ); - - /* do some minimal validation to ensure the structure isn't corrupt */ - if( ct->lim.lam < 1 || ct->lim.lam > 100000 - || ct->lim.phi < 1 || ct->lim.phi > 100000 ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - pj_dalloc( ct ); - return nullptr; - } - - /* trim white space and newlines off id */ - for( id_end = (int)strlen(ct->id)-1; id_end > 0; id_end-- ) - { - if( ct->id[id_end] == '\n' || ct->id[id_end] == ' ' ) - ct->id[id_end] = '\0'; - else - break; - } - - ct->cvs = nullptr; - - return ct; -} - -/************************************************************************/ -/* nad_init() */ -/* */ -/* Read a datum shift file in any of the supported binary formats. */ -/************************************************************************/ - -struct CTABLE *nad_init(projCtx ctx, char *name) -{ - struct CTABLE *ct; - PAFile fid; - - ctx->last_errno = 0; - -/* -------------------------------------------------------------------- */ -/* Open the file using the usual search rules. */ -/* -------------------------------------------------------------------- */ - if (!(fid = pj_open_lib(ctx, name, "rb"))) { - return nullptr; - } - - ct = nad_ctable_init( ctx, (struct projFileAPI_t*)fid ); - if( ct != nullptr ) - { - if( !nad_ctable_load( ctx, ct, (struct projFileAPI_t*)fid ) ) - { - nad_free( ct ); - ct = nullptr; - } - } - - pj_ctx_fclose(ctx, fid); - return ct; -} - -/************************************************************************/ -/* nad_free() */ -/* */ -/* Free a CTABLE grid shift structure produced by nad_init(). */ -/************************************************************************/ - -void nad_free(struct CTABLE *ct) -{ - if (ct) { - if( ct->cvs != nullptr ) - pj_dalloc(ct->cvs); - - pj_dalloc(ct); - } -} diff --git a/src/nad_intr.cpp b/src/nad_intr.cpp deleted file mode 100644 index 36ab0a99..00000000 --- a/src/nad_intr.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/* Determine nad table correction value */ -#define PJ_LIB__ -#include "proj.h" -#include "proj_internal.h" -#include <math.h> - -PJ_LP nad_intr(PJ_LP t, struct CTABLE *ct) { - PJ_LP val, frct; - ILP indx; - double m00, m10, m01, m11; - FLP *f00, *f10, *f01, *f11; - long index; - int in; - - t.lam /= ct->del.lam; - indx.lam = isnan(t.lam) ? 0 : (pj_int32)lround(floor(t.lam)); - t.phi /= ct->del.phi; - indx.phi = isnan(t.phi) ? 0 : (pj_int32)lround(floor(t.phi)); - - frct.lam = t.lam - indx.lam; - frct.phi = t.phi - indx.phi; - val.lam = val.phi = HUGE_VAL; - if (indx.lam < 0) { - if (indx.lam == -1 && frct.lam > 0.99999999999) { - ++indx.lam; - frct.lam = 0.; - } else - return val; - } else if ((in = indx.lam + 1) >= ct->lim.lam) { - if (in == ct->lim.lam && frct.lam < 1e-11) { - --indx.lam; - frct.lam = 1.; - } else - return val; - } - if (indx.phi < 0) { - if (indx.phi == -1 && frct.phi > 0.99999999999) { - ++indx.phi; - frct.phi = 0.; - } else - return val; - } else if ((in = indx.phi + 1) >= ct->lim.phi) { - if (in == ct->lim.phi && frct.phi < 1e-11) { - --indx.phi; - frct.phi = 1.; - } else - return val; - } - index = indx.phi * ct->lim.lam + indx.lam; - f00 = ct->cvs + index++; - f10 = ct->cvs + index; - index += ct->lim.lam; - f11 = ct->cvs + index--; - f01 = ct->cvs + index; - m11 = m10 = frct.lam; - m00 = m01 = 1. - frct.lam; - m11 *= frct.phi; - m01 *= frct.phi; - frct.phi = 1. - frct.phi; - m00 *= frct.phi; - m10 *= frct.phi; - val.lam = m00 * f00->lam + m10 * f10->lam + - m01 * f01->lam + m11 * f11->lam; - val.phi = m00 * f00->phi + m10 * f10->phi + - m01 * f01->phi + m11 * f11->phi; - return val; -} diff --git a/src/networkfilemanager.cpp b/src/networkfilemanager.cpp new file mode 100644 index 00000000..64969d0f --- /dev/null +++ b/src/networkfilemanager.cpp @@ -0,0 +1,2544 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: Functionality related to network access and caching + * Author: Even Rouault, <even.rouault at spatialys.com> + * + ****************************************************************************** + * Copyright (c) 2019-2020, Even Rouault, <even.rouault at spatialys.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef FROM_PROJ_CPP +#define FROM_PROJ_CPP +#endif +#define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS + +#include <stdlib.h> + +#include <algorithm> +#include <limits> +#include <string> + +#include "filemanager.hpp" +#include "proj.h" +#include "proj/internal/internal.hpp" +#include "proj/internal/lru_cache.hpp" +#include "proj_internal.h" +#include "sqlite3_utils.hpp" + +#ifdef __MINGW32__ +// mingw32-win32 doesn't implement std::mutex +namespace { +class MyMutex { + public: + // cppcheck-suppress functionStatic + void lock() { pj_acquire_lock(); } + // cppcheck-suppress functionStatic + void unlock() { pj_release_lock(); } +}; +} +#else +#include <mutex> +#define MyMutex std::mutex +#endif + +#ifdef CURL_ENABLED +#include <curl/curl.h> +#include <sqlite3.h> // for sqlite3_snprintf +#endif + +#include <sys/stat.h> + +#ifdef _WIN32 +#include <shlobj.h> +#else +#include <sys/types.h> +#include <unistd.h> +#endif + +#if defined(_WIN32) +#include <windows.h> +#elif defined(__MACH__) && defined(__APPLE__) +#include <mach-o/dyld.h> +#elif defined(__FreeBSD__) +#include <sys/sysctl.h> +#include <sys/types.h> +#endif + +#include <time.h> + +//! @cond Doxygen_Suppress + +#define STR_HELPER(x) #x +#define STR(x) STR_HELPER(x) + +using namespace NS_PROJ::internal; + +NS_PROJ_START + +// --------------------------------------------------------------------------- + +static void sleep_ms(int ms) { +#ifdef _WIN32 + Sleep(ms); +#else + usleep(ms * 1000); +#endif +} + +// --------------------------------------------------------------------------- + +constexpr size_t DOWNLOAD_CHUNK_SIZE = 16 * 1024; +constexpr int MAX_CHUNKS = 64; + +struct FileProperties { + unsigned long long size = 0; + time_t lastChecked = 0; + std::string lastModified{}; + std::string etag{}; +}; + +class NetworkChunkCache { + public: + void insert(PJ_CONTEXT *ctx, const std::string &url, + unsigned long long chunkIdx, std::vector<unsigned char> &&data); + + std::shared_ptr<std::vector<unsigned char>> + get(PJ_CONTEXT *ctx, const std::string &url, unsigned long long chunkIdx); + + std::shared_ptr<std::vector<unsigned char>> get(PJ_CONTEXT *ctx, + const std::string &url, + unsigned long long chunkIdx, + FileProperties &props); + + void clearMemoryCache(); + + static void clearDiskChunkCache(PJ_CONTEXT *ctx); + + private: + struct Key { + std::string url; + unsigned long long chunkIdx; + + Key(const std::string &urlIn, unsigned long long chunkIdxIn) + : url(urlIn), chunkIdx(chunkIdxIn) {} + bool operator==(const Key &other) const { + return url == other.url && chunkIdx == other.chunkIdx; + } + }; + + struct KeyHasher { + std::size_t operator()(const Key &k) const { + return std::hash<std::string>{}(k.url) ^ + (std::hash<unsigned long long>{}(k.chunkIdx) << 1); + } + }; + + lru11::Cache< + Key, std::shared_ptr<std::vector<unsigned char>>, MyMutex, + std::unordered_map< + Key, + typename std::list<lru11::KeyValuePair< + Key, std::shared_ptr<std::vector<unsigned char>>>>::iterator, + KeyHasher>> + cache_{MAX_CHUNKS}; +}; + +// --------------------------------------------------------------------------- + +static NetworkChunkCache gNetworkChunkCache{}; + +// --------------------------------------------------------------------------- + +class NetworkFilePropertiesCache { + public: + void insert(PJ_CONTEXT *ctx, const std::string &url, FileProperties &props); + + bool tryGet(PJ_CONTEXT *ctx, const std::string &url, FileProperties &props); + + void clearMemoryCache(); + + private: + lru11::Cache<std::string, FileProperties, MyMutex> cache_{}; +}; + +// --------------------------------------------------------------------------- + +static NetworkFilePropertiesCache gNetworkFileProperties{}; + +// --------------------------------------------------------------------------- + +class DiskChunkCache { + PJ_CONTEXT *ctx_ = nullptr; + std::string path_{}; + sqlite3 *hDB_ = nullptr; + std::string thisNamePtr_{}; + std::unique_ptr<SQLite3VFS> vfs_{}; + + explicit DiskChunkCache(PJ_CONTEXT *ctx, const std::string &path); + + bool initialize(); + void commitAndClose(); + + bool createDBStructure(); + bool checkConsistency(); + bool get_links(sqlite3_int64 chunk_id, sqlite3_int64 &link_id, + sqlite3_int64 &prev, sqlite3_int64 &next, + sqlite3_int64 &head, sqlite3_int64 &tail); + bool update_links_of_prev_and_next_links(sqlite3_int64 prev, + sqlite3_int64 next); + bool update_linked_chunks(sqlite3_int64 link_id, sqlite3_int64 prev, + sqlite3_int64 next); + bool update_linked_chunks_head_tail(sqlite3_int64 head, sqlite3_int64 tail); + + DiskChunkCache(const DiskChunkCache &) = delete; + DiskChunkCache &operator=(const DiskChunkCache &) = delete; + + public: + static std::unique_ptr<DiskChunkCache> open(PJ_CONTEXT *ctx); + ~DiskChunkCache(); + + sqlite3 *handle() { return hDB_; } + std::unique_ptr<SQLiteStatement> prepare(const char *sql); + bool move_to_head(sqlite3_int64 chunk_id); + bool move_to_tail(sqlite3_int64 chunk_id); + void closeAndUnlink(); +}; + +// --------------------------------------------------------------------------- + +static bool pj_context_get_grid_cache_is_enabled(PJ_CONTEXT *ctx) { + pj_load_ini(ctx); + return ctx->gridChunkCache.enabled; +} + +// --------------------------------------------------------------------------- + +static long long pj_context_get_grid_cache_max_size(PJ_CONTEXT *ctx) { + pj_load_ini(ctx); + return ctx->gridChunkCache.max_size; +} + +// --------------------------------------------------------------------------- + +static int pj_context_get_grid_cache_ttl(PJ_CONTEXT *ctx) { + pj_load_ini(ctx); + return ctx->gridChunkCache.ttl; +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<DiskChunkCache> DiskChunkCache::open(PJ_CONTEXT *ctx) { + if (!pj_context_get_grid_cache_is_enabled(ctx)) { + return nullptr; + } + const auto cachePath = pj_context_get_grid_cache_filename(ctx); + if (cachePath.empty()) { + return nullptr; + } + + auto diskCache = + std::unique_ptr<DiskChunkCache>(new DiskChunkCache(ctx, cachePath)); + if (!diskCache->initialize()) + diskCache.reset(); + return diskCache; +} + +// --------------------------------------------------------------------------- + +DiskChunkCache::DiskChunkCache(PJ_CONTEXT *ctx, const std::string &path) + : ctx_(ctx), path_(path) {} + +// --------------------------------------------------------------------------- + +bool DiskChunkCache::initialize() { + std::string vfsName; + if (ctx_->custom_sqlite3_vfs_name.empty()) { + vfs_ = SQLite3VFS::create(true, false, false); + if (vfs_ == nullptr) { + return false; + } + vfsName = vfs_->name(); + } else { + vfsName = ctx_->custom_sqlite3_vfs_name; + } + sqlite3_open_v2(path_.c_str(), &hDB_, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + vfsName.c_str()); + if (!hDB_) { + pj_log(ctx_, PJ_LOG_ERROR, "Cannot open %s", path_.c_str()); + return false; + } + + // Cannot run more than 30 times / a bit more than one second. + for (int i = 0;; i++) { + int ret = + sqlite3_exec(hDB_, "BEGIN EXCLUSIVE", nullptr, nullptr, nullptr); + if (ret == SQLITE_OK) { + break; + } + if (ret != SQLITE_BUSY) { + pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_)); + sqlite3_close(hDB_); + hDB_ = nullptr; + return false; + } + const char *max_iters = getenv("PROJ_LOCK_MAX_ITERS"); + if (i >= (max_iters && max_iters[0] ? atoi(max_iters) + : 30)) { // A bit more than 1 second + pj_log(ctx_, PJ_LOG_ERROR, "Cannot take exclusive lock on %s", + path_.c_str()); + sqlite3_close(hDB_); + hDB_ = nullptr; + return false; + } + pj_log(ctx_, PJ_LOG_TRACE, "Lock taken on cache. Waiting a bit..."); + // Retry every 5 ms for 50 ms, then every 10 ms for 100 ms, then + // every 100 ms + sleep_ms(i < 10 ? 5 : i < 20 ? 10 : 100); + } + char **pasResult = nullptr; + int nRows = 0; + int nCols = 0; + sqlite3_get_table(hDB_, + "SELECT 1 FROM sqlite_master WHERE name = 'properties'", + &pasResult, &nRows, &nCols, nullptr); + sqlite3_free_table(pasResult); + if (nRows == 0) { + if (!createDBStructure()) { + sqlite3_close(hDB_); + hDB_ = nullptr; + return false; + } + } + + if (getenv("PROJ_CHECK_CACHE_CONSISTENCY")) { + checkConsistency(); + } + return true; +} + +// --------------------------------------------------------------------------- + +static const char *cache_db_structure_sql = + "CREATE TABLE properties(" + " url TEXT PRIMARY KEY NOT NULL," + " lastChecked TIMESTAMP NOT NULL," + " fileSize INTEGER NOT NULL," + " lastModified TEXT," + " etag TEXT" + ");" + "CREATE TABLE downloaded_file_properties(" + " url TEXT PRIMARY KEY NOT NULL," + " lastChecked TIMESTAMP NOT NULL," + " fileSize INTEGER NOT NULL," + " lastModified TEXT," + " etag TEXT" + ");" + "CREATE TABLE chunk_data(" + " id INTEGER PRIMARY KEY AUTOINCREMENT CHECK (id > 0)," + " data BLOB NOT NULL" + ");" + "CREATE TABLE chunks(" + " id INTEGER PRIMARY KEY AUTOINCREMENT CHECK (id > 0)," + " url TEXT NOT NULL," + " offset INTEGER NOT NULL," + " data_id INTEGER NOT NULL," + " data_size INTEGER NOT NULL," + " CONSTRAINT fk_chunks_url FOREIGN KEY (url) REFERENCES properties(url)," + " CONSTRAINT fk_chunks_data FOREIGN KEY (data_id) REFERENCES chunk_data(id)" + ");" + "CREATE INDEX idx_chunks ON chunks(url, offset);" + "CREATE TABLE linked_chunks(" + " id INTEGER PRIMARY KEY AUTOINCREMENT CHECK (id > 0)," + " chunk_id INTEGER NOT NULL," + " prev INTEGER," + " next INTEGER," + " CONSTRAINT fk_links_chunkid FOREIGN KEY (chunk_id) REFERENCES chunks(id)," + " CONSTRAINT fk_links_prev FOREIGN KEY (prev) REFERENCES linked_chunks(id)," + " CONSTRAINT fk_links_next FOREIGN KEY (next) REFERENCES linked_chunks(id)" + ");" + "CREATE INDEX idx_linked_chunks_chunk_id ON linked_chunks(chunk_id);" + "CREATE TABLE linked_chunks_head_tail(" + " head INTEGER," + " tail INTEGER," + " CONSTRAINT lht_head FOREIGN KEY (head) REFERENCES linked_chunks(id)," + " CONSTRAINT lht_tail FOREIGN KEY (tail) REFERENCES linked_chunks(id)" + ");" + "INSERT INTO linked_chunks_head_tail VALUES (NULL, NULL);"; + +bool DiskChunkCache::createDBStructure() { + + pj_log(ctx_, PJ_LOG_TRACE, "Creating cache DB structure"); + if (sqlite3_exec(hDB_, cache_db_structure_sql, nullptr, nullptr, nullptr) != + SQLITE_OK) { + pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_)); + return false; + } + return true; +} + +// --------------------------------------------------------------------------- + +// Used by checkConsistency() and insert() +#define INVALIDATED_SQL_LITERAL "'invalidated'" + +bool DiskChunkCache::checkConsistency() { + + auto stmt = prepare("SELECT * FROM chunk_data WHERE id NOT IN (SELECT " + "data_id FROM chunks)"); + if (!stmt) { + return false; + } + if (stmt->execute() != SQLITE_DONE) { + fprintf(stderr, "Rows in chunk_data not referenced by chunks.\n"); + return false; + } + + stmt = prepare("SELECT * FROM chunks WHERE id NOT IN (SELECT chunk_id FROM " + "linked_chunks)"); + if (!stmt) { + return false; + } + if (stmt->execute() != SQLITE_DONE) { + fprintf(stderr, "Rows in chunks not referenced by linked_chunks.\n"); + return false; + } + + stmt = prepare("SELECT * FROM chunks WHERE url <> " INVALIDATED_SQL_LITERAL + " AND url " + "NOT IN (SELECT url FROM properties)"); + if (!stmt) { + return false; + } + if (stmt->execute() != SQLITE_DONE) { + fprintf(stderr, "url values in chunks not referenced by properties.\n"); + return false; + } + + stmt = prepare("SELECT head, tail FROM linked_chunks_head_tail"); + if (!stmt) { + return false; + } + if (stmt->execute() != SQLITE_ROW) { + fprintf(stderr, "linked_chunks_head_tail empty.\n"); + return false; + } + const auto head = stmt->getInt64(); + const auto tail = stmt->getInt64(); + if (stmt->execute() != SQLITE_DONE) { + fprintf(stderr, "linked_chunks_head_tail has more than one row.\n"); + return false; + } + + stmt = prepare("SELECT COUNT(*) FROM linked_chunks"); + if (!stmt) { + return false; + } + if (stmt->execute() != SQLITE_ROW) { + fprintf(stderr, "linked_chunks_head_tail empty.\n"); + return false; + } + const auto count_linked_chunks = stmt->getInt64(); + + if (head) { + auto id = head; + std::set<sqlite3_int64> visitedIds; + stmt = prepare("SELECT next FROM linked_chunks WHERE id = ?"); + if (!stmt) { + return false; + } + while (true) { + visitedIds.insert(id); + stmt->reset(); + stmt->bindInt64(id); + if (stmt->execute() != SQLITE_ROW) { + fprintf(stderr, "cannot find linked_chunks.id = %d.\n", + static_cast<int>(id)); + return false; + } + auto next = stmt->getInt64(); + if (next == 0) { + if (id != tail) { + fprintf(stderr, + "last item when following next is not tail.\n"); + return false; + } + break; + } + if (visitedIds.find(next) != visitedIds.end()) { + fprintf(stderr, "found cycle on linked_chunks.next = %d.\n", + static_cast<int>(next)); + return false; + } + id = next; + } + if (visitedIds.size() != static_cast<size_t>(count_linked_chunks)) { + fprintf(stderr, + "ghost items in linked_chunks when following next.\n"); + return false; + } + } else if (count_linked_chunks) { + fprintf(stderr, "linked_chunks_head_tail.head = NULL but linked_chunks " + "not empty.\n"); + return false; + } + + if (tail) { + auto id = tail; + std::set<sqlite3_int64> visitedIds; + stmt = prepare("SELECT prev FROM linked_chunks WHERE id = ?"); + if (!stmt) { + return false; + } + while (true) { + visitedIds.insert(id); + stmt->reset(); + stmt->bindInt64(id); + if (stmt->execute() != SQLITE_ROW) { + fprintf(stderr, "cannot find linked_chunks.id = %d.\n", + static_cast<int>(id)); + return false; + } + auto prev = stmt->getInt64(); + if (prev == 0) { + if (id != head) { + fprintf(stderr, + "last item when following prev is not head.\n"); + return false; + } + break; + } + if (visitedIds.find(prev) != visitedIds.end()) { + fprintf(stderr, "found cycle on linked_chunks.prev = %d.\n", + static_cast<int>(prev)); + return false; + } + id = prev; + } + if (visitedIds.size() != static_cast<size_t>(count_linked_chunks)) { + fprintf(stderr, + "ghost items in linked_chunks when following prev.\n"); + return false; + } + } else if (count_linked_chunks) { + fprintf(stderr, "linked_chunks_head_tail.tail = NULL but linked_chunks " + "not empty.\n"); + return false; + } + + fprintf(stderr, "check ok\n"); + return true; +} + +// --------------------------------------------------------------------------- + +void DiskChunkCache::commitAndClose() { + if (hDB_) { + if( sqlite3_exec(hDB_, "COMMIT", nullptr, nullptr, nullptr) != SQLITE_OK ) { + pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_)); + } + sqlite3_close(hDB_); + hDB_ = nullptr; + } +} + +// --------------------------------------------------------------------------- + +DiskChunkCache::~DiskChunkCache() { + commitAndClose(); +} + +// --------------------------------------------------------------------------- + +void DiskChunkCache::closeAndUnlink() { + commitAndClose(); + if (vfs_) { + vfs_->raw()->xDelete(vfs_->raw(), path_.c_str(), 0); + } +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<SQLiteStatement> DiskChunkCache::prepare(const char *sql) { + sqlite3_stmt *hStmt = nullptr; + sqlite3_prepare_v2(hDB_, sql, -1, &hStmt, nullptr); + if (!hStmt) { + pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_)); + return nullptr; + } + return std::unique_ptr<SQLiteStatement>(new SQLiteStatement(hStmt)); +} + +// --------------------------------------------------------------------------- + +bool DiskChunkCache::get_links(sqlite3_int64 chunk_id, sqlite3_int64 &link_id, + sqlite3_int64 &prev, sqlite3_int64 &next, + sqlite3_int64 &head, sqlite3_int64 &tail) { + auto stmt = + prepare("SELECT id, prev, next FROM linked_chunks WHERE chunk_id = ?"); + if (!stmt) + return false; + stmt->bindInt64(chunk_id); + { + const auto ret = stmt->execute(); + if (ret != SQLITE_ROW) { + pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_)); + return false; + } + } + link_id = stmt->getInt64(); + prev = stmt->getInt64(); + next = stmt->getInt64(); + + stmt = prepare("SELECT head, tail FROM linked_chunks_head_tail"); + { + const auto ret = stmt->execute(); + if (ret != SQLITE_ROW) { + pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_)); + return false; + } + } + head = stmt->getInt64(); + tail = stmt->getInt64(); + return true; +} + +// --------------------------------------------------------------------------- + +bool DiskChunkCache::update_links_of_prev_and_next_links(sqlite3_int64 prev, + sqlite3_int64 next) { + if (prev) { + auto stmt = prepare("UPDATE linked_chunks SET next = ? WHERE id = ?"); + if (!stmt) + return false; + if (next) + stmt->bindInt64(next); + else + stmt->bindNull(); + stmt->bindInt64(prev); + const auto ret = stmt->execute(); + if (ret != SQLITE_DONE) { + pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_)); + return false; + } + } + + if (next) { + auto stmt = prepare("UPDATE linked_chunks SET prev = ? WHERE id = ?"); + if (!stmt) + return false; + if (prev) + stmt->bindInt64(prev); + else + stmt->bindNull(); + stmt->bindInt64(next); + const auto ret = stmt->execute(); + if (ret != SQLITE_DONE) { + pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_)); + return false; + } + } + return true; +} + +// --------------------------------------------------------------------------- + +bool DiskChunkCache::update_linked_chunks(sqlite3_int64 link_id, + sqlite3_int64 prev, + sqlite3_int64 next) { + auto stmt = + prepare("UPDATE linked_chunks SET prev = ?, next = ? WHERE id = ?"); + if (!stmt) + return false; + if (prev) + stmt->bindInt64(prev); + else + stmt->bindNull(); + if (next) + stmt->bindInt64(next); + else + stmt->bindNull(); + stmt->bindInt64(link_id); + const auto ret = stmt->execute(); + if (ret != SQLITE_DONE) { + pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_)); + return false; + } + return true; +} + +// --------------------------------------------------------------------------- + +bool DiskChunkCache::update_linked_chunks_head_tail(sqlite3_int64 head, + sqlite3_int64 tail) { + auto stmt = + prepare("UPDATE linked_chunks_head_tail SET head = ?, tail = ?"); + if (!stmt) + return false; + if (head) + stmt->bindInt64(head); + else + stmt->bindNull(); // shouldn't happen normally + if (tail) + stmt->bindInt64(tail); + else + stmt->bindNull(); // shouldn't happen normally + const auto ret = stmt->execute(); + if (ret != SQLITE_DONE) { + pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_)); + return false; + } + return true; +} + +// --------------------------------------------------------------------------- + +bool DiskChunkCache::move_to_head(sqlite3_int64 chunk_id) { + + sqlite3_int64 link_id = 0; + sqlite3_int64 prev = 0; + sqlite3_int64 next = 0; + sqlite3_int64 head = 0; + sqlite3_int64 tail = 0; + if (!get_links(chunk_id, link_id, prev, next, head, tail)) { + return false; + } + + if (link_id == head) { + return true; + } + + if (!update_links_of_prev_and_next_links(prev, next)) { + return false; + } + + if (head) { + auto stmt = prepare("UPDATE linked_chunks SET prev = ? WHERE id = ?"); + if (!stmt) + return false; + stmt->bindInt64(link_id); + stmt->bindInt64(head); + const auto ret = stmt->execute(); + if (ret != SQLITE_DONE) { + pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_)); + return false; + } + } + + return update_linked_chunks(link_id, 0, head) && + update_linked_chunks_head_tail(link_id, + (link_id == tail) ? prev : tail); +} + +// --------------------------------------------------------------------------- + +bool DiskChunkCache::move_to_tail(sqlite3_int64 chunk_id) { + sqlite3_int64 link_id = 0; + sqlite3_int64 prev = 0; + sqlite3_int64 next = 0; + sqlite3_int64 head = 0; + sqlite3_int64 tail = 0; + if (!get_links(chunk_id, link_id, prev, next, head, tail)) { + return false; + } + + if (link_id == tail) { + return true; + } + + if (!update_links_of_prev_and_next_links(prev, next)) { + return false; + } + + if (tail) { + auto stmt = prepare("UPDATE linked_chunks SET next = ? WHERE id = ?"); + if (!stmt) + return false; + stmt->bindInt64(link_id); + stmt->bindInt64(tail); + const auto ret = stmt->execute(); + if (ret != SQLITE_DONE) { + pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_)); + return false; + } + } + + return update_linked_chunks(link_id, tail, 0) && + update_linked_chunks_head_tail((link_id == head) ? next : head, + link_id); +} + +// --------------------------------------------------------------------------- + +void NetworkChunkCache::insert(PJ_CONTEXT *ctx, const std::string &url, + unsigned long long chunkIdx, + std::vector<unsigned char> &&data) { + auto dataPtr(std::make_shared<std::vector<unsigned char>>(std::move(data))); + cache_.insert(Key(url, chunkIdx), dataPtr); + + auto diskCache = DiskChunkCache::open(ctx); + if (!diskCache) + return; + auto hDB = diskCache->handle(); + + // Always insert DOWNLOAD_CHUNK_SIZE bytes to avoid fragmentation + std::vector<unsigned char> blob(*dataPtr); + assert(blob.size() <= DOWNLOAD_CHUNK_SIZE); + blob.resize(DOWNLOAD_CHUNK_SIZE); + + // Check if there is an existing entry for that URL and offset + auto stmt = diskCache->prepare( + "SELECT id, data_id FROM chunks WHERE url = ? AND offset = ?"); + if (!stmt) + return; + stmt->bindText(url.c_str()); + stmt->bindInt64(chunkIdx * DOWNLOAD_CHUNK_SIZE); + + const auto mainRet = stmt->execute(); + if (mainRet == SQLITE_ROW) { + const auto chunk_id = stmt->getInt64(); + const auto data_id = stmt->getInt64(); + stmt = + diskCache->prepare("UPDATE chunk_data SET data = ? WHERE id = ?"); + if (!stmt) + return; + stmt->bindBlob(blob.data(), blob.size()); + stmt->bindInt64(data_id); + { + const auto ret = stmt->execute(); + if (ret != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + } + + diskCache->move_to_head(chunk_id); + + return; + } else if (mainRet != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + + // Lambda to recycle an existing entry that was either invalidated, or + // least recently used. + const auto reuseExistingEntry = [ctx, &blob, &diskCache, hDB, &url, + chunkIdx, &dataPtr]( + std::unique_ptr<SQLiteStatement> &stmtIn) { + const auto chunk_id = stmtIn->getInt64(); + const auto data_id = stmtIn->getInt64(); + if (data_id <= 0) { + pj_log(ctx, PJ_LOG_ERROR, "data_id <= 0"); + return; + } + + auto l_stmt = + diskCache->prepare("UPDATE chunk_data SET data = ? WHERE id = ?"); + if (!l_stmt) + return; + l_stmt->bindBlob(blob.data(), blob.size()); + l_stmt->bindInt64(data_id); + { + const auto ret2 = l_stmt->execute(); + if (ret2 != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + } + + l_stmt = diskCache->prepare("UPDATE chunks SET url = ?, " + "offset = ?, data_size = ?, data_id = ? " + "WHERE id = ?"); + if (!l_stmt) + return; + l_stmt->bindText(url.c_str()); + l_stmt->bindInt64(chunkIdx * DOWNLOAD_CHUNK_SIZE); + l_stmt->bindInt64(dataPtr->size()); + l_stmt->bindInt64(data_id); + l_stmt->bindInt64(chunk_id); + { + const auto ret2 = l_stmt->execute(); + if (ret2 != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + } + + diskCache->move_to_head(chunk_id); + }; + + // Find if there is an invalidated chunk we can reuse + stmt = diskCache->prepare( + "SELECT id, data_id FROM chunks " + "WHERE id = (SELECT tail FROM linked_chunks_head_tail) AND " + "url = " INVALIDATED_SQL_LITERAL); + if (!stmt) + return; + { + const auto ret = stmt->execute(); + if (ret == SQLITE_ROW) { + reuseExistingEntry(stmt); + return; + } else if (ret != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + } + + // Check if we have not reached the max size of the cache + stmt = diskCache->prepare("SELECT COUNT(*) FROM chunks"); + if (!stmt) + return; + { + const auto ret = stmt->execute(); + if (ret != SQLITE_ROW) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + } + + const auto max_size = pj_context_get_grid_cache_max_size(ctx); + if (max_size > 0 && + static_cast<long long>(stmt->getInt64() * DOWNLOAD_CHUNK_SIZE) >= + max_size) { + stmt = diskCache->prepare( + "SELECT id, data_id FROM chunks " + "WHERE id = (SELECT tail FROM linked_chunks_head_tail)"); + if (!stmt) + return; + + const auto ret = stmt->execute(); + if (ret != SQLITE_ROW) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + reuseExistingEntry(stmt); + return; + } + + // Otherwise just append a new entry + stmt = diskCache->prepare("INSERT INTO chunk_data(data) VALUES (?)"); + if (!stmt) + return; + stmt->bindBlob(blob.data(), blob.size()); + { + const auto ret = stmt->execute(); + if (ret != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + } + + const auto chunk_data_id = sqlite3_last_insert_rowid(hDB); + + stmt = diskCache->prepare("INSERT INTO chunks(url, offset, data_id, " + "data_size) VALUES (?,?,?,?)"); + if (!stmt) + return; + stmt->bindText(url.c_str()); + stmt->bindInt64(chunkIdx * DOWNLOAD_CHUNK_SIZE); + stmt->bindInt64(chunk_data_id); + stmt->bindInt64(dataPtr->size()); + { + const auto ret = stmt->execute(); + if (ret != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + } + + const auto chunk_id = sqlite3_last_insert_rowid(hDB); + + stmt = diskCache->prepare( + "INSERT INTO linked_chunks(chunk_id, prev, next) VALUES (?,NULL,NULL)"); + if (!stmt) + return; + stmt->bindInt64(chunk_id); + if (stmt->execute() != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + + stmt = diskCache->prepare("SELECT head FROM linked_chunks_head_tail"); + if (!stmt) + return; + if (stmt->execute() != SQLITE_ROW) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + if (stmt->getInt64() == 0) { + stmt = diskCache->prepare( + "UPDATE linked_chunks_head_tail SET head = ?, tail = ?"); + if (!stmt) + return; + stmt->bindInt64(chunk_id); + stmt->bindInt64(chunk_id); + if (stmt->execute() != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + } + + diskCache->move_to_head(chunk_id); +} + +// --------------------------------------------------------------------------- + +std::shared_ptr<std::vector<unsigned char>> +NetworkChunkCache::get(PJ_CONTEXT *ctx, const std::string &url, + unsigned long long chunkIdx) { + std::shared_ptr<std::vector<unsigned char>> ret; + if (cache_.tryGet(Key(url, chunkIdx), ret)) { + return ret; + } + + auto diskCache = DiskChunkCache::open(ctx); + if (!diskCache) + return ret; + auto hDB = diskCache->handle(); + + auto stmt = diskCache->prepare( + "SELECT chunks.id, chunks.data_size, chunk_data.data FROM chunks " + "JOIN chunk_data ON chunks.id = chunk_data.id " + "WHERE chunks.url = ? AND chunks.offset = ?"); + if (!stmt) + return ret; + + stmt->bindText(url.c_str()); + stmt->bindInt64(chunkIdx * DOWNLOAD_CHUNK_SIZE); + + const auto mainRet = stmt->execute(); + if (mainRet == SQLITE_ROW) { + const auto chunk_id = stmt->getInt64(); + const auto data_size = stmt->getInt64(); + int blob_size = 0; + const void *blob = stmt->getBlob(blob_size); + if (blob_size < data_size) { + pj_log(ctx, PJ_LOG_ERROR, + "blob_size=%d < data_size for chunk_id=%d", blob_size, + static_cast<int>(chunk_id)); + return ret; + } + if (data_size > static_cast<sqlite3_int64>(DOWNLOAD_CHUNK_SIZE)) { + pj_log(ctx, PJ_LOG_ERROR, "data_size > DOWNLOAD_CHUNK_SIZE"); + return ret; + } + ret.reset(new std::vector<unsigned char>()); + ret->assign(reinterpret_cast<const unsigned char *>(blob), + reinterpret_cast<const unsigned char *>(blob) + + static_cast<size_t>(data_size)); + cache_.insert(Key(url, chunkIdx), ret); + + if (!diskCache->move_to_head(chunk_id)) + return ret; + } else if (mainRet != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + } + + return ret; +} + +// --------------------------------------------------------------------------- + +std::shared_ptr<std::vector<unsigned char>> +NetworkChunkCache::get(PJ_CONTEXT *ctx, const std::string &url, + unsigned long long chunkIdx, FileProperties &props) { + if (!gNetworkFileProperties.tryGet(ctx, url, props)) { + return nullptr; + } + + return get(ctx, url, chunkIdx); +} + +// --------------------------------------------------------------------------- + +void NetworkChunkCache::clearMemoryCache() { cache_.clear(); } + +// --------------------------------------------------------------------------- + +void NetworkChunkCache::clearDiskChunkCache(PJ_CONTEXT *ctx) { + auto diskCache = DiskChunkCache::open(ctx); + if (!diskCache) + return; + diskCache->closeAndUnlink(); +} + +// --------------------------------------------------------------------------- + +void NetworkFilePropertiesCache::insert(PJ_CONTEXT *ctx, const std::string &url, + FileProperties &props) { + time(&props.lastChecked); + cache_.insert(url, props); + + auto diskCache = DiskChunkCache::open(ctx); + if (!diskCache) + return; + auto hDB = diskCache->handle(); + auto stmt = diskCache->prepare("SELECT fileSize, lastModified, etag " + "FROM properties WHERE url = ?"); + if (!stmt) + return; + stmt->bindText(url.c_str()); + if (stmt->execute() == SQLITE_ROW) { + FileProperties cachedProps; + cachedProps.size = stmt->getInt64(); + const char *lastModified = stmt->getText(); + cachedProps.lastModified = lastModified ? lastModified : std::string(); + const char *etag = stmt->getText(); + cachedProps.etag = etag ? etag : std::string(); + if (props.size != cachedProps.size || + props.lastModified != cachedProps.lastModified || + props.etag != cachedProps.etag) { + + // If cached properties don't match recent fresh ones, invalidate + // cached chunks + stmt = diskCache->prepare("SELECT id FROM chunks WHERE url = ?"); + if (!stmt) + return; + stmt->bindText(url.c_str()); + std::vector<sqlite3_int64> ids; + while (stmt->execute() == SQLITE_ROW) { + ids.emplace_back(stmt->getInt64()); + stmt->resetResIndex(); + } + + for (const auto id : ids) { + diskCache->move_to_tail(id); + } + + stmt = diskCache->prepare( + "UPDATE chunks SET url = " INVALIDATED_SQL_LITERAL ", " + "offset = -1, data_size = 0 WHERE url = ?"); + if (!stmt) + return; + stmt->bindText(url.c_str()); + if (stmt->execute() != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + } + + stmt = diskCache->prepare("UPDATE properties SET lastChecked = ?, " + "fileSize = ?, lastModified = ?, etag = ? " + "WHERE url = ?"); + if (!stmt) + return; + stmt->bindInt64(props.lastChecked); + stmt->bindInt64(props.size); + if (props.lastModified.empty()) + stmt->bindNull(); + else + stmt->bindText(props.lastModified.c_str()); + if (props.etag.empty()) + stmt->bindNull(); + else + stmt->bindText(props.etag.c_str()); + stmt->bindText(url.c_str()); + if (stmt->execute() != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + } else { + stmt = diskCache->prepare("INSERT INTO properties (url, lastChecked, " + "fileSize, lastModified, etag) VALUES " + "(?,?,?,?,?)"); + if (!stmt) + return; + stmt->bindText(url.c_str()); + stmt->bindInt64(props.lastChecked); + stmt->bindInt64(props.size); + if (props.lastModified.empty()) + stmt->bindNull(); + else + stmt->bindText(props.lastModified.c_str()); + if (props.etag.empty()) + stmt->bindNull(); + else + stmt->bindText(props.etag.c_str()); + if (stmt->execute() != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return; + } + } +} + +// --------------------------------------------------------------------------- + +bool NetworkFilePropertiesCache::tryGet(PJ_CONTEXT *ctx, const std::string &url, + FileProperties &props) { + if (cache_.tryGet(url, props)) { + return true; + } + + auto diskCache = DiskChunkCache::open(ctx); + if (!diskCache) + return false; + auto stmt = + diskCache->prepare("SELECT lastChecked, fileSize, lastModified, etag " + "FROM properties WHERE url = ?"); + if (!stmt) + return false; + stmt->bindText(url.c_str()); + if (stmt->execute() != SQLITE_ROW) { + return false; + } + props.lastChecked = stmt->getInt64(); + props.size = stmt->getInt64(); + const char *lastModified = stmt->getText(); + props.lastModified = lastModified ? lastModified : std::string(); + const char *etag = stmt->getText(); + props.etag = etag ? etag : std::string(); + + const auto ttl = pj_context_get_grid_cache_ttl(ctx); + if (ttl > 0) { + time_t curTime; + time(&curTime); + if (curTime > props.lastChecked + ttl) { + props = FileProperties(); + return false; + } + } + cache_.insert(url, props); + return true; +} + +// --------------------------------------------------------------------------- + +void NetworkFilePropertiesCache::clearMemoryCache() { cache_.clear(); } + +// --------------------------------------------------------------------------- + +class NetworkFile : public File { + PJ_CONTEXT *m_ctx; + std::string m_url; + PROJ_NETWORK_HANDLE *m_handle; + unsigned long long m_pos = 0; + size_t m_nBlocksToDownload = 1; + unsigned long long m_lastDownloadedOffset; + FileProperties m_props; + proj_network_close_cbk_type m_closeCbk; + bool m_hasChanged = false; + + NetworkFile(const NetworkFile &) = delete; + NetworkFile &operator=(const NetworkFile &) = delete; + + protected: + NetworkFile(PJ_CONTEXT *ctx, const std::string &url, + PROJ_NETWORK_HANDLE *handle, + unsigned long long lastDownloadOffset, + const FileProperties &props) + : File(url), m_ctx(ctx), m_url(url), m_handle(handle), + m_lastDownloadedOffset(lastDownloadOffset), m_props(props), + m_closeCbk(ctx->networking.close) {} + + public: + ~NetworkFile() override; + + size_t read(void *buffer, size_t sizeBytes) override; + size_t write(const void *, size_t) override { return 0; } + bool seek(unsigned long long offset, int whence) override; + unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override; + bool hasChanged() const override { return m_hasChanged; } + + static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename); + + static bool get_props_from_headers(PJ_CONTEXT *ctx, + PROJ_NETWORK_HANDLE *handle, + FileProperties &props); +}; + +// --------------------------------------------------------------------------- + +bool NetworkFile::get_props_from_headers(PJ_CONTEXT *ctx, + PROJ_NETWORK_HANDLE *handle, + FileProperties &props) { + const char *contentRange = ctx->networking.get_header_value( + ctx, handle, "Content-Range", ctx->networking.user_data); + if (contentRange) { + const char *slash = strchr(contentRange, '/'); + if (slash) { + props.size = std::stoull(slash + 1); + + const char *lastModified = ctx->networking.get_header_value( + ctx, handle, "Last-Modified", ctx->networking.user_data); + if (lastModified) + props.lastModified = lastModified; + + const char *etag = ctx->networking.get_header_value( + ctx, handle, "ETag", ctx->networking.user_data); + if (etag) + props.etag = etag; + + return true; + } + } + return false; +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<File> NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { + FileProperties props; + if (gNetworkChunkCache.get(ctx, filename, 0, props)) { + return std::unique_ptr<File>(new NetworkFile( + ctx, filename, nullptr, + std::numeric_limits<unsigned long long>::max(), props)); + } else { + std::vector<unsigned char> buffer(DOWNLOAD_CHUNK_SIZE); + size_t size_read = 0; + std::string errorBuffer; + errorBuffer.resize(1024); + + auto handle = ctx->networking.open( + ctx, filename, 0, buffer.size(), &buffer[0], &size_read, + errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data); + buffer.resize(size_read); + if (!handle) { + errorBuffer.resize(strlen(errorBuffer.data())); + pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", filename, + errorBuffer.c_str()); + pj_ctx_set_errno(ctx, PJD_ERR_NETWORK_ERROR); + } + + bool ok = false; + if (handle) { + if (get_props_from_headers(ctx, handle, props)) { + ok = true; + gNetworkFileProperties.insert(ctx, filename, props); + gNetworkChunkCache.insert(ctx, filename, 0, std::move(buffer)); + } + } + + return std::unique_ptr<File>( + ok ? new NetworkFile(ctx, filename, handle, size_read, props) + : nullptr); + } +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<File> pj_network_file_open(PJ_CONTEXT* ctx, const char* filename) { + return NetworkFile::open(ctx, filename); +} + +// --------------------------------------------------------------------------- + +size_t NetworkFile::read(void *buffer, size_t sizeBytes) { + + if (sizeBytes == 0) + return 0; + + auto iterOffset = m_pos; + while (sizeBytes) { + const auto chunkIdxToDownload = iterOffset / DOWNLOAD_CHUNK_SIZE; + const auto offsetToDownload = chunkIdxToDownload * DOWNLOAD_CHUNK_SIZE; + std::vector<unsigned char> region; + auto pChunk = gNetworkChunkCache.get(m_ctx, m_url, chunkIdxToDownload); + if (pChunk != nullptr) { + region = *pChunk; + } else { + if (offsetToDownload == m_lastDownloadedOffset) { + // In case of consecutive reads (of small size), we use a + // heuristic that we will read the file sequentially, so + // we double the requested size to decrease the number of + // client/server roundtrips. + if (m_nBlocksToDownload < 100) + m_nBlocksToDownload *= 2; + } else { + // Random reads. Cancel the above heuristics. + m_nBlocksToDownload = 1; + } + + // Ensure that we will request at least the number of blocks + // to satisfy the remaining buffer size to read. + const auto endOffsetToDownload = + ((iterOffset + sizeBytes + DOWNLOAD_CHUNK_SIZE - 1) / + DOWNLOAD_CHUNK_SIZE) * + DOWNLOAD_CHUNK_SIZE; + const auto nMinBlocksToDownload = static_cast<size_t>( + (endOffsetToDownload - offsetToDownload) / DOWNLOAD_CHUNK_SIZE); + if (m_nBlocksToDownload < nMinBlocksToDownload) + m_nBlocksToDownload = nMinBlocksToDownload; + + // Avoid reading already cached data. + // Note: this might get evicted if concurrent reads are done, but + // this should not cause bugs. Just missed optimization. + for (size_t i = 1; i < m_nBlocksToDownload; i++) { + if (gNetworkChunkCache.get(m_ctx, m_url, + chunkIdxToDownload + i) != nullptr) { + m_nBlocksToDownload = i; + break; + } + } + + if (m_nBlocksToDownload > MAX_CHUNKS) + m_nBlocksToDownload = MAX_CHUNKS; + + region.resize(m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE); + size_t nRead = 0; + std::string errorBuffer; + errorBuffer.resize(1024); + if (!m_handle) { + m_handle = m_ctx->networking.open( + m_ctx, m_url.c_str(), offsetToDownload, + m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + &nRead, errorBuffer.size(), &errorBuffer[0], + m_ctx->networking.user_data); + if (!m_handle) { + pj_ctx_set_errno(m_ctx, PJD_ERR_NETWORK_ERROR); + return 0; + } + } else { + nRead = m_ctx->networking.read_range( + m_ctx, m_handle, offsetToDownload, + m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + errorBuffer.size(), &errorBuffer[0], + m_ctx->networking.user_data); + } + if (nRead == 0) { + errorBuffer.resize(strlen(errorBuffer.data())); + if (!errorBuffer.empty()) { + pj_log(m_ctx, PJ_LOG_ERROR, "Cannot read in %s: %s", + m_url.c_str(), errorBuffer.c_str()); + } + pj_ctx_set_errno(m_ctx, PJD_ERR_NETWORK_ERROR); + return 0; + } + + if (!m_hasChanged) { + FileProperties props; + if (get_props_from_headers(m_ctx, m_handle, props)) { + if (props.size != m_props.size || + props.lastModified != m_props.lastModified || + props.etag != m_props.etag) { + gNetworkFileProperties.insert(m_ctx, m_url, props); + gNetworkChunkCache.clearMemoryCache(); + m_hasChanged = true; + } + } + } + + region.resize(nRead); + m_lastDownloadedOffset = offsetToDownload + nRead; + + const auto nChunks = + (region.size() + DOWNLOAD_CHUNK_SIZE - 1) / DOWNLOAD_CHUNK_SIZE; + for (size_t i = 0; i < nChunks; i++) { + std::vector<unsigned char> chunk( + region.data() + i * DOWNLOAD_CHUNK_SIZE, + region.data() + + std::min((i + 1) * DOWNLOAD_CHUNK_SIZE, region.size())); + gNetworkChunkCache.insert(m_ctx, m_url, chunkIdxToDownload + i, + std::move(chunk)); + } + } + const size_t nToCopy = static_cast<size_t>( + std::min(static_cast<unsigned long long>(sizeBytes), + region.size() - (iterOffset - offsetToDownload))); + memcpy(buffer, region.data() + iterOffset - offsetToDownload, nToCopy); + buffer = static_cast<char *>(buffer) + nToCopy; + iterOffset += nToCopy; + sizeBytes -= nToCopy; + if (region.size() < static_cast<size_t>(DOWNLOAD_CHUNK_SIZE) && + sizeBytes != 0) { + break; + } + } + + size_t nRead = static_cast<size_t>(iterOffset - m_pos); + m_pos = iterOffset; + return nRead; +} + +// --------------------------------------------------------------------------- + +bool NetworkFile::seek(unsigned long long offset, int whence) { + if (whence == SEEK_SET) { + m_pos = offset; + } else if (whence == SEEK_CUR) { + m_pos += offset; + } else { + if (offset != 0) + return false; + m_pos = m_props.size; + } + return true; +} + +// --------------------------------------------------------------------------- + +unsigned long long NetworkFile::tell() { return m_pos; } + +// --------------------------------------------------------------------------- + +NetworkFile::~NetworkFile() { + if (m_handle) { + m_ctx->networking.close(m_ctx, m_handle, m_ctx->networking.user_data); + } +} + +// --------------------------------------------------------------------------- + +void NetworkFile::reassign_context(PJ_CONTEXT *ctx) { + m_ctx = ctx; + if (m_closeCbk != m_ctx->networking.close) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Networking close callback has changed following context " + "reassignment ! This is highly suspicious"); + } +} + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +struct CurlFileHandle { + std::string m_url; + CURL *m_handle; + std::string m_headers{}; + std::string m_lastval{}; + std::string m_useragent{}; + char m_szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; + + CurlFileHandle(const CurlFileHandle &) = delete; + CurlFileHandle &operator=(const CurlFileHandle &) = delete; + + explicit CurlFileHandle(const char *url, CURL *handle); + ~CurlFileHandle(); + + static PROJ_NETWORK_HANDLE * + open(PJ_CONTEXT *, const char *url, unsigned long long offset, + size_t size_to_read, void *buffer, size_t *out_size_read, + size_t error_string_max_size, char *out_error_string, void *); +}; + +// --------------------------------------------------------------------------- + +static std::string GetExecutableName() { +#if defined(__linux) + std::string path; + path.resize(1024); + const auto ret = readlink("/proc/self/exe", &path[0], path.size()); + if (ret > 0) { + path.resize(ret); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(_WIN32) + std::string path; + path.resize(1024); + if (GetModuleFileNameA(nullptr, &path[0], + static_cast<DWORD>(path.size()))) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('\\'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(__MACH__) && defined(__APPLE__) + std::string path; + path.resize(1024); + uint32_t size = static_cast<uint32_t>(path.size()); + if (_NSGetExecutablePath(&path[0], &size) == 0) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(__FreeBSD__) + int mib[4]; + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PATHNAME; + mib[3] = -1; + std::string path; + path.resize(1024); + size_t size = path.size(); + if (sysctl(mib, 4, &path[0], &size, nullptr, 0) == 0) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#endif + + return std::string(); +} + +// --------------------------------------------------------------------------- + +CurlFileHandle::CurlFileHandle(const char *url, CURL *handle) + : m_url(url), m_handle(handle) { + curl_easy_setopt(handle, CURLOPT_URL, m_url.c_str()); + + if (getenv("PROJ_CURL_VERBOSE")) + curl_easy_setopt(handle, CURLOPT_VERBOSE, 1); + +// CURLOPT_SUPPRESS_CONNECT_HEADERS is defined in curl 7.54.0 or newer. +#if LIBCURL_VERSION_NUM >= 0x073600 + curl_easy_setopt(handle, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L); +#endif + + // Enable following redirections. Requires libcurl 7.10.1 at least. + curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10); + + if (getenv("PROJ_UNSAFE_SSL")) { + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); + } + + curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, m_szCurlErrBuf); + + if (getenv("PROJ_NO_USERAGENT") == nullptr) { + m_useragent = "PROJ " STR(PROJ_VERSION_MAJOR) "." STR( + PROJ_VERSION_MINOR) "." STR(PROJ_VERSION_PATCH); + const auto exeName = GetExecutableName(); + if (!exeName.empty()) { + m_useragent = exeName + " using " + m_useragent; + } + curl_easy_setopt(handle, CURLOPT_USERAGENT, m_useragent.data()); + } +} + +// --------------------------------------------------------------------------- + +CurlFileHandle::~CurlFileHandle() { curl_easy_cleanup(m_handle); } + +// --------------------------------------------------------------------------- + +static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, + void *req) { + const size_t nSize = count * nmemb; + auto pStr = static_cast<std::string *>(req); + if (pStr->size() + nSize > pStr->capacity()) { + // to avoid servers not honouring Range to cause excessive memory + // allocation + return 0; + } + pStr->append(static_cast<const char *>(buffer), nSize); + return nmemb; +} + +// --------------------------------------------------------------------------- + +static double GetNewRetryDelay(int response_code, double dfOldDelay, + const char *pszErrBuf, + const char *pszCurlError) { + if (response_code == 429 || response_code == 500 || + (response_code >= 502 && response_code <= 504) || + // S3 sends some client timeout errors as 400 Client Error + (response_code == 400 && pszErrBuf && + strstr(pszErrBuf, "RequestTimeout")) || + (pszCurlError && strstr(pszCurlError, "Connection timed out"))) { + // Use an exponential backoff factor of 2 plus some random jitter + // We don't care about cryptographic quality randomness, hence: + // coverity[dont_call] + return dfOldDelay * (2 + rand() * 0.5 / RAND_MAX); + } else { + return 0; + } +} + +// --------------------------------------------------------------------------- + +constexpr double MIN_RETRY_DELAY_MS = 500; +constexpr double MAX_RETRY_DELAY_MS = 60000; + +PROJ_NETWORK_HANDLE *CurlFileHandle::open(PJ_CONTEXT *ctx, const char *url, + unsigned long long offset, + size_t size_to_read, void *buffer, + size_t *out_size_read, + size_t error_string_max_size, + char *out_error_string, void *) { + CURL *hCurlHandle = curl_easy_init(); + if (!hCurlHandle) + return nullptr; + + auto file = + std::unique_ptr<CurlFileHandle>(new CurlFileHandle(url, hCurlHandle)); + + double oldDelay = MIN_RETRY_DELAY_MS; + std::string headers; + std::string body; + + char szBuffer[128]; + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, + offset + size_to_read - 1); + + while (true) { + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); + + headers.clear(); + headers.reserve(16 * 1024); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &headers); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, + pj_curl_write_func); + + body.clear(); + body.reserve(size_to_read); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, + pj_curl_write_func); + + file->m_szCurlErrBuf[0] = '\0'; + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, nullptr); + + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + + if (response_code == 0 || response_code >= 300) { + const double delay = + GetNewRetryDelay(static_cast<int>(response_code), oldDelay, + body.c_str(), file->m_szCurlErrBuf); + if (delay != 0 && delay < MAX_RETRY_DELAY_MS) { + pj_log(ctx, PJ_LOG_TRACE, + "Got a HTTP %ld error. Retrying in %d ms", response_code, + static_cast<int>(delay)); + sleep_ms(static_cast<int>(delay)); + oldDelay = delay; + } else { + if (out_error_string) { + if (file->m_szCurlErrBuf[0]) { + snprintf(out_error_string, error_string_max_size, "%s", + file->m_szCurlErrBuf); + } else { + snprintf(out_error_string, error_string_max_size, + "HTTP error %ld: %s", response_code, + body.c_str()); + } + } + return nullptr; + } + } else { + break; + } + } + + if (out_error_string && error_string_max_size) { + out_error_string[0] = '\0'; + } + + if (!body.empty()) { + memcpy(buffer, body.data(), std::min(size_to_read, body.size())); + } + *out_size_read = std::min(size_to_read, body.size()); + + file->m_headers = std::move(headers); + return reinterpret_cast<PROJ_NETWORK_HANDLE *>(file.release()); +} + +// --------------------------------------------------------------------------- + +static void pj_curl_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *handle, + void * /*user_data*/) { + delete reinterpret_cast<CurlFileHandle *>(handle); +} + +// --------------------------------------------------------------------------- + +static size_t pj_curl_read_range(PJ_CONTEXT *ctx, + PROJ_NETWORK_HANDLE *raw_handle, + unsigned long long offset, size_t size_to_read, + void *buffer, size_t error_string_max_size, + char *out_error_string, void *) { + auto handle = reinterpret_cast<CurlFileHandle *>(raw_handle); + auto hCurlHandle = handle->m_handle; + + double oldDelay = MIN_RETRY_DELAY_MS; + std::string headers; + std::string body; + + char szBuffer[128]; + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, + offset + size_to_read - 1); + + while (true) { + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); + + headers.clear(); + headers.reserve(16 * 1024); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &headers); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, + pj_curl_write_func); + + body.clear(); + body.reserve(size_to_read); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, + pj_curl_write_func); + + handle->m_szCurlErrBuf[0] = '\0'; + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + + if (response_code == 0 || response_code >= 300) { + const double delay = + GetNewRetryDelay(static_cast<int>(response_code), oldDelay, + body.c_str(), handle->m_szCurlErrBuf); + if (delay != 0 && delay < MAX_RETRY_DELAY_MS) { + pj_log(ctx, PJ_LOG_TRACE, + "Got a HTTP %ld error. Retrying in %d ms", response_code, + static_cast<int>(delay)); + sleep_ms(static_cast<int>(delay)); + oldDelay = delay; + } else { + if (out_error_string) { + if (handle->m_szCurlErrBuf[0]) { + snprintf(out_error_string, error_string_max_size, "%s", + handle->m_szCurlErrBuf); + } else { + snprintf(out_error_string, error_string_max_size, + "HTTP error %ld: %s", response_code, + body.c_str()); + } + } + return 0; + } + } else { + break; + } + } + if (out_error_string && error_string_max_size) { + out_error_string[0] = '\0'; + } + + if (!body.empty()) { + memcpy(buffer, body.data(), std::min(size_to_read, body.size())); + } + handle->m_headers = std::move(headers); + + return std::min(size_to_read, body.size()); +} + +// --------------------------------------------------------------------------- + +static const char *pj_curl_get_header_value(PJ_CONTEXT *, + PROJ_NETWORK_HANDLE *raw_handle, + const char *header_name, void *) { + auto handle = reinterpret_cast<CurlFileHandle *>(raw_handle); + auto pos = ci_find(handle->m_headers, header_name); + if (pos == std::string::npos) + return nullptr; + pos += strlen(header_name); + const char *c_str = handle->m_headers.c_str(); + if (c_str[pos] == ':') + pos++; + while (c_str[pos] == ' ') + pos++; + auto posEnd = pos; + while (c_str[posEnd] != '\r' && c_str[posEnd] != '\n' && + c_str[posEnd] != '\0') + posEnd++; + handle->m_lastval = handle->m_headers.substr(pos, posEnd - pos); + return handle->m_lastval.c_str(); +} + +#else + +// --------------------------------------------------------------------------- + +static PROJ_NETWORK_HANDLE * +no_op_network_open(PJ_CONTEXT *, const char * /* url */, + unsigned long long, /* offset */ + size_t, /* size to read */ + void *, /* buffer to update with bytes read*/ + size_t *, /* output: size actually read */ + size_t error_string_max_size, char *out_error_string, + void * /*user_data*/) { + if (out_error_string) { + snprintf(out_error_string, error_string_max_size, "%s", + "Network functionality not available"); + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, + void * /*user_data*/) {} + +#endif + +// --------------------------------------------------------------------------- + +void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { +#ifdef CURL_ENABLED + ctx->networking.open = CurlFileHandle::open; + ctx->networking.close = pj_curl_close; + ctx->networking.read_range = pj_curl_read_range; + ctx->networking.get_header_value = pj_curl_get_header_value; +#else + ctx->networking.open = no_op_network_open; + ctx->networking.close = no_op_network_close; +#endif +} + +// --------------------------------------------------------------------------- + +void FileManager::clearMemoryCache() { + gNetworkChunkCache.clearMemoryCache(); + gNetworkFileProperties.clearMemoryCache(); +} + +NS_PROJ_END + +//! @endcond + +// --------------------------------------------------------------------------- + +#ifdef WIN32 +static const char dir_chars[] = "/\\"; +#else +static const char dir_chars[] = "/"; +#endif + +static bool is_tilde_slash(const char *name) { + return *name == '~' && strchr(dir_chars, name[1]); +} + +static bool is_rel_or_absolute_filename(const char *name) { + return strchr(dir_chars, *name) || + (*name == '.' && strchr(dir_chars, name[1])) || + (!strncmp(name, "..", 2) && strchr(dir_chars, name[2])) || + (name[0] != '\0' && name[1] == ':' && strchr(dir_chars, name[2])); +} + +static std::string build_url(PJ_CONTEXT *ctx, const char *name) { + if (!is_tilde_slash(name) && !is_rel_or_absolute_filename(name) && + !starts_with(name, "http://") && !starts_with(name, "https://")) { + std::string remote_file(pj_context_get_url_endpoint(ctx)); + if (!remote_file.empty()) { + if (remote_file.back() != '/') { + remote_file += '/'; + } + remote_file += name; + auto pos = remote_file.rfind('.'); + if (pos + 4 == remote_file.size()) { + remote_file = remote_file.substr(0, pos) + ".tif"; + } else { + // For example for resource files like 'alaska' + remote_file += ".tif"; + } + } + return remote_file; + } + return name; +} + +// --------------------------------------------------------------------------- + +/** Define a custom set of callbacks for network access. + * + * All callbacks should be provided (non NULL pointers). + * + * @param ctx PROJ context, or NULL + * @param open_cbk Callback to open a remote file given its URL + * @param close_cbk Callback to close a remote file. + * @param get_header_value_cbk Callback to get HTTP headers + * @param read_range_cbk Callback to read a range of bytes inside a remote file. + * @param user_data Arbitrary pointer provided by the user, and passed to the + * above callbacks. May be NULL. + * @return TRUE in case of success. + * @since 7.0 + */ +int proj_context_set_network_callbacks( + PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk, + proj_network_close_cbk_type close_cbk, + proj_network_get_header_value_cbk_type get_header_value_cbk, + proj_network_read_range_type read_range_cbk, void *user_data) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk) { + return false; + } + ctx->networking.open = open_cbk; + ctx->networking.close = close_cbk; + ctx->networking.get_header_value = get_header_value_cbk; + ctx->networking.read_range = read_range_cbk; + ctx->networking.user_data = user_data; + return true; +} + +// --------------------------------------------------------------------------- + +/** Enable or disable network access. +* +* This overrides the default endpoint in the PROJ configuration file or with +* the PROJ_NETWORK environment variable. +* +* @param ctx PROJ context, or NULL +* @param enable TRUE if network access is allowed. +* @return TRUE if network access is possible. That is either libcurl is +* available, or an alternate interface has been set. +* @since 7.0 +*/ +int proj_context_set_enable_network(PJ_CONTEXT *ctx, int enable) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + // Load ini file, now so as to override its network settings + pj_load_ini(ctx); + ctx->networking.enabled_env_variable_checked = true; + ctx->networking.enabled = enable != FALSE; +#ifdef CURL_ENABLED + return ctx->networking.enabled; +#else + return ctx->networking.enabled && + ctx->networking.open != NS_PROJ::no_op_network_open; +#endif +} + +//! @endcond + +// --------------------------------------------------------------------------- + +/** Define the URL endpoint to query for remote grids. +* +* This overrides the default endpoint in the PROJ configuration file or with +* the PROJ_NETWORK_ENDPOINT environment variable. +* +* @param ctx PROJ context, or NULL +* @param url Endpoint URL. Must NOT be NULL. +* @since 7.0 +*/ +void proj_context_set_url_endpoint(PJ_CONTEXT *ctx, const char *url) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + // Load ini file, now so as to override its network settings + pj_load_ini(ctx); + ctx->endpoint = url; +} + +// --------------------------------------------------------------------------- + +/** Enable or disable the local cache of grid chunks +* +* This overrides the setting in the PROJ configuration file. +* +* @param ctx PROJ context, or NULL +* @param enabled TRUE if the cache is enabled. +* @since 7.0 +*/ +void proj_grid_cache_set_enable(PJ_CONTEXT *ctx, int enabled) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + // Load ini file, now so as to override its settings + pj_load_ini(ctx); + ctx->gridChunkCache.enabled = enabled != FALSE; +} + +// --------------------------------------------------------------------------- + +/** Override, for the considered context, the path and file of the local +* cache of grid chunks. +* +* @param ctx PROJ context, or NULL +* @param fullname Full name to the cache (encoded in UTF-8). If set to NULL, +* caching will be disabled. +* @since 7.0 +*/ +void proj_grid_cache_set_filename(PJ_CONTEXT *ctx, const char *fullname) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + // Load ini file, now so as to override its settings + pj_load_ini(ctx); + ctx->gridChunkCache.filename = fullname ? fullname : std::string(); +} + +// --------------------------------------------------------------------------- + +/** Override, for the considered context, the maximum size of the local +* cache of grid chunks. +* +* @param ctx PROJ context, or NULL +* @param max_size_MB Maximum size, in mega-bytes (1024*1024 bytes), or +* negative value to set unlimited size. +* @since 7.0 +*/ +void proj_grid_cache_set_max_size(PJ_CONTEXT *ctx, int max_size_MB) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + // Load ini file, now so as to override its settings + pj_load_ini(ctx); + ctx->gridChunkCache.max_size = + max_size_MB < 0 ? -1 + : static_cast<long long>(max_size_MB) * 1024 * 1024; + if (max_size_MB == 0) { + // For debug purposes only + const char *env_var = getenv("PROJ_GRID_CACHE_MAX_SIZE_BYTES"); + if (env_var && env_var[0] != '\0') { + ctx->gridChunkCache.max_size = atoi(env_var); + } + } +} + +// --------------------------------------------------------------------------- + +/** Override, for the considered context, the time-to-live delay for +* re-checking if the cached properties of files are still up-to-date. +* +* @param ctx PROJ context, or NULL +* @param ttl_seconds Delay in seconds. Use negative value for no expiration. +* @since 7.0 +*/ +void proj_grid_cache_set_ttl(PJ_CONTEXT *ctx, int ttl_seconds) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + // Load ini file, now so as to override its settings + pj_load_ini(ctx); + ctx->gridChunkCache.ttl = ttl_seconds; +} + +// --------------------------------------------------------------------------- + +/** Clear the local cache of grid chunks. +* +* @param ctx PROJ context, or NULL +* @since 7.0 +*/ +void proj_grid_cache_clear(PJ_CONTEXT *ctx) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + NS_PROJ::gNetworkChunkCache.clearDiskChunkCache(ctx); +} + +// --------------------------------------------------------------------------- + +/** Return if a file must be downloaded or is already available in the + * PROJ user-writable directory. + * + * The file will be determinted to have to be downloaded if it does not exist + * yet in the user-writable directory, or if it is determined that a more recent + * version exists. To determine if a more recent version exists, PROJ will + * use the "downloaded_file_properties" table of its grid cache database. + * Consequently files manually placed in the user-writable + * directory without using this function would be considered as + * non-existing/obsolete and would be unconditionnaly downloaded again. + * + * This function can only be used if networking is enabled, and either + * the default curl network API or a custom one have been installed. + * + * @param ctx PROJ context, or NULL + * @param url_or_filename URL or filename (without directory component) + * @param ignore_ttl_setting If set to FALSE, PROJ will only check the + * recentness of an already downloaded file, if + * the delay between the last time it has been + * verified and the current time exceeds the TTL + * setting. This can save network accesses. + * If set to TRUE, PROJ will unconditionnally + * check from the server the recentness of the file. + * @return TRUE if the file must be downloaded with proj_download_file() + * @since 7.0 + */ + +int proj_is_download_needed(PJ_CONTEXT *ctx, const char *url_or_filename, + int ignore_ttl_setting) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (!pj_context_is_network_enabled(ctx)) { + pj_log(ctx, PJ_LOG_ERROR, "Networking capabilities are not enabled"); + return false; + } + + const auto url(build_url(ctx, url_or_filename)); + const char *filename = strrchr(url.c_str(), '/'); + if (filename == nullptr) + return false; + const auto localFilename( + pj_context_get_user_writable_directory(ctx, false) + filename); + + auto f = NS_PROJ::FileManager::open(ctx, localFilename.c_str(), + NS_PROJ::FileAccess::READ_ONLY); + if (!f) { + return true; + } + f.reset(); + + auto diskCache = NS_PROJ::DiskChunkCache::open(ctx); + if (!diskCache) + return false; + auto stmt = + diskCache->prepare("SELECT lastChecked, fileSize, lastModified, etag " + "FROM downloaded_file_properties WHERE url = ?"); + if (!stmt) + return true; + stmt->bindText(url.c_str()); + if (stmt->execute() != SQLITE_ROW) { + return true; + } + + NS_PROJ::FileProperties cachedProps; + cachedProps.lastChecked = stmt->getInt64(); + cachedProps.size = stmt->getInt64(); + const char *lastModified = stmt->getText(); + cachedProps.lastModified = lastModified ? lastModified : std::string(); + const char *etag = stmt->getText(); + cachedProps.etag = etag ? etag : std::string(); + + if (!ignore_ttl_setting) { + const auto ttl = NS_PROJ::pj_context_get_grid_cache_ttl(ctx); + if (ttl > 0) { + time_t curTime; + time(&curTime); + if (curTime > cachedProps.lastChecked + ttl) { + + unsigned char dummy; + size_t size_read = 0; + std::string errorBuffer; + errorBuffer.resize(1024); + auto handle = ctx->networking.open( + ctx, url.c_str(), 0, 1, &dummy, &size_read, + errorBuffer.size(), &errorBuffer[0], + ctx->networking.user_data); + if (!handle) { + errorBuffer.resize(strlen(errorBuffer.data())); + pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", url.c_str(), + errorBuffer.c_str()); + return false; + } + NS_PROJ::FileProperties props; + if (!NS_PROJ::NetworkFile::get_props_from_headers(ctx, handle, + props)) { + ctx->networking.close(ctx, handle, + ctx->networking.user_data); + return false; + } + ctx->networking.close(ctx, handle, ctx->networking.user_data); + + if (props.size != cachedProps.size || + props.lastModified != cachedProps.lastModified || + props.etag != cachedProps.etag) { + return true; + } + + stmt = diskCache->prepare( + "UPDATE downloaded_file_properties SET lastChecked = ? " + "WHERE url = ?"); + if (!stmt) + return false; + stmt->bindInt64(curTime); + stmt->bindText(url.c_str()); + if (stmt->execute() != SQLITE_DONE) { + auto hDB = diskCache->handle(); + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return false; + } + } + } + } + + return false; +} + +// --------------------------------------------------------------------------- + +/** Download a file in the PROJ user-writable directory. + * + * The file will only be downloaded if it does not exist yet in the + * user-writable directory, or if it is determined that a more recent + * version exists. To determine if a more recent version exists, PROJ will + * use the "downloaded_file_properties" table of its grid cache database. + * Consequently files manually placed in the user-writable + * directory without using this function would be considered as + * non-existing/obsolete and would be unconditionnaly downloaded again. + * + * This function can only be used if networking is enabled, and either + * the default curl network API or a custom one have been installed. + * + * @param ctx PROJ context, or NULL + * @param url_or_filename URL or filename (without directory component) + * @param ignore_ttl_setting If set to FALSE, PROJ will only check the + * recentness of an already downloaded file, if + * the delay between the last time it has been + * verified and the current time exceeds the TTL + * setting. This can save network accesses. + * If set to TRUE, PROJ will unconditionnally + * check from the server the recentness of the file. + * @param progress_cbk Progress callback, or NULL. + * The passed percentage is in the [0, 1] range. + * The progress callback must return TRUE + * if download must be continued. + * @param user_data User data to provide to the progress callback, or NULL + * @return TRUE if the download was successful (or not needed) + * @since 7.0 + */ + +int proj_download_file(PJ_CONTEXT *ctx, const char *url_or_filename, + int ignore_ttl_setting, + int (*progress_cbk)(PJ_CONTEXT *, double pct, + void *user_data), + void *user_data) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (!pj_context_is_network_enabled(ctx)) { + pj_log(ctx, PJ_LOG_ERROR, "Networking capabilities are not enabled"); + return false; + } + if (!proj_is_download_needed(ctx, url_or_filename, ignore_ttl_setting)) { + return true; + } + + const auto url(build_url(ctx, url_or_filename)); + const char *filename = strrchr(url.c_str(), '/'); + if (filename == nullptr) + return false; + const auto localFilename(pj_context_get_user_writable_directory(ctx, true) + + filename); + +#ifdef _WIN32 + const int nPID = GetCurrentProcessId(); +#else + const int nPID = getpid(); +#endif + char szUniqueSuffix[128]; + snprintf(szUniqueSuffix, sizeof(szUniqueSuffix), "%d_%p", nPID, &url); + const auto localFilenameTmp(localFilename + szUniqueSuffix); + auto f = NS_PROJ::FileManager::open(ctx, localFilenameTmp.c_str(), + NS_PROJ::FileAccess::CREATE); + if (!f) { + pj_log(ctx, PJ_LOG_ERROR, "Cannot create %s", localFilenameTmp.c_str()); + return false; + } + + constexpr size_t FULL_FILE_CHUNK_SIZE = 1024 * 1024; + std::vector<unsigned char> buffer(FULL_FILE_CHUNK_SIZE); + // For testing purposes only + const char *env_var_PROJ_FULL_FILE_CHUNK_SIZE = + getenv("PROJ_FULL_FILE_CHUNK_SIZE"); + if (env_var_PROJ_FULL_FILE_CHUNK_SIZE && + env_var_PROJ_FULL_FILE_CHUNK_SIZE[0] != '\0') { + buffer.resize(atoi(env_var_PROJ_FULL_FILE_CHUNK_SIZE)); + } + size_t size_read = 0; + std::string errorBuffer; + errorBuffer.resize(1024); + auto handle = ctx->networking.open( + ctx, url.c_str(), 0, buffer.size(), &buffer[0], &size_read, + errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data); + if (!handle) { + errorBuffer.resize(strlen(errorBuffer.data())); + pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", url.c_str(), + errorBuffer.c_str()); + f.reset(); + NS_PROJ::FileManager::unlink(ctx, localFilenameTmp.c_str()); + return false; + } + + time_t curTime; + time(&curTime); + NS_PROJ::FileProperties props; + if (!NS_PROJ::NetworkFile::get_props_from_headers(ctx, handle, props)) { + ctx->networking.close(ctx, handle, ctx->networking.user_data); + f.reset(); + NS_PROJ::FileManager::unlink(ctx, localFilenameTmp.c_str()); + return false; + } + + if (size_read < + std::min(static_cast<unsigned long long>(buffer.size()), props.size)) { + pj_log(ctx, PJ_LOG_ERROR, "Did not get as many bytes as expected"); + ctx->networking.close(ctx, handle, ctx->networking.user_data); + f.reset(); + NS_PROJ::FileManager::unlink(ctx, localFilenameTmp.c_str()); + return false; + } + if (f->write(buffer.data(), size_read) != size_read) { + pj_log(ctx, PJ_LOG_ERROR, "Write error"); + ctx->networking.close(ctx, handle, ctx->networking.user_data); + f.reset(); + NS_PROJ::FileManager::unlink(ctx, localFilenameTmp.c_str()); + return false; + } + + unsigned long long totalDownloaded = size_read; + while (totalDownloaded < props.size) { + if (totalDownloaded + buffer.size() > props.size) { + buffer.resize(static_cast<size_t>(props.size - totalDownloaded)); + } + errorBuffer.resize(1024); + size_read = ctx->networking.read_range( + ctx, handle, totalDownloaded, buffer.size(), &buffer[0], + errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data); + + if (size_read < buffer.size()) { + pj_log(ctx, PJ_LOG_ERROR, "Did not get as many bytes as expected"); + ctx->networking.close(ctx, handle, ctx->networking.user_data); + f.reset(); + NS_PROJ::FileManager::unlink(ctx, localFilenameTmp.c_str()); + return false; + } + if (f->write(buffer.data(), size_read) != size_read) { + pj_log(ctx, PJ_LOG_ERROR, "Write error"); + ctx->networking.close(ctx, handle, ctx->networking.user_data); + f.reset(); + NS_PROJ::FileManager::unlink(ctx, localFilenameTmp.c_str()); + return false; + } + + totalDownloaded += size_read; + if (progress_cbk && + !progress_cbk(ctx, double(totalDownloaded) / props.size, + user_data)) { + ctx->networking.close(ctx, handle, ctx->networking.user_data); + f.reset(); + NS_PROJ::FileManager::unlink(ctx, localFilenameTmp.c_str()); + return false; + } + } + + ctx->networking.close(ctx, handle, ctx->networking.user_data); + f.reset(); + NS_PROJ::FileManager::unlink(ctx, localFilename.c_str()); + if (!NS_PROJ::FileManager::rename(ctx, localFilenameTmp.c_str(), + localFilename.c_str())) { + pj_log(ctx, PJ_LOG_ERROR, "Cannot rename %s to %s", + localFilenameTmp.c_str(), localFilename.c_str()); + return false; + } + + auto diskCache = NS_PROJ::DiskChunkCache::open(ctx); + if (!diskCache) + return false; + auto stmt = + diskCache->prepare("SELECT lastChecked, fileSize, lastModified, etag " + "FROM downloaded_file_properties WHERE url = ?"); + if (!stmt) + return false; + stmt->bindText(url.c_str()); + + props.lastChecked = curTime; + auto hDB = diskCache->handle(); + + if (stmt->execute() == SQLITE_ROW) { + stmt = diskCache->prepare( + "UPDATE downloaded_file_properties SET lastChecked = ?, " + "fileSize = ?, lastModified = ?, etag = ? " + "WHERE url = ?"); + if (!stmt) + return false; + stmt->bindInt64(props.lastChecked); + stmt->bindInt64(props.size); + if (props.lastModified.empty()) + stmt->bindNull(); + else + stmt->bindText(props.lastModified.c_str()); + if (props.etag.empty()) + stmt->bindNull(); + else + stmt->bindText(props.etag.c_str()); + stmt->bindText(url.c_str()); + if (stmt->execute() != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return false; + } + } else { + stmt = diskCache->prepare( + "INSERT INTO downloaded_file_properties (url, lastChecked, " + "fileSize, lastModified, etag) VALUES " + "(?,?,?,?,?)"); + if (!stmt) + return false; + stmt->bindText(url.c_str()); + stmt->bindInt64(props.lastChecked); + stmt->bindInt64(props.size); + if (props.lastModified.empty()) + stmt->bindNull(); + else + stmt->bindText(props.lastModified.c_str()); + if (props.etag.empty()) + stmt->bindNull(); + else + stmt->bindText(props.etag.c_str()); + if (stmt->execute() != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return false; + } + } + return true; +} + +// --------------------------------------------------------------------------- + +//! @cond Doxygen_Suppress + +bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (ctx->networking.enabled_env_variable_checked) { + return ctx->networking.enabled; + } + const char *enabled = getenv("PROJ_NETWORK"); + if (enabled && enabled[0] != '\0') { + ctx->networking.enabled = ci_equal(enabled, "ON") || + ci_equal(enabled, "YES") || + ci_equal(enabled, "TRUE"); + } + pj_load_ini(ctx); + ctx->networking.enabled_env_variable_checked = true; + return ctx->networking.enabled; +} + +// --------------------------------------------------------------------------- + +std::string pj_context_get_grid_cache_filename(PJ_CONTEXT *ctx) { + pj_load_ini(ctx); + if (!ctx->gridChunkCache.filename.empty()) { + return ctx->gridChunkCache.filename; + } + const std::string path(pj_context_get_user_writable_directory(ctx, true)); + ctx->gridChunkCache.filename = path + "/cache.db"; + return ctx->gridChunkCache.filename; +} + +//! @endcond diff --git a/src/open_lib.cpp b/src/open_lib.cpp deleted file mode 100644 index 24c31033..00000000 --- a/src/open_lib.cpp +++ /dev/null @@ -1,359 +0,0 @@ -/****************************************************************************** - * Project: PROJ.4 - * Purpose: Implementation of pj_open_lib(), and pj_set_finder(). These - * provide a standard interface for opening projections support - * data files. - * Author: Gerald Evenden, Frank Warmerdam <warmerdam@pobox.com> - * - ****************************************************************************** - * Copyright (c) 1995, Gerald Evenden - * Copyright (c) 2002, Frank Warmerdam <warmerdam@pobox.com> - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - *****************************************************************************/ - -#define PJ_LIB__ - -#ifndef FROM_PROJ_CPP -#define FROM_PROJ_CPP -#endif - -#include <assert.h> -#include <errno.h> -#include <stddef.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include "proj/internal/internal.hpp" - -#include "proj_internal.h" - -static const char * proj_lib_name = -#ifdef PROJ_LIB -PROJ_LIB; -#else -nullptr; -#endif - -/************************************************************************/ -/* pj_set_finder() */ -/************************************************************************/ - -void pj_set_finder( const char *(*new_finder)(const char *) ) - -{ - auto ctx = pj_get_default_ctx(); - if( ctx ) { - ctx->file_finder_legacy = new_finder; - } -} - -/************************************************************************/ -/* proj_context_set_file_finder() */ -/************************************************************************/ - -/** \brief Assign a file finder callback to a context. - * - * This callback will be used whenever PROJ must open one of its resource files - * (proj.db database, grids, etc...) - * - * The callback will be called with the context currently in use at the moment - * where it is used (not necessarily the one provided during this call), and - * with the provided user_data (which may be NULL). - * The user_data must remain valid during the whole lifetime of the context. - * - * A finder set on the default context will be inherited by contexts created - * later. - * - * @param ctx PROJ context, or NULL for the default context. - * @param finder Finder callback. May be NULL - * @param user_data User data provided to the finder callback. May be NULL. - * - * @since PROJ 6.0 - */ -void proj_context_set_file_finder(PJ_CONTEXT *ctx, proj_file_finder finder, - void* user_data) -{ - if( !ctx ) - ctx = pj_get_default_ctx(); - if( !ctx ) - return; - ctx->file_finder = finder; - ctx->file_finder_user_data = user_data; -} - -/************************************************************************/ -/* proj_context_set_search_paths() */ -/************************************************************************/ - - -/** \brief Sets search paths. - * - * Those search paths will be used whenever PROJ must open one of its resource files - * (proj.db database, grids, etc...) - * - * If set on the default context, they will be inherited by contexts created - * later. - * - * @param ctx PROJ context, or NULL for the default context. - * @param count_paths Number of paths. 0 if paths == NULL. - * @param paths Paths. May be NULL. - * - * @since PROJ 6.0 - */ -void proj_context_set_search_paths(PJ_CONTEXT *ctx, - int count_paths, - const char* const* paths) -{ - if( !ctx ) - ctx = pj_get_default_ctx(); - if( !ctx ) - return; - try { - std::vector<std::string> vector_of_paths; - for (int i = 0; i < count_paths; i++) - { - vector_of_paths.emplace_back(paths[i]); - } - ctx->set_search_paths(vector_of_paths); - } catch( const std::exception& ) - { - } -} - -/************************************************************************/ -/* pj_set_searchpath() */ -/* */ -/* Path control for callers that can't practically provide */ -/* pj_set_finder() style callbacks. Call with (0,NULL) as args */ -/* to clear the searchpath set. */ -/************************************************************************/ - -void pj_set_searchpath ( int count, const char **path ) -{ - proj_context_set_search_paths( nullptr, count, const_cast<const char* const*>(path) ); -} - -#ifdef _WIN32 -#include <windows.h> -#include <sys/stat.h> -static const char *get_path_from_win32_projlib(const char *name, std::string& out) { - /* Check if proj.db lieves in a share/proj dir parallel to bin/proj.dll */ - /* Based in https://stackoverflow.com/questions/9112893/how-to-get-path-to-executable-in-c-running-on-windows */ - - DWORD path_size = 1024; - - for (;;) { - out.resize(path_size); - memset(&out[0], 0, path_size); - DWORD result = GetModuleFileNameA(nullptr, &out[0], path_size - 1); - DWORD last_error = GetLastError(); - - if (result == 0) { - return nullptr; - } - else if (result == path_size - 1) { - if (ERROR_INSUFFICIENT_BUFFER != last_error) { - return nullptr; - } - path_size = path_size * 2; - } - else { - break; - } - } - // Now remove the program's name. It was (example) "C:\programs\gmt6\bin\gdal_translate.exe" - size_t k = strlen(out.c_str()); - while (k > 0 && out[--k] != '\\') {} - out.resize(k); - - out += "/../share/proj/"; - out += name; - - struct stat fileInfo; - if (stat(out.c_str(), &fileInfo) == 0) // Check if file exists (probably there are simpler ways) - return out.c_str(); - else { - return nullptr; - } -} -#endif - -/************************************************************************/ -/* pj_open_lib_ex() */ -/************************************************************************/ - -static PAFile -pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, - char* out_full_filename, size_t out_full_filename_size) { - try { - std::string fname; - const char *sysname = nullptr; - PAFile fid = nullptr; -#ifdef WIN32 - static const char dir_chars[] = "/\\"; - const char dirSeparator = ';'; -#else - static const char dir_chars[] = "/"; - const char dirSeparator = ':'; -#endif - - if( ctx == nullptr ) { - ctx = pj_get_default_ctx(); - } - - if( out_full_filename != nullptr && out_full_filename_size > 0 ) - out_full_filename[0] = '\0'; - - /* check if ~/name */ - if (*name == '~' && strchr(dir_chars,name[1]) ) - if ((sysname = getenv("HOME")) != nullptr) { - fname = sysname; - fname += DIR_CHAR; - fname += name; - sysname = fname.c_str(); - } else - return nullptr; - - /* or fixed path: /name, ./name or ../name */ - else if (strchr(dir_chars,*name) - || (*name == '.' && strchr(dir_chars,name[1])) - || (!strncmp(name, "..", 2) && strchr(dir_chars,name[2])) - || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])) ) - sysname = name; - - /* or try to use application provided file finder */ - else if( ctx->file_finder != nullptr && (sysname = ctx->file_finder( ctx, name, ctx->file_finder_user_data )) != nullptr ) - ; - - else if( ctx->file_finder_legacy != nullptr && (sysname = ctx->file_finder_legacy( name )) != nullptr ) - ; - - /* The user has search paths set */ - else if( !ctx->search_paths.empty() ) { - for( const auto& path: ctx->search_paths ) { - try { - fname = path; - fname += DIR_CHAR; - fname += name; - sysname = fname.c_str(); - fid = pj_ctx_fopen(ctx, sysname, mode); - } catch( const std::exception& ) - { - } - if( fid ) - break; - } - } - /* if is environment PROJ_LIB defined */ - else if ((sysname = getenv("PROJ_LIB")) != nullptr) { - auto paths = NS_PROJ::internal::split(std::string(sysname), dirSeparator); - for( const auto& path: paths ) { - fname = path; - fname += DIR_CHAR; - fname += name; - sysname = fname.c_str(); - fid = pj_ctx_fopen(ctx, sysname, mode); - if( fid ) - break; - } -#ifdef _WIN32 - /* check if it lives in a ../share/proj dir of the proj dll */ - } else if ((sysname = get_path_from_win32_projlib(name, fname)) != nullptr) { -#endif - /* or hardcoded path */ - } else if ((sysname = proj_lib_name) != nullptr) { - fname = sysname; - fname += DIR_CHAR; - fname += name; - sysname = fname.c_str(); - /* just try it bare bones */ - } else { - sysname = name; - } - - assert(sysname); // to make Coverity Scan happy - if ( fid != nullptr || (fid = pj_ctx_fopen(ctx, sysname, mode)) != nullptr) - { - if( out_full_filename != nullptr && out_full_filename_size > 0 ) - { - // cppcheck-suppress nullPointer - strncpy(out_full_filename, sysname, out_full_filename_size); - out_full_filename[out_full_filename_size-1] = '\0'; - } - errno = 0; - } - - if( ctx->last_errno == 0 && errno != 0 ) - pj_ctx_set_errno( ctx, errno ); - - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "pj_open_lib(%s): call fopen(%s) - %s", - name, sysname, - fid == nullptr ? "failed" : "succeeded" ); - - return(fid); - } - catch( const std::exception& ) { - - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "pj_open_lib(%s): out of memory", - name ); - - return nullptr; - } -} - -/************************************************************************/ -/* pj_open_lib() */ -/************************************************************************/ - -PAFile -pj_open_lib(projCtx ctx, const char *name, const char *mode) { - return pj_open_lib_ex(ctx, name, mode, nullptr, 0); -} - -/************************************************************************/ -/* pj_find_file() */ -/************************************************************************/ - -/** Returns the full filename corresponding to a proj resource file specified - * as a short filename. - * - * @param ctx context. - * @param short_filename short filename (e.g. egm96_15.gtx). Must not be NULL. - * @param out_full_filename output buffer, of size out_full_filename_size, that - * will receive the full filename on success. - * Will be zero-terminated. - * @param out_full_filename_size size of out_full_filename. - * @return 1 if the file was found, 0 otherwise. - */ -int pj_find_file(projCtx ctx, const char *short_filename, - char* out_full_filename, size_t out_full_filename_size) -{ - PAFile f = pj_open_lib_ex(ctx, short_filename, "rb", out_full_filename, - out_full_filename_size); - if( f != nullptr ) - { - pj_ctx_fclose(ctx, f); - return 1; - } - return 0; -} diff --git a/src/pipeline.cpp b/src/pipeline.cpp index 96767143..511a69fe 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -155,7 +155,7 @@ static PJ_LPZ pipeline_reverse_3d (PJ_XYZ xyz, PJ *P); static PJ_XY pipeline_forward (PJ_LP lp, PJ *P); static PJ_LP pipeline_reverse (PJ_XY xy, PJ *P); -void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ) +static void pipeline_reassign_context( PJ* P, PJ_CONTEXT* ctx ) { auto pipeline = static_cast<struct Pipeline*>(P->opaque); for( auto& step: pipeline->steps ) @@ -170,6 +170,9 @@ static PJ_COORD pipeline_forward_4d (PJ_COORD point, PJ *P) { if( !step.omit_fwd ) { point = proj_trans (step.pj, PJ_FWD, point); + if( point.xyzt.x == HUGE_VAL ) { + break; + } } } @@ -186,6 +189,9 @@ static PJ_COORD pipeline_reverse_4d (PJ_COORD point, PJ *P) { if( !step.omit_inv ) { point = proj_trans (step.pj, PJ_INV, point); + if( point.xyzt.x == HUGE_VAL ) { + break; + } } } @@ -204,6 +210,9 @@ static PJ_XYZ pipeline_forward_3d (PJ_LPZ lpz, PJ *P) { if( !step.omit_fwd ) { point = pj_approx_3D_trans (step.pj, PJ_FWD, point); + if( point.xyzt.x == HUGE_VAL ) { + break; + } } } @@ -222,6 +231,9 @@ static PJ_LPZ pipeline_reverse_3d (PJ_XYZ xyz, PJ *P) { if( !step.omit_inv ) { point = proj_trans (step.pj, PJ_INV, point); + if( point.xyzt.x == HUGE_VAL ) { + break; + } } } @@ -240,6 +252,9 @@ static PJ_XY pipeline_forward (PJ_LP lp, PJ *P) { if( !step.omit_fwd ) { point = pj_approx_2D_trans (step.pj, PJ_FWD, point); + if( point.xyzt.x == HUGE_VAL ) { + break; + } } } @@ -258,6 +273,9 @@ static PJ_LP pipeline_reverse (PJ_XY xy, PJ *P) { if( !step.omit_inv ) { point = pj_approx_2D_trans (step.pj, PJ_INV, point); + if( point.xyzt.x == HUGE_VAL ) { + break; + } } } @@ -413,7 +431,7 @@ PJ *OPERATION(pipeline,0) { P->fwd = pipeline_forward; P->inv = pipeline_reverse; P->destructor = destructor; - P->is_pipeline = 1; + P->reassign_context = pipeline_reassign_context; /* Currently, the pipeline driver is a raw bit mover, enabling other operations */ /* to collaborate efficiently. All prep/fin stuff is done at the step levels. */ diff --git a/src/pj_list.h b/src/pj_list.h index 0923bba8..9798a36b 100644 --- a/src/pj_list.h +++ b/src/pj_list.h @@ -170,3 +170,4 @@ PROJ_HEAD(weren, "Werenskiold I") PROJ_HEAD(wink1, "Winkel I") PROJ_HEAD(wink2, "Winkel II") PROJ_HEAD(wintri, "Winkel Tripel") +PROJ_HEAD(xyzgridshift, "XYZ grid shift") @@ -353,6 +353,166 @@ void PROJ_DLL proj_context_set_search_paths(PJ_CONTEXT *ctx, int count_paths, co void PROJ_DLL proj_context_use_proj4_init_rules(PJ_CONTEXT *ctx, int enable); int PROJ_DLL proj_context_get_use_proj4_init_rules(PJ_CONTEXT *ctx, int from_legacy_code_path); +/*! @endcond */ + +/** Opaque structure for PROJ for a file handle. Implementations might cast it to their + * structure/class of choice. */ +typedef struct PROJ_FILE_HANDLE PROJ_FILE_HANDLE; + +/** Open access / mode */ +typedef enum PROJ_OPEN_ACCESS +{ + /** Read-only access. Equivalent to "rb" */ + PROJ_OPEN_ACCESS_READ_ONLY, + + /** Read-update access. File should be created if not existing. Equivalent to "r+b" */ + PROJ_OPEN_ACCESS_READ_UPDATE, + + /** Create access. File should be truncated to 0-byte if already existing. Equivalent to "w+b" */ + PROJ_OPEN_ACCESS_CREATE +} PROJ_OPEN_ACCESS; + +/** File API callbacks */ +typedef struct PROJ_FILE_API +{ + /** Version of this structure. Should be set to 1 currently. */ + int version; + + /** Open file. Return NULL if error */ + PROJ_FILE_HANDLE* (*open_cbk)(PJ_CONTEXT *ctx, const char *filename, PROJ_OPEN_ACCESS access, void* user_data); + + /** Read sizeBytes into buffer from current position and return number of bytes read */ + size_t (*read_cbk)(PJ_CONTEXT *ctx, PROJ_FILE_HANDLE*, void* buffer, size_t sizeBytes, void* user_data); + + /** Write sizeBytes into buffer from current position and return number of bytes written */ + size_t (*write_cbk)(PJ_CONTEXT *ctx, PROJ_FILE_HANDLE*, const void* buffer, size_t sizeBytes, void* user_data); + + /** Seek to offset using whence=SEEK_SET/SEEK_CUR/SEEK_END. Return TRUE in case of success */ + int (*seek_cbk)(PJ_CONTEXT *ctx, PROJ_FILE_HANDLE*, long long offset, int whence, void* user_data); + + /** Return current file position */ + unsigned long long (*tell_cbk)(PJ_CONTEXT *ctx, PROJ_FILE_HANDLE*, void* user_data); + + /** Close file */ + void (*close_cbk)(PJ_CONTEXT *ctx, PROJ_FILE_HANDLE*, void* user_data); + + /** Return TRUE if a file exists */ + int (*exists_cbk)(PJ_CONTEXT *ctx, const char *filename, void* user_data); + + /** Return TRUE if directory exists or could be created */ + int (*mkdir_cbk)(PJ_CONTEXT *ctx, const char *filename, void* user_data); + + /** Return TRUE if file could be removed */ + int (*unlink_cbk)(PJ_CONTEXT *ctx, const char *filename, void* user_data); + + /** Return TRUE if file could be renamed */ + int (*rename_cbk)(PJ_CONTEXT *ctx, const char *oldPath, const char *newPath, void* user_data); +} PROJ_FILE_API; + +int PROJ_DLL proj_context_set_fileapi( + PJ_CONTEXT* ctx, const PROJ_FILE_API* fileapi, void* user_data); + +void PROJ_DLL proj_context_set_sqlite3_vfs_name(PJ_CONTEXT* ctx, const char* name); + +/** Opaque structure for PROJ for a network handle. Implementations might cast it to their + * structure/class of choice. */ +typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; + +/** Network access: open callback + * + * Should try to read the size_to_read first bytes at the specified offset of + * the file given by URL url, + * and write them to buffer. *out_size_read should be updated with the actual + * amount of bytes read (== size_to_read if the file is larger than size_to_read). + * During this read, the implementation should make sure to store the HTTP + * headers from the server response to be able to respond to + * proj_network_get_header_value_cbk_type callback. + * + * error_string_max_size should be the maximum size that can be written into + * the out_error_string buffer (including terminating nul character). + * + * @return a non-NULL opaque handle in case of success. + */ +typedef PROJ_NETWORK_HANDLE* (*proj_network_open_cbk_type)( + PJ_CONTEXT* ctx, + const char* url, + unsigned long long offset, + size_t size_to_read, + void* buffer, + size_t* out_size_read, + size_t error_string_max_size, + char* out_error_string, + void* user_data); + +/** Network access: close callback */ +typedef void (*proj_network_close_cbk_type)(PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + void* user_data); + +/** Network access: get HTTP headers */ +typedef const char* (*proj_network_get_header_value_cbk_type)( + PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + const char* header_name, + void* user_data); + +/** Network access: read range + * + * Read size_to_read bytes from handle, starting at offset, into + * buffer. + * During this read, the implementation should make sure to store the HTTP + * headers from the server response to be able to respond to + * proj_network_get_header_value_cbk_type callback. + * + * error_string_max_size should be the maximum size that can be written into + * the out_error_string buffer (including terminating nul character). + * + * @return the number of bytes actually read (0 in case of error) + */ +typedef size_t (*proj_network_read_range_type)( + PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + unsigned long long offset, + size_t size_to_read, + void* buffer, + size_t error_string_max_size, + char* out_error_string, + void* user_data); + +int PROJ_DLL proj_context_set_network_callbacks( + PJ_CONTEXT* ctx, + proj_network_open_cbk_type open_cbk, + proj_network_close_cbk_type close_cbk, + proj_network_get_header_value_cbk_type get_header_value_cbk, + proj_network_read_range_type read_range_cbk, + void* user_data); + +int PROJ_DLL proj_context_set_enable_network(PJ_CONTEXT* ctx, + int enabled); + +void PROJ_DLL proj_context_set_url_endpoint(PJ_CONTEXT* ctx, const char* url); + +void PROJ_DLL proj_grid_cache_set_enable(PJ_CONTEXT* ctx, int enabled); + +void PROJ_DLL proj_grid_cache_set_filename(PJ_CONTEXT* ctx, const char* fullname); + +void PROJ_DLL proj_grid_cache_set_max_size(PJ_CONTEXT* ctx, int max_size_MB); + +void PROJ_DLL proj_grid_cache_set_ttl(PJ_CONTEXT* ctx, int ttl_seconds); + +void PROJ_DLL proj_grid_cache_clear(PJ_CONTEXT* ctx); + +int PROJ_DLL proj_is_download_needed(PJ_CONTEXT* ctx, + const char* url_or_filename, + int ignore_ttl_setting); +int PROJ_DLL proj_download_file(PJ_CONTEXT *ctx, const char *url_or_filename, + int ignore_ttl_setting, + int (*progress_cbk)(PJ_CONTEXT *, double pct, + void *user_data), + void *user_data); + +/*! @cond Doxygen_Suppress */ + /* Manage the transformation definition object PJ */ PJ PROJ_DLL *proj_create (PJ_CONTEXT *ctx, const char *definition); PJ PROJ_DLL *proj_create_argv (PJ_CONTEXT *ctx, int argc, char **argv); @@ -624,6 +784,12 @@ typedef enum { /** Ignore grid availability at all. Results will be presented as if * all grids were available. */ PROJ_GRID_AVAILABILITY_IGNORED, + + /** Results will be presented as if grids known to PROJ (that is + * registered in the grid_alternatives table of its database) were + * available. Used typically when networking is enabled. + */ + PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE, } PROJ_GRID_AVAILABILITY_USE; /** \brief PROJ string version. */ diff --git a/src/proj_internal.h b/src/proj_internal.h index 3e219682..a587c037 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -195,14 +195,6 @@ PJ_COORD pj_inv4d (PJ_COORD coo, PJ *P); PJ_COORD PROJ_DLL pj_approx_2D_trans (PJ *P, PJ_DIRECTION direction, PJ_COORD coo); PJ_COORD PROJ_DLL pj_approx_3D_trans (PJ *P, PJ_DIRECTION direction, PJ_COORD coo); - -/* Grid functionality */ -int proj_vgrid_init(PJ *P, const char *grids); -int proj_hgrid_init(PJ *P, const char *grids); -double proj_vgrid_value(PJ *P, PJ_LP lp, double vmultiplier); -PJ_LP proj_hgrid_value(PJ *P, PJ_LP lp); -PJ_LP proj_hgrid_apply(PJ *P, PJ_LP lp, PJ_DIRECTION direction); - void PROJ_DLL proj_log_error (PJ *P, const char *fmt, ...); void proj_log_debug (PJ *P, const char *fmt, ...); void proj_log_trace (PJ *P, const char *fmt, ...); @@ -222,9 +214,6 @@ size_t pj_trim_argc (char *args); char **pj_trim_argv (size_t argc, char *args); char *pj_make_args (size_t argc, char **argv); -/* Lowest level: Minimum support for fileapi */ -void proj_fileapi_set (PJ *P, void *fileapi); - typedef struct { double r, i; } COMPLEX; /* Forward declarations and typedefs for stuff needed inside the PJ object */ @@ -354,6 +343,7 @@ struct PJconsts { PJ_OPERATOR inv4d = nullptr; PJ_DESTRUCTOR destructor = nullptr; + void (*reassign_context)(PJ*, projCtx_t *) = nullptr; /************************************************************************************* @@ -421,7 +411,6 @@ struct PJconsts { int geoc = 0; /* Geocentric latitude flag */ int is_latlong = 0; /* proj=latlong ... not really a projection at all */ int is_geocent = 0; /* proj=geocent ... not really a projection at all */ - int is_pipeline = 0; /* 1 if PJ represents a pipeline */ int need_ellps = 0; /* 0 for operations that are purely cartesian */ int skip_fwd_prepare = 0; int skip_fwd_finalize = 0; @@ -478,32 +467,16 @@ struct PJconsts { int datum_type = PJD_UNKNOWN; /* PJD_UNKNOWN/3PARAM/7PARAM/GRIDSHIFT/WGS84 */ double datum_params[7] = {0,0,0,0,0,0,0}; /* Parameters for 3PARAM and 7PARAM */ - struct _pj_gi **gridlist = nullptr; /* TODO: Description needed */ - int gridlist_count = 0; - int has_geoid_vgrids = 0; /* TODO: Description needed */ - struct _pj_gi **vgridlist_geoid = nullptr; /* TODO: Description needed */ - int vgridlist_geoid_count = 0; + int has_geoid_vgrids = 0; /* used by legacy transform.cpp */ + void* hgrids_legacy = nullptr; /* used by legacy transform.cpp. Is a pointer to a ListOfHGrids* */ + void* vgrids_legacy = nullptr; /* used by legacy transform.cpp. Is a pointer to a ListOfVGrids* */ double from_greenwich = 0.0; /* prime meridian offset (in radians) */ double long_wrap_center = 0.0; /* 0.0 for -180 to 180, actually in radians*/ int is_long_wrap_set = 0; char axis[4] = {0,0,0,0}; /* Axis order, pj_transform/pj_adjust_axis */ - /* New Datum Shift Grid Catalogs */ - char *catalog_name = nullptr; - struct _PJ_GridCatalog *catalog = nullptr; - - double datum_date = 0.0; /* TODO: Description needed */ - - struct _pj_gi *last_before_grid = nullptr; /* TODO: Description needed */ - PJ_Region last_before_region = {0,0,0,0}; /* TODO: Description needed */ - double last_before_date = 0.0; /* TODO: Description needed */ - - struct _pj_gi *last_after_grid = nullptr; /* TODO: Description needed */ - PJ_Region last_after_region = {0,0,0,0}; /* TODO: Description needed */ - double last_after_date = 0.0; /* TODO: Description needed */ - /************************************************************************************* ISO-19111 interface **************************************************************************************/ @@ -685,24 +658,63 @@ struct FACTORS { #define PJD_ERR_INCONSISTENT_UNIT -59 #define PJD_ERR_MUTUALLY_EXCLUSIVE_ARGS -60 #define PJD_ERR_GENERIC_ERROR -61 +#define PJD_ERR_NETWORK_ERROR -62 /* NOTE: Remember to update src/strerrno.cpp, src/apps/gie.cpp and transient_error in */ /* src/transform.cpp when adding new value */ +// Legacy struct projFileAPI_t; struct projCppContext; +struct projNetworkCallbacksAndData +{ + bool enabled = false; + bool enabled_env_variable_checked = false; // whereas we have checked PROJ_NETWORK env variable + proj_network_open_cbk_type open = nullptr; + proj_network_close_cbk_type close = nullptr; + proj_network_get_header_value_cbk_type get_header_value = nullptr; + proj_network_read_range_type read_range = nullptr; + void* user_data = nullptr; +}; + +struct projGridChunkCache +{ + bool enabled = true; + std::string filename{}; + long long max_size = 300 * 1024 * 1024; + int ttl = 86400; // 1 day +}; + +struct projFileApiCallbackAndData +{ + PROJ_FILE_HANDLE* (*open_cbk)(PJ_CONTEXT *ctx, const char *filename, PROJ_OPEN_ACCESS access, void* user_data) = nullptr; + size_t (*read_cbk)(PJ_CONTEXT *ctx, PROJ_FILE_HANDLE*, void* buffer, size_t size, void* user_data) = nullptr; + size_t (*write_cbk)(PJ_CONTEXT *ctx, PROJ_FILE_HANDLE*, const void* buffer, size_t size, void* user_data) = nullptr; + int (*seek_cbk)(PJ_CONTEXT *ctx, PROJ_FILE_HANDLE*, long long offset, int whence, void* user_data) = nullptr; + unsigned long long (*tell_cbk)(PJ_CONTEXT *ctx, PROJ_FILE_HANDLE*, void* user_data) = nullptr; + void (*close_cbk)(PJ_CONTEXT *ctx, PROJ_FILE_HANDLE*, void* user_data) = nullptr; + + int (*exists_cbk)(PJ_CONTEXT *ctx, const char *filename, void* user_data) = nullptr; + int (*mkdir_cbk)(PJ_CONTEXT *ctx, const char *filename, void* user_data) = nullptr; + int (*unlink_cbk)(PJ_CONTEXT *ctx, const char *filename, void* user_data) = nullptr; + int (*rename_cbk)(PJ_CONTEXT *ctx, const char *oldPath, const char *newPath, void* user_data) = nullptr; + + void* user_data = nullptr; +}; + /* proj thread context */ struct projCtx_t { int last_errno = 0; int debug_level = 0; void (*logger)(void *, int, const char *) = nullptr; void *logger_app_data = nullptr; - struct projFileAPI_t *fileapi = nullptr; + struct projFileAPI_t *fileapi_legacy = nullptr; // for proj_api.h legacy API struct projCppContext* cpp_context = nullptr; /* internal context for C++ code */ int use_proj4_init_rules = -1; /* -1 = unknown, 0 = no, 1 = yes */ int epsg_file_exists = -1; /* -1 = unknown, 0 = no, 1 = yes */ + std::string env_var_proj_lib{}; // content of PROJ_LIB environment variable. Use Filemanager::getProjLibEnvVar() to access std::vector<std::string> search_paths{}; const char **c_compat_paths = nullptr; // same, but for projinfo usage @@ -710,6 +722,18 @@ struct projCtx_t { const char* (*file_finder) (PJ_CONTEXT *, const char*, void* user_data) = nullptr; void* file_finder_user_data = nullptr; + projNetworkCallbacksAndData networking{}; + bool defer_grid_opening = false; // set by pj_obj_create() + + projFileApiCallbackAndData fileApi{}; + std::string custom_sqlite3_vfs_name{}; + + bool iniFileLoaded = false; + std::string endpoint{}; + + std::string user_writable_directory{}; + projGridChunkCache gridChunkCache{}; + int projStringParserCreateFromPROJStringRecursionCounter = 0; // to avoid potential infinite recursion in PROJStringParser::createFromPROJString() projCtx_t() = default; @@ -766,56 +790,6 @@ PJ *pj_projection_specific_setup_##name (PJ *P) #endif /* def PJ_LIB__ */ - -#define MAX_TAB_ID 80 -typedef struct { float lam, phi; } FLP; -typedef struct { pj_int32 lam, phi; } ILP; - -struct CTABLE { - char id[MAX_TAB_ID]; /* ascii info */ - PJ_LP ll; /* lower left corner coordinates */ - PJ_LP del; /* size of cells */ - ILP lim; /* limits of conversion matrix */ - FLP *cvs; /* conversion matrix */ -}; - -typedef struct _pj_gi { - char *gridname; /* identifying name of grid, eg "conus" or ntv2_0.gsb */ - char *filename; /* full path to filename */ - - const char *format; /* format of this grid, ie "ctable", "ntv1", - "ntv2" or "missing". */ - - long grid_offset; /* offset in file, for delayed loading */ - int must_swap; /* only for NTv2 */ - - struct CTABLE *ct; - - struct _pj_gi *next; - struct _pj_gi *child; -} PJ_GRIDINFO; - -typedef struct { - PJ_Region region; - int priority; /* higher used before lower */ - double date; /* year.fraction */ - char *definition; /* usually the gridname */ - - PJ_GRIDINFO *gridinfo; - int available; /* 0=unknown, 1=true, -1=false */ -} PJ_GridCatalogEntry; - -typedef struct _PJ_GridCatalog { - char *catalog_name; - - PJ_Region region; /* maximum extent of catalog data */ - - int entry_count; - PJ_GridCatalogEntry *entries; - - struct _PJ_GridCatalog *next; -} PJ_GridCatalog; - /* procedure prototypes */ double PROJ_DLL dmstor(const char *, char **); double dmstor_ctx(projCtx_t *ctx, const char *, char **); @@ -862,52 +836,6 @@ COMPLEX pj_zpolyd1(COMPLEX, const COMPLEX *, int, COMPLEX *); int pj_deriv(PJ_LP, double, const PJ *, struct DERIVS *); int pj_factors(PJ_LP, const PJ *, double, struct FACTORS *); -/* nadcon related protos */ -struct CTABLE* find_ctable(projCtx_t *ctx, PJ_LP input, int grid_count, PJ_GRIDINFO **tables); - -PJ_LP nad_intr(PJ_LP, struct CTABLE *); -PJ_LP nad_cvt(projCtx_t *ctx, PJ_LP in, int inverse, struct CTABLE *ct, int grid_count, PJ_GRIDINFO **tables); -struct CTABLE *nad_init(projCtx_t *ctx, char *); -struct CTABLE *nad_ctable_init( projCtx_t *ctx, struct projFileAPI_t* fid ); -int nad_ctable_load( projCtx_t *ctx, struct CTABLE *, struct projFileAPI_t* fid ); -struct CTABLE *nad_ctable2_init( projCtx_t *ctx, struct projFileAPI_t* fid ); -int nad_ctable2_load( projCtx_t *ctx, struct CTABLE *, struct projFileAPI_t* fid ); -void nad_free(struct CTABLE *); - -/* higher level handling of datum grid shift files */ - -int pj_apply_vgridshift( PJ *defn, const char *listname, - PJ_GRIDINFO ***gridlist_p, - int *gridlist_count_p, - int inverse, - long point_count, int point_offset, - double *x, double *y, double *z ); -int pj_apply_gridshift_2( PJ *defn, int inverse, - long point_count, int point_offset, - double *x, double *y, double *z ); -int pj_apply_gridshift_3( projCtx_t *ctx, - PJ_GRIDINFO **gridlist, int gridlist_count, - int inverse, long point_count, int point_offset, - double *x, double *y, double *z ); - -PJ_GRIDINFO **pj_gridlist_from_nadgrids( projCtx_t *, const char *, int * ); - -PJ_GRIDINFO *pj_gridinfo_init( projCtx_t *, const char * ); -int pj_gridinfo_load( projCtx_t *, PJ_GRIDINFO * ); -void pj_gridinfo_free( projCtx_t *, PJ_GRIDINFO * ); - -PJ_GridCatalog *pj_gc_findcatalog( projCtx_t *, const char * ); -PJ_GridCatalog *pj_gc_readcatalog( projCtx_t *, const char * ); -void pj_gc_unloadall( projCtx_t *); -int pj_gc_apply_gridshift( PJ *defn, int inverse, - long point_count, int point_offset, - double *x, double *y, double *z ); -int pj_gc_apply_gridshift( PJ *defn, int inverse, - long point_count, int point_offset, - double *x, double *y, double *z ); - -double pj_gc_parsedate( projCtx_t *, const char * ); - void *proj_mdist_ini(double); double proj_mdist(double, double, double, const void *); double proj_inv_mdist(projCtx_t *ctx, double, const void *); @@ -932,7 +860,16 @@ std::string pj_double_quote_string_param_if_needed(const std::string& str); PJ *pj_create_internal (PJ_CONTEXT *ctx, const char *definition); PJ *pj_create_argv_internal (PJ_CONTEXT *ctx, int argc, char **argv); -void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ); +// For use by projinfo +bool PROJ_DLL pj_context_is_network_enabled(PJ_CONTEXT* ctx); + +std::string pj_context_get_url_endpoint(PJ_CONTEXT* ctx); + +void pj_load_ini(PJ_CONTEXT* ctx); + +// Exported for testing purposes only +std::string PROJ_DLL pj_context_get_grid_cache_filename(PJ_CONTEXT *ctx); +std::string PROJ_DLL pj_context_get_user_writable_directory(PJ_CONTEXT *ctx, bool create); /* classic public API */ #include "proj_api.h" diff --git a/src/sqlite3_utils.cpp b/src/sqlite3_utils.cpp new file mode 100644 index 00000000..673eb89c --- /dev/null +++ b/src/sqlite3_utils.cpp @@ -0,0 +1,194 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: SQLite3 related utilities + * Author: Even Rouault, <even.rouault at spatialys.com> + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Weffc++" +#endif + +#include "sqlite3_utils.hpp" + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +#include <cstdlib> +#include <cstring> +#include <sstream> // std::ostringstream + +NS_PROJ_START + +// --------------------------------------------------------------------------- + +SQLite3VFS::SQLite3VFS(sqlite3_vfs *vfs) : vfs_(vfs) {} + +// --------------------------------------------------------------------------- + +SQLite3VFS::~SQLite3VFS() { + if (vfs_) { + sqlite3_vfs_unregister(vfs_); + delete vfs_; + } +} + +// --------------------------------------------------------------------------- + +struct pj_sqlite3_vfs : public sqlite3_vfs { + std::string namePtr{}; + bool fakeSync = false; + bool fakeLock = false; +}; + +// --------------------------------------------------------------------------- + +const char *SQLite3VFS::name() const { + return static_cast<pj_sqlite3_vfs *>(vfs_)->namePtr.c_str(); +} + +// --------------------------------------------------------------------------- + +typedef int (*ClosePtr)(sqlite3_file *); + +// --------------------------------------------------------------------------- + +static int VFSClose(sqlite3_file *file) { + sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr); + assert(defaultVFS); + ClosePtr defaultClosePtr; + std::memcpy(&defaultClosePtr, + reinterpret_cast<char *>(file) + defaultVFS->szOsFile, + sizeof(ClosePtr)); + void *methods = const_cast<sqlite3_io_methods *>(file->pMethods); + int ret = defaultClosePtr(file); + std::free(methods); + return ret; +} + +// --------------------------------------------------------------------------- + +static int VSFNoOpLockUnlockSync(sqlite3_file *, int) { return SQLITE_OK; } + +// --------------------------------------------------------------------------- + +static int VFSCustomOpen(sqlite3_vfs *vfs, const char *name, sqlite3_file *file, + int flags, int *outFlags) { + auto realVFS = static_cast<pj_sqlite3_vfs *>(vfs); + sqlite3_vfs *defaultVFS = static_cast<sqlite3_vfs *>(vfs->pAppData); + int ret = defaultVFS->xOpen(defaultVFS, name, file, flags, outFlags); + if (ret == SQLITE_OK) { + ClosePtr defaultClosePtr = file->pMethods->xClose; + assert(defaultClosePtr); + sqlite3_io_methods *methods = static_cast<sqlite3_io_methods *>( + std::malloc(sizeof(sqlite3_io_methods))); + if (!methods) { + file->pMethods->xClose(file); + return SQLITE_NOMEM; + } + memcpy(methods, file->pMethods, sizeof(sqlite3_io_methods)); + methods->xClose = VFSClose; + if (realVFS->fakeSync) { + // Disable xSync because it can be significantly slow and we don't + // need + // that level of data integrity garanty for the cache. + methods->xSync = VSFNoOpLockUnlockSync; + } + if (realVFS->fakeLock) { + methods->xLock = VSFNoOpLockUnlockSync; + methods->xUnlock = VSFNoOpLockUnlockSync; + } + file->pMethods = methods; + // Save original xClose pointer at end of file structure + std::memcpy(reinterpret_cast<char *>(file) + defaultVFS->szOsFile, + &defaultClosePtr, sizeof(ClosePtr)); + } + return ret; +} + +// --------------------------------------------------------------------------- + +static int VFSCustomAccess(sqlite3_vfs *vfs, const char *zName, int flags, + int *pResOut) { + sqlite3_vfs *defaultVFS = static_cast<sqlite3_vfs *>(vfs->pAppData); + // Do not bother stat'ing for journal or wal files + if (std::strstr(zName, "-journal") || std::strstr(zName, "-wal")) { + *pResOut = false; + return SQLITE_OK; + } + return defaultVFS->xAccess(defaultVFS, zName, flags, pResOut); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<SQLite3VFS> SQLite3VFS::create(bool fakeSync, bool fakeLock, + bool skipStatJournalAndWAL) { + sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr); + assert(defaultVFS); + + auto vfs = new pj_sqlite3_vfs(); + vfs->fakeSync = fakeSync; + vfs->fakeLock = fakeLock; + + auto vfsUnique = std::unique_ptr<SQLite3VFS>(new SQLite3VFS(vfs)); + + std::ostringstream buffer; + buffer << vfs; + vfs->namePtr = buffer.str(); + + vfs->iVersion = 1; + vfs->szOsFile = defaultVFS->szOsFile + sizeof(ClosePtr); + vfs->mxPathname = defaultVFS->mxPathname; + vfs->zName = vfs->namePtr.c_str(); + vfs->pAppData = defaultVFS; + vfs->xOpen = VFSCustomOpen; + vfs->xDelete = defaultVFS->xDelete; + vfs->xAccess = + skipStatJournalAndWAL ? VFSCustomAccess : defaultVFS->xAccess; + vfs->xFullPathname = defaultVFS->xFullPathname; + vfs->xDlOpen = defaultVFS->xDlOpen; + vfs->xDlError = defaultVFS->xDlError; + vfs->xDlSym = defaultVFS->xDlSym; + vfs->xDlClose = defaultVFS->xDlClose; + vfs->xRandomness = defaultVFS->xRandomness; + vfs->xSleep = defaultVFS->xSleep; + vfs->xCurrentTime = defaultVFS->xCurrentTime; + vfs->xGetLastError = defaultVFS->xGetLastError; + vfs->xCurrentTimeInt64 = defaultVFS->xCurrentTimeInt64; + if (sqlite3_vfs_register(vfs, false) == SQLITE_OK) { + return vfsUnique; + } + delete vfsUnique->vfs_; + vfsUnique->vfs_ = nullptr; + return nullptr; +} + +// --------------------------------------------------------------------------- + +SQLiteStatement::SQLiteStatement(sqlite3_stmt *hStmtIn) : hStmt(hStmtIn) {} + +// --------------------------------------------------------------------------- + +NS_PROJ_END diff --git a/src/sqlite3_utils.hpp b/src/sqlite3_utils.hpp new file mode 100644 index 00000000..ef141d1f --- /dev/null +++ b/src/sqlite3_utils.hpp @@ -0,0 +1,127 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: SQLite3 related utilities + * Author: Even Rouault, <even.rouault at spatialys.com> + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef SQLITE3_HPP_INCLUDED +#define SQLITE3_HPP_INCLUDED + +#include <memory> + +#include <sqlite3.h> + +#include "proj.h" +#include "proj/util.hpp" + +NS_PROJ_START + +//! @cond Doxygen_Suppress + +class SQLite3VFS { + sqlite3_vfs *vfs_ = nullptr; + + explicit SQLite3VFS(sqlite3_vfs *vfs); + + SQLite3VFS(const SQLite3VFS &) = delete; + SQLite3VFS &operator=(const SQLite3VFS &) = delete; + + public: + ~SQLite3VFS(); + + static std::unique_ptr<SQLite3VFS> create(bool fakeSync, bool fakeLock, + bool skipStatJournalAndWAL); + const char *name() const; + sqlite3_vfs *raw() { return vfs_; } +}; + +// --------------------------------------------------------------------------- + +class SQLiteStatement { + sqlite3_stmt *hStmt = nullptr; + int iBindIdx = 1; + int iResIdx = 0; + SQLiteStatement(const SQLiteStatement &) = delete; + SQLiteStatement &operator=(const SQLiteStatement &) = delete; + + public: + explicit SQLiteStatement(sqlite3_stmt *hStmtIn); + ~SQLiteStatement() { sqlite3_finalize(hStmt); } + + int execute() { return sqlite3_step(hStmt); } + + void bindNull() { + sqlite3_bind_null(hStmt, iBindIdx); + iBindIdx++; + } + + void bindText(const char *txt) { + sqlite3_bind_text(hStmt, iBindIdx, txt, -1, nullptr); + iBindIdx++; + } + + void bindInt64(sqlite3_int64 v) { + sqlite3_bind_int64(hStmt, iBindIdx, v); + iBindIdx++; + } + + void bindBlob(const void *blob, size_t blob_size) { + sqlite3_bind_blob(hStmt, iBindIdx, blob, static_cast<int>(blob_size), + nullptr); + iBindIdx++; + } + + const char *getText() { + auto ret = sqlite3_column_text(hStmt, iResIdx); + iResIdx++; + return reinterpret_cast<const char *>(ret); + } + + sqlite3_int64 getInt64() { + auto ret = sqlite3_column_int64(hStmt, iResIdx); + iResIdx++; + return ret; + } + + const void *getBlob(int &size) { + size = sqlite3_column_bytes(hStmt, iResIdx); + auto ret = sqlite3_column_blob(hStmt, iResIdx); + iResIdx++; + return ret; + } + + void reset() { + sqlite3_reset(hStmt); + iBindIdx = 1; + iResIdx = 0; + } + + void resetResIndex() { iResIdx = 0; } +}; + +//! @endcond Doxygen_Suppress + +NS_PROJ_END + +#endif // SQLITE3_HPP_INCLUDED diff --git a/src/strerrno.cpp b/src/strerrno.cpp index 9bf5f45a..5ae0d7e1 100644 --- a/src/strerrno.cpp +++ b/src/strerrno.cpp @@ -71,6 +71,7 @@ pj_err_list[] = { "inconsistent unit type between input and output", /* -59 */ "arguments are mutually exclusive", /* -60 */ "generic error of unknown origin", /* -61 */ + "network error", /* -62 */ /* When adding error messages, remember to update ID defines in projects.h, and transient_error array in pj_transform */ diff --git a/src/transform.cpp b/src/transform.cpp index 781c0061..ea3d9ae2 100644 --- a/src/transform.cpp +++ b/src/transform.cpp @@ -34,6 +34,9 @@ #include "proj.h" #include "proj_internal.h" #include "geocent.h" +#include "grids.hpp" + +using namespace NS_PROJ; static int adjust_axis( projCtx ctx, const char *axis, int denormalize_flag, long point_count, int point_offset, @@ -447,6 +450,78 @@ static int height_unit (PJ *P, PJ_DIRECTION dir, long n, int dist, double *z) { } +/************************************************************************/ +/* pj_apply_vgridshift() */ +/* */ +/* This implementation takes uses the gridlist from a coordinate */ +/* system definition. If the gridlist has not yet been */ +/* populated in the coordinate system definition we set it up */ +/* now. */ +/************************************************************************/ +static int pj_apply_vgridshift( PJ *defn, + int inverse, + long point_count, int point_offset, + double *x, double *y, double *z ) + +{ + if( defn->vgrids_legacy == nullptr ) + { + defn->vgrids_legacy = new ListOfVGrids; + auto vgrids = pj_vgrid_init(defn, "geoidgrids"); + if( vgrids.empty() ) + return 0; + *static_cast<ListOfVGrids*>(defn->vgrids_legacy) = std::move(vgrids); + } + if( static_cast<ListOfVGrids*>(defn->vgrids_legacy)->empty() ) + { + return 0; + } + + for( int i = 0; i < point_count; i++ ) + { + double value; + long io = i * point_offset; + PJ_LP input; + + input.phi = y[io]; + input.lam = x[io]; + + value = pj_vgrid_value(defn, *static_cast<ListOfVGrids*>(defn->vgrids_legacy), input, 1.0); + + if( inverse ) + z[io] -= value; + else + z[io] += value; + + if( value == HUGE_VAL ) + { + std::string gridlist; + + proj_log_debug(defn, + "pj_apply_vgridshift(): failed to find a grid shift table for\n" + " location (%.7fdW,%.7fdN)", + x[io] * RAD_TO_DEG, + y[io] * RAD_TO_DEG ); + + for( const auto& gridset: *static_cast<ListOfVGrids*>(defn->vgrids_legacy) ) + { + if( gridlist.empty() ) + gridlist += " tried: "; + else + gridlist += ','; + gridlist += gridset->name(); + } + + proj_log_debug(defn, "%s", gridlist.c_str()); + pj_ctx_set_errno( defn->ctx, PJD_ERR_GRID_AREA ); + + return PJD_ERR_GRID_AREA; + } + } + + return 0; +} + /* -------------------------------------------------------------------- */ /* Transform to ellipsoidal heights if needed */ @@ -457,10 +532,7 @@ static int geometric_to_orthometric (PJ *P, PJ_DIRECTION dir, long n, int dist, return 0; if (z==nullptr) return PJD_ERR_GEOCENTRIC; - err = pj_apply_vgridshift (P, "sgeoidgrids", - &(P->vgridlist_geoid), - &(P->vgridlist_geoid_count), - dir==PJ_FWD ? 1 : 0, n, dist, x, y, z ); + err = pj_apply_vgridshift (P, dir==PJ_FWD ? 1 : 0, n, dist, x, y, z ); if (err) return pj_ctx_get_errno(P->ctx); return 0; @@ -822,6 +894,66 @@ int pj_geocentric_from_wgs84( PJ *defn, return 0; } + +/************************************************************************/ +/* pj_apply_gridshift_2() */ +/* */ +/* This implementation uses the gridlist from a coordinate */ +/* system definition. If the gridlist has not yet been */ +/* populated in the coordinate system definition we set it up */ +/* now. */ +/************************************************************************/ +static +int pj_apply_gridshift_2( PJ *defn, int inverse, + long point_count, int point_offset, + double *x, double *y, double * /*z*/ ) + +{ + if( defn->hgrids_legacy == nullptr ) + { + defn->hgrids_legacy = new ListOfHGrids; + auto hgrids = pj_hgrid_init(defn, "nadgrids"); + if( hgrids.empty() ) + return 0; + *static_cast<ListOfHGrids*>(defn->hgrids_legacy) = std::move(hgrids); + } + if( static_cast<ListOfHGrids*>(defn->hgrids_legacy)->empty() ) + { + return 0; + } + + for( long i = 0; i < point_count; i++ ) + { + PJ_LP input; + + long io = i * point_offset; + input.phi = y[io]; + input.lam = x[io]; + + auto output = pj_hgrid_apply(defn->ctx, *static_cast<ListOfHGrids*>(defn->hgrids_legacy), input, inverse ? PJ_INV : PJ_FWD); + + if ( output.lam != HUGE_VAL ) + { + y[io] = output.phi; + x[io] = output.lam; + } + else + { + if( defn->ctx->debug_level >= PJ_LOG_DEBUG_MAJOR ) + { + pj_log( defn->ctx, PJ_LOG_DEBUG_MAJOR, + "pj_apply_gridshift(): failed to find a grid shift table for\n" + " location (%.7fdW,%.7fdN)", + x[io] * RAD_TO_DEG, + y[io] * RAD_TO_DEG ); + } + } + } + + return 0; +} + + /************************************************************************/ /* pj_datum_transform() */ /* */ @@ -1062,3 +1194,8 @@ static int adjust_axis( projCtx ctx, return 0; } +// --------------------------------------------------------------------------- + +void pj_deallocate_grids() +{ +} diff --git a/src/transformations/deformation.cpp b/src/transformations/deformation.cpp index f1311a54..8aee50c9 100644 --- a/src/transformations/deformation.cpp +++ b/src/transformations/deformation.cpp @@ -56,20 +56,90 @@ grid-values in units of mm/year in ENU-space. #include "proj.h" #include "proj_internal.h" #include <math.h> +#include "grids.hpp" + +#include <algorithm> PROJ_HEAD(deformation, "Kinematic grid shift"); #define TOL 1e-8 #define MAX_ITERATIONS 10 +using namespace NS_PROJ; + namespace { // anonymous namespace -struct pj_opaque { - double dt; - double t_epoch; - PJ *cart; +struct deformationData { + double dt = 0; + double t_epoch = 0; + PJ *cart = nullptr; + ListOfGenericGrids grids{}; + ListOfHGrids hgrids{}; + ListOfVGrids vgrids{}; }; } // anonymous namespace +// --------------------------------------------------------------------------- + +static bool get_grid_values(PJ* P, + deformationData* Q, + const PJ_LP& lp, + double& vx, + double& vy, + double& vz) +{ + GenericShiftGridSet* gridset = nullptr; + auto grid = pj_find_generic_grid(Q->grids, lp, gridset); + if( !grid ) { + return false; + } + if( grid->isNullGrid() ) { + vx = 0; + vy = 0; + vz = 0; + return true; + } + const auto samplesPerPixel = grid->samplesPerPixel(); + if( samplesPerPixel < 3 ) { + proj_log_error(P, "deformation: grid has not enough samples"); + return false; + } + int sampleE = 0; + int sampleN = 1; + int sampleU = 2; + for( int i = 0; i < samplesPerPixel; i++ ) + { + const auto desc = grid->description(i); + if( desc == "east_velocity") { + sampleE = i; + } else if( desc == "north_velocity") { + sampleN = i; + } else if( desc == "up_velocity") { + sampleU = i; + } + } + const auto unit = grid->unit(sampleE); + if( !unit.empty() && unit != "millimetres per year" ) { + proj_log_error(P, "deformation: Only unit=millimetres per year currently handled"); + return false; + } + + bool must_retry = false; + if( !pj_bilinear_interpolation_three_samples(grid, lp, + sampleE, sampleN, sampleU, + vx, vy, vz, + must_retry) ) + { + if( must_retry ) + return get_grid_values( P, Q, lp, vx, vy, vz); + return false; + } + // divide by 1000 to get m/year + vx /= 1000; + vy /= 1000; + vz /= 1000; + return true; +} + /********************************************************************************/ static PJ_XYZ get_grid_shift(PJ* P, const PJ_XYZ& cartesian) { /******************************************************************************** @@ -85,22 +155,39 @@ static PJ_XYZ get_grid_shift(PJ* P, const PJ_XYZ& cartesian) { PJ_COORD geodetic, shift, temp; double sp, cp, sl, cl; int previous_errno = proj_errno_reset(P); + auto Q = static_cast<deformationData*>(P->opaque); /* cartesian to geodetic */ - geodetic.lpz = pj_inv3d(cartesian, static_cast<struct pj_opaque*>(P->opaque)->cart); + geodetic.lpz = pj_inv3d(cartesian, Q->cart); /* look up correction values in grids */ - shift.lp = proj_hgrid_value(P, geodetic.lp); - shift.enu.u = proj_vgrid_value(P, geodetic.lp, 1.0); - - if (proj_errno(P) == PJD_ERR_GRID_AREA) - proj_log_debug(P, "deformation: coordinate (%.3f, %.3f) outside deformation model", - proj_todeg(geodetic.lpz.lam), proj_todeg(geodetic.lpz.phi)); - - /* grid values are stored as mm/yr, we need m/yr */ - shift.xyz.x /= 1000; - shift.xyz.y /= 1000; - shift.xyz.z /= 1000; + if( !Q->grids.empty() ) + { + double vx = 0; + double vy = 0; + double vz = 0; + if( !get_grid_values(P, Q, geodetic.lp, vx, vy, vz) ) + { + return proj_coord_error().xyz; + } + shift.xyz.x = vx; + shift.xyz.y = vy; + shift.xyz.z = vz; + } + else + { + shift.lp = pj_hgrid_value(P, Q->hgrids, geodetic.lp); + shift.enu.u = pj_vgrid_value(P, Q->vgrids, geodetic.lp, 1.0); + + if (proj_errno(P) == PJD_ERR_GRID_AREA) + proj_log_debug(P, "deformation: coordinate (%.3f, %.3f) outside deformation model", + proj_todeg(geodetic.lpz.lam), proj_todeg(geodetic.lpz.phi)); + + /* grid values are stored as mm/yr, we need m/yr */ + shift.xyz.x /= 1000; + shift.xyz.y /= 1000; + shift.xyz.z /= 1000; + } /* pre-calc cosines and sines */ sp = sin(geodetic.lpz.phi); @@ -130,6 +217,9 @@ static PJ_XYZ reverse_shift(PJ *P, PJ_XYZ input, double dt) { int i = MAX_ITERATIONS; delta = get_grid_shift(P, input); + if (delta.x == HUGE_VAL) { + return delta; + } /* Store the origial z shift for later application */ z0 = delta.z; @@ -163,7 +253,7 @@ static PJ_XYZ reverse_shift(PJ *P, PJ_XYZ input, double dt) { } static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { - struct pj_opaque *Q = (struct pj_opaque *) P->opaque; + struct deformationData *Q = (struct deformationData *) P->opaque; PJ_COORD out, in; PJ_XYZ shift; in.lpz = lpz; @@ -176,6 +266,9 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { } shift = get_grid_shift(P, in.xyz); + if (shift.x == HUGE_VAL) { + return shift; + } out.xyz.x += Q->dt * shift.x; out.xyz.y += Q->dt * shift.y; @@ -186,7 +279,7 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { static PJ_COORD forward_4d(PJ_COORD in, PJ *P) { - struct pj_opaque *Q = (struct pj_opaque *) P->opaque; + struct deformationData *Q = (struct deformationData *) P->opaque; double dt; PJ_XYZ shift; PJ_COORD out = in; @@ -209,7 +302,7 @@ static PJ_COORD forward_4d(PJ_COORD in, PJ *P) { static PJ_LPZ reverse_3d(PJ_XYZ in, PJ *P) { - struct pj_opaque *Q = (struct pj_opaque *) P->opaque; + struct deformationData *Q = (struct deformationData *) P->opaque; PJ_COORD out; out.xyz = in; @@ -225,7 +318,7 @@ static PJ_LPZ reverse_3d(PJ_XYZ in, PJ *P) { } static PJ_COORD reverse_4d(PJ_COORD in, PJ *P) { - struct pj_opaque *Q = (struct pj_opaque *) P->opaque; + struct deformationData *Q = (struct deformationData *) P->opaque; PJ_COORD out = in; double dt; @@ -244,23 +337,23 @@ static PJ *destructor(PJ *P, int errlev) { if (nullptr==P) return nullptr; - if (nullptr==P->opaque) - return pj_default_destructor (P, errlev); - - if (static_cast<struct pj_opaque*>(P->opaque)->cart) - static_cast<struct pj_opaque*>(P->opaque)->cart->destructor (static_cast<struct pj_opaque*>(P->opaque)->cart, errlev); + auto Q = static_cast<struct deformationData*>(P->opaque); + if( Q ) + { + if (Q->cart) + Q->cart->destructor (Q->cart, errlev); + delete Q; + } + P->opaque = nullptr; return pj_default_destructor(P, errlev); } PJ *TRANSFORMATION(deformation,1) { - int has_xy_grids = 0; - int has_z_grids = 0; - struct pj_opaque *Q = static_cast<struct pj_opaque*>(pj_calloc (1, sizeof (struct pj_opaque))); - if (nullptr==Q) - return destructor(P, ENOMEM); + auto Q = new deformationData; P->opaque = (void *) Q; + P->destructor = destructor; // Pass a dummy ellipsoid definition that will be overridden just afterwards Q->cart = proj_create(P->ctx, "+proj=cart +a=1"); @@ -270,25 +363,38 @@ PJ *TRANSFORMATION(deformation,1) { /* inherit ellipsoid definition from P to Q->cart */ pj_inherit_ellipsoid_def (P, Q->cart); - has_xy_grids = pj_param(P->ctx, P->params, "txy_grids").i; - has_z_grids = pj_param(P->ctx, P->params, "tz_grids").i; + int has_xy_grids = pj_param(P->ctx, P->params, "txy_grids").i; + int has_z_grids = pj_param(P->ctx, P->params, "tz_grids").i; + int has_grids = pj_param(P->ctx, P->params, "tgrids").i; /* Build gridlists. Both horizontal and vertical grids are mandatory. */ - if (!has_xy_grids || !has_z_grids) { - proj_log_error(P, "deformation: Both +xy_grids and +z_grids should be specified."); + if ( !has_grids && (!has_xy_grids || !has_z_grids)) { + proj_log_error(P, "deformation: Either +grids or (+xy_grids and +z_grids) should be specified."); return destructor(P, PJD_ERR_NO_ARGS ); } - proj_hgrid_init(P, "xy_grids"); - if (proj_errno(P)) { - proj_log_error(P, "deformation: could not find requested xy_grid(s)."); - return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + if( has_grids ) + { + Q->grids = pj_generic_grid_init(P, "grids"); + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "deformation: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } } - - proj_vgrid_init(P, "z_grids"); - if (proj_errno(P)) { - proj_log_error(P, "deformation: could not find requested z_grid(s)."); - return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + else + { + Q->hgrids = pj_hgrid_init(P, "xy_grids"); + if (proj_errno(P)) { + proj_log_error(P, "deformation: could not find requested xy_grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } + + Q->vgrids = pj_vgrid_init(P, "z_grids"); + if (proj_errno(P)) { + proj_log_error(P, "deformation: could not find requested z_grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } } Q->dt = HUGE_VAL; @@ -325,7 +431,6 @@ PJ *TRANSFORMATION(deformation,1) { P->left = PJ_IO_UNITS_CARTESIAN; P->right = PJ_IO_UNITS_CARTESIAN; - P->destructor = destructor; return P; } diff --git a/src/transformations/hgridshift.cpp b/src/transformations/hgridshift.cpp index 90633939..122a7ab2 100644 --- a/src/transformations/hgridshift.cpp +++ b/src/transformations/hgridshift.cpp @@ -6,24 +6,38 @@ #include <time.h> #include "proj_internal.h" +#include "grids.hpp" PROJ_HEAD(hgridshift, "Horizontal grid shift"); +using namespace NS_PROJ; + namespace { // anonymous namespace -struct pj_opaque_hgridshift { - double t_final; - double t_epoch; +struct hgridshiftData { + double t_final = 0; + double t_epoch = 0; + ListOfHGrids grids{}; + bool defer_grid_opening = false; }; } // anonymous namespace static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { + auto Q = static_cast<hgridshiftData*>(P->opaque); PJ_COORD point = {{0,0,0,0}}; point.lpz = lpz; - if (P->gridlist != nullptr) { + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = pj_hgrid_init(P, "grids"); + if ( proj_errno(P) ) { + return proj_coord_error().xyz; + } + } + + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ - point.lp = proj_hgrid_apply(P, point.lp, PJ_FWD); + point.lp = pj_hgrid_apply(P->ctx, Q->grids, point.lp, PJ_FWD); } return point.xyz; @@ -31,20 +45,29 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { + auto Q = static_cast<hgridshiftData*>(P->opaque); PJ_COORD point = {{0,0,0,0}}; point.xyz = xyz; - if (P->gridlist != nullptr) { + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = pj_hgrid_init(P, "grids"); + if ( proj_errno(P) ) { + return proj_coord_error().lpz; + } + } + + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ - point.lp = proj_hgrid_apply(P, point.lp, PJ_INV); + point.lp = pj_hgrid_apply(P->ctx, Q->grids, point.lp, PJ_INV); } return point.lpz; } static PJ_COORD forward_4d(PJ_COORD obs, PJ *P) { - struct pj_opaque_hgridshift *Q = (struct pj_opaque_hgridshift *) P->opaque; + struct hgridshiftData *Q = (struct hgridshiftData *) P->opaque; PJ_COORD point = obs; /* If transformation is not time restricted, we always call it */ @@ -62,7 +85,7 @@ static PJ_COORD forward_4d(PJ_COORD obs, PJ *P) { } static PJ_COORD reverse_4d(PJ_COORD obs, PJ *P) { - struct pj_opaque_hgridshift *Q = (struct pj_opaque_hgridshift *) P->opaque; + struct hgridshiftData *Q = (struct hgridshiftData *) P->opaque; PJ_COORD point = obs; /* If transformation is not time restricted, we always call it */ @@ -78,12 +101,29 @@ static PJ_COORD reverse_4d(PJ_COORD obs, PJ *P) { return point; } +static PJ *destructor (PJ *P, int errlev) { + if (nullptr==P) + return nullptr; + + delete static_cast<struct hgridshiftData*>(P->opaque); + P->opaque = nullptr; + + return pj_default_destructor(P, errlev); +} + +static void reassign_context( PJ* P, PJ_CONTEXT* ctx ) +{ + auto Q = (struct hgridshiftData *) P->opaque; + for( auto& grid: Q->grids ) { + grid->reassign_context(ctx); + } +} PJ *TRANSFORMATION(hgridshift,0) { - struct pj_opaque_hgridshift *Q = static_cast<struct pj_opaque_hgridshift*>(pj_calloc (1, sizeof (struct pj_opaque_hgridshift))); - if (nullptr==Q) - return pj_default_destructor (P, ENOMEM); + auto Q = new hgridshiftData; P->opaque = (void *) Q; + P->destructor = destructor; + P->reassign_context = reassign_context; P->fwd4d = forward_4d; P->inv4d = reverse_4d; @@ -97,12 +137,12 @@ PJ *TRANSFORMATION(hgridshift,0) { if (0==pj_param(P->ctx, P->params, "tgrids").i) { proj_log_error(P, "hgridshift: +grids parameter missing."); - return pj_default_destructor (P, PJD_ERR_NO_ARGS); + return destructor (P, PJD_ERR_NO_ARGS); } - /* TODO: Refactor into shared function that can be used */ - /* by both vgridshift and hgridshift */ - if (pj_param(P->ctx, P->params, "tt_final").i) { + /* TODO: Refactor into shared function that can be used */ + /* by both vgridshift and hgridshift */ + if (pj_param(P->ctx, P->params, "tt_final").i) { Q->t_final = pj_param (P->ctx, P->params, "dt_final").f; if (Q->t_final == 0) { /* a number wasn't passed to +t_final, let's see if it was "now" */ @@ -117,16 +157,21 @@ PJ *TRANSFORMATION(hgridshift,0) { } } - if (pj_param(P->ctx, P->params, "tt_epoch").i) + if (pj_param(P->ctx, P->params, "tt_epoch").i) Q->t_epoch = pj_param (P->ctx, P->params, "dt_epoch").f; - proj_hgrid_init(P, "grids"); - /* Was gridlist compiled properly? */ - if ( proj_errno(P) ) { - proj_log_error(P, "hgridshift: could not find required grid(s)."); - return pj_default_destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + if( P->ctx->defer_grid_opening ) { + Q->defer_grid_opening = true; } + else { + Q->grids = pj_hgrid_init(P, "grids"); + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "hgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } + } return P; } diff --git a/src/transformations/vgridshift.cpp b/src/transformations/vgridshift.cpp index de0cdd8c..121b795a 100644 --- a/src/transformations/vgridshift.cpp +++ b/src/transformations/vgridshift.cpp @@ -6,26 +6,67 @@ #include <time.h> #include "proj_internal.h" +#include "grids.hpp" PROJ_HEAD(vgridshift, "Vertical grid shift"); +using namespace NS_PROJ; + namespace { // anonymous namespace -struct pj_opaque_vgridshift { - double t_final; - double t_epoch; - double forward_multiplier; +struct vgridshiftData { + double t_final = 0; + double t_epoch = 0; + double forward_multiplier = 0; + ListOfVGrids grids{}; + bool defer_grid_opening = false; }; } // anonymous namespace +static void deal_with_vertcon_gtx_hack(PJ *P) +{ + struct vgridshiftData *Q = (struct vgridshiftData *) P->opaque; + // The .gtx VERTCON files stored millimetres, but the .tif files + // are in metres. + if( Q->forward_multiplier != 0.001 ) { + return; + } + const char* gridname = pj_param(P->ctx, P->params, "sgrids").s; + if( !gridname ) { + return; + } + if( strcmp(gridname, "vertconw.gtx") != 0 && + strcmp(gridname, "vertconc.gtx") != 0 && + strcmp(gridname, "vertcone.gtx") != 0 ) { + return; + } + if( Q->grids.empty() ) { + return; + } + const auto& grids = Q->grids[0]->grids(); + if( !grids.empty() && + grids[0]->name().find(".tif") != std::string::npos ) { + Q->forward_multiplier = 1.0; + } +} + static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { - struct pj_opaque_vgridshift *Q = (struct pj_opaque_vgridshift *) P->opaque; + struct vgridshiftData *Q = (struct vgridshiftData *) P->opaque; PJ_COORD point = {{0,0,0,0}}; point.lpz = lpz; - if (P->vgridlist_geoid != nullptr) { + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = pj_vgrid_init(P, "grids"); + deal_with_vertcon_gtx_hack(P); + if ( proj_errno(P) ) { + return proj_coord_error().xyz; + } + } + + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ - point.xyz.z += proj_vgrid_value(P, point.lp, Q->forward_multiplier); + point.xyz.z += pj_vgrid_value(P, Q->grids, point.lp, Q->forward_multiplier); } return point.xyz; @@ -33,14 +74,23 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { - struct pj_opaque_vgridshift *Q = (struct pj_opaque_vgridshift *) P->opaque; + struct vgridshiftData *Q = (struct vgridshiftData *) P->opaque; PJ_COORD point = {{0,0,0,0}}; point.xyz = xyz; - if (P->vgridlist_geoid != nullptr) { + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = pj_vgrid_init(P, "grids"); + deal_with_vertcon_gtx_hack(P); + if ( proj_errno(P) ) { + return proj_coord_error().lpz; + } + } + + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ - point.xyz.z -= proj_vgrid_value(P, point.lp, Q->forward_multiplier); + point.xyz.z -= pj_vgrid_value(P, Q->grids, point.lp, Q->forward_multiplier); } return point.lpz; @@ -48,7 +98,7 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { static PJ_COORD forward_4d(PJ_COORD obs, PJ *P) { - struct pj_opaque_vgridshift *Q = (struct pj_opaque_vgridshift *) P->opaque; + struct vgridshiftData *Q = (struct vgridshiftData *) P->opaque; PJ_COORD point = obs; /* If transformation is not time restricted, we always call it */ @@ -66,7 +116,7 @@ static PJ_COORD forward_4d(PJ_COORD obs, PJ *P) { } static PJ_COORD reverse_4d(PJ_COORD obs, PJ *P) { - struct pj_opaque_vgridshift *Q = (struct pj_opaque_vgridshift *) P->opaque; + struct vgridshiftData *Q = (struct vgridshiftData *) P->opaque; PJ_COORD point = obs; /* If transformation is not time restricted, we always call it */ @@ -82,16 +132,34 @@ static PJ_COORD reverse_4d(PJ_COORD obs, PJ *P) { return point; } +static PJ *destructor (PJ *P, int errlev) { + if (nullptr==P) + return nullptr; + + delete static_cast<struct vgridshiftData*>(P->opaque); + P->opaque = nullptr; + + return pj_default_destructor(P, errlev); +} + +static void reassign_context( PJ* P, PJ_CONTEXT* ctx ) +{ + auto Q = (struct vgridshiftData *) P->opaque; + for( auto& grid: Q->grids ) { + grid->reassign_context(ctx); + } +} + PJ *TRANSFORMATION(vgridshift,0) { - struct pj_opaque_vgridshift *Q = static_cast<struct pj_opaque_vgridshift*>(pj_calloc (1, sizeof (struct pj_opaque_vgridshift))); - if (nullptr==Q) - return pj_default_destructor (P, ENOMEM); + auto Q = new vgridshiftData; P->opaque = (void *) Q; + P->destructor = destructor; + P->reassign_context = reassign_context; if (!pj_param(P->ctx, P->params, "tgrids").i) { proj_log_error(P, "vgridshift: +grids parameter missing."); - return pj_default_destructor(P, PJD_ERR_NO_ARGS); + return destructor(P, PJD_ERR_NO_ARGS); } /* TODO: Refactor into shared function that can be used */ @@ -120,13 +188,18 @@ PJ *TRANSFORMATION(vgridshift,0) { Q->forward_multiplier = pj_param(P->ctx, P->params, "dmultiplier").f; } - /* Build gridlist. P->vgridlist_geoid can be empty if +grids only ask for optional grids. */ - proj_vgrid_init(P, "grids"); - - /* Was gridlist compiled properly? */ - if ( proj_errno(P) ) { - proj_log_error(P, "vgridshift: could not find required grid(s)."); - return pj_default_destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + if( P->ctx->defer_grid_opening ) { + Q->defer_grid_opening = true; + } + else { + /* Build gridlist. P->vgridlist_geoid can be empty if +grids only ask for optional grids. */ + Q->grids = pj_vgrid_init(P, "grids"); + + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "vgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } } P->fwd4d = forward_4d; diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp new file mode 100644 index 00000000..e1c76518 --- /dev/null +++ b/src/transformations/xyzgridshift.cpp @@ -0,0 +1,303 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: Geocentric translation using a grid + * Author: Even Rouault, <even.rouault at spatialys.com> + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#define PJ_LIB__ + +#include <errno.h> +#include <string.h> +#include <stddef.h> +#include <time.h> + +#include "proj_internal.h" +#include "grids.hpp" + +#include <algorithm> + +PROJ_HEAD(xyzgridshift, "Geocentric grid shift"); + +using namespace NS_PROJ; + +namespace { // anonymous namespace +struct xyzgridshiftData { + PJ *cart = nullptr; + bool grid_ref_is_input = true; + ListOfGenericGrids grids{}; + bool defer_grid_opening = false; + double multiplier = 1.0; +}; +} // anonymous namespace + +// --------------------------------------------------------------------------- + +static bool get_grid_values(PJ* P, + xyzgridshiftData* Q, + const PJ_LP& lp, + double& dx, + double& dy, + double& dz) +{ + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = pj_generic_grid_init(P, "grids"); + if ( proj_errno(P) ) { + return false; + } + } + + GenericShiftGridSet* gridset = nullptr; + auto grid = pj_find_generic_grid(Q->grids, lp, gridset); + if( !grid ) { + return false; + } + if( grid->isNullGrid() ) { + dx = 0; + dy = 0; + dz = 0; + return true; + } + const auto samplesPerPixel = grid->samplesPerPixel(); + if( samplesPerPixel < 3 ) { + proj_log_error(P, "xyzgridshift: grid has not enough samples"); + return false; + } + int sampleX = 0; + int sampleY = 1; + int sampleZ = 2; + for( int i = 0; i < samplesPerPixel; i++ ) + { + const auto desc = grid->description(i); + if( desc == "x_translation") { + sampleX = i; + } else if( desc == "y_translation") { + sampleY = i; + } else if( desc == "z_translation") { + sampleZ = i; + } + } + const auto unit = grid->unit(sampleX); + if( !unit.empty() && unit != "metre" ) { + proj_log_error(P, "xyzgridshift: Only unit=metre currently handled"); + return false; + } + + bool must_retry = false; + if( !pj_bilinear_interpolation_three_samples(grid, lp, + sampleX, sampleY, sampleZ, + dx, dy, dz, + must_retry) ) + { + if( must_retry ) + return get_grid_values( P, Q, lp, dx, dy, dz); + return false; + } + + dx *= Q->multiplier; + dy *= Q->multiplier; + dz *= Q->multiplier; + return true; +} + +// --------------------------------------------------------------------------- + +#define SQUARE(x) ((x)*(x)) + +// --------------------------------------------------------------------------- + +static PJ_COORD iterative_adjustment(PJ* P, + xyzgridshiftData* Q, + const PJ_COORD& pointInit, + double factor) +{ + PJ_COORD point = pointInit; + for(int i = 0; i < 10; i++) { + PJ_COORD geodetic; + geodetic.lpz = pj_inv3d(point.xyz, Q->cart); + + double dx, dy, dz; + if( !get_grid_values(P, Q, geodetic.lp, dx, dy, dz) ) { + return proj_coord_error(); + } + + dx *= factor; + dy *= factor; + dz *= factor; + + const double err = SQUARE((point.xyz.x - pointInit.xyz.x) - dx) + + SQUARE((point.xyz.y - pointInit.xyz.y) - dy) + + SQUARE((point.xyz.z - pointInit.xyz.z) - dz); + + point.xyz.x = pointInit.xyz.x + dx; + point.xyz.y = pointInit.xyz.y + dy; + point.xyz.z = pointInit.xyz.z + dz; + if( err < 1e-10 ) { + break; + } + } + return point; +} + +// --------------------------------------------------------------------------- + +static PJ_COORD direct_adjustment(PJ* P, + xyzgridshiftData* Q, + PJ_COORD point, + double factor) +{ + PJ_COORD geodetic; + geodetic.lpz = pj_inv3d(point.xyz, Q->cart); + + double dx, dy, dz; + if( !get_grid_values(P, Q, geodetic.lp, dx, dy, dz) ) { + return proj_coord_error(); + } + point.xyz.x += factor * dx; + point.xyz.y += factor * dy; + point.xyz.z += factor * dz; + return point; +} + +// --------------------------------------------------------------------------- + +static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { + auto Q = static_cast<xyzgridshiftData*>(P->opaque); + PJ_COORD point = {{0,0,0,0}}; + point.lpz = lpz; + + if( Q->grid_ref_is_input ) { + point = direct_adjustment(P, Q, point, 1.0); + } + else { + point = iterative_adjustment(P, Q, point, 1.0); + } + + return point.xyz; +} + + +static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { + auto Q = static_cast<xyzgridshiftData*>(P->opaque); + PJ_COORD point = {{0,0,0,0}}; + point.xyz = xyz; + + if( Q->grid_ref_is_input ) { + point = iterative_adjustment(P, Q, point, -1.0); + } + else { + point = direct_adjustment(P, Q, point, -1.0); + } + + return point.lpz; +} + +static PJ *destructor (PJ *P, int errlev) { + if (nullptr==P) + return nullptr; + + auto Q = static_cast<struct xyzgridshiftData*>(P->opaque); + if( Q ) + { + if (Q->cart) + Q->cart->destructor (Q->cart, errlev); + delete Q; + } + P->opaque = nullptr; + + return pj_default_destructor(P, errlev); +} + +static void reassign_context( PJ* P, PJ_CONTEXT* ctx ) +{ + auto Q = (struct xyzgridshiftData *) P->opaque; + for( auto& grid: Q->grids ) { + grid->reassign_context(ctx); + } +} + + +PJ *TRANSFORMATION(xyzgridshift,0) { + auto Q = new xyzgridshiftData; + P->opaque = (void *) Q; + P->destructor = destructor; + P->reassign_context = reassign_context; + + P->fwd4d = nullptr; + P->inv4d = nullptr; + P->fwd3d = forward_3d; + P->inv3d = reverse_3d; + P->fwd = nullptr; + P->inv = nullptr; + + P->left = PJ_IO_UNITS_CARTESIAN; + P->right = PJ_IO_UNITS_CARTESIAN; + + // Pass a dummy ellipsoid definition that will be overridden just afterwards + Q->cart = proj_create(P->ctx, "+proj=cart +a=1"); + if (Q->cart == nullptr) + return destructor(P, ENOMEM); + + /* inherit ellipsoid definition from P to Q->cart */ + pj_inherit_ellipsoid_def (P, Q->cart); + + const char* grid_ref = pj_param (P->ctx, P->params, "sgrid_ref").s; + if( grid_ref ) { + if (strcmp(grid_ref, "input_crs") == 0 ) { + // default + } else if (strcmp(grid_ref, "output_crs") == 0 ) { + // Convention use for example for NTF->RGF93 grid that contains + // delta x,y,z from NTF to RGF93, but the grid itself is referenced + // in RGF93 + Q->grid_ref_is_input = false; + } else { + proj_log_error(P, "xyzgridshift: unusupported value for grid_ref"); + return destructor (P, PJD_ERR_NO_ARGS); + } + } + + if (0==pj_param(P->ctx, P->params, "tgrids").i) { + proj_log_error(P, "xyzgridshift: +grids parameter missing."); + return destructor (P, PJD_ERR_NO_ARGS); + } + + /* multiplier for delta x,y,z */ + if (pj_param(P->ctx, P->params, "tmultiplier").i) { + Q->multiplier = pj_param(P->ctx, P->params, "dmultiplier").f; + } + + if( P->ctx->defer_grid_opening ) { + Q->defer_grid_opening = true; + } + else { + Q->grids = pj_generic_grid_init(P, "grids"); + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "xyzgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } + } + + return P; +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ae721d46..a7aac755 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -10,6 +10,7 @@ proj_add_gie_test("GDA" "gie/GDA.gie") proj_add_gie_test("4D-API-cs2cs-style" "gie/4D-API_cs2cs-style.gie") proj_add_gie_test("DHDN_ETRS89" "gie/DHDN_ETRS89.gie") proj_add_gie_test("Unitconvert" "gie/unitconvert.gie") +proj_add_gie_test("geotiff_grids" "gie/geotiff_grids.gie") # GIGS tests. Uncommented tests are expected to fail due to issues with # various projections. Should be investigated further and fixed. diff --git a/test/cli/Makefile.am b/test/cli/Makefile.am index 33878743..417ec137 100644 --- a/test/cli/Makefile.am +++ b/test/cli/Makefile.am @@ -29,7 +29,7 @@ EXTRA_DIST = pj_out27.dist pj_out83.dist td_out.dist \ CMakeLists.txt testprojinfo-check: - PROJ_LIB=$(PROJ_LIB) $(TESTPROJINFO) $(PROJINFOEXE) + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTPROJINFO) $(PROJINFOEXE) test27-check: $(TEST27) $(PROJEXE) @@ -41,24 +41,24 @@ testproj-check: $(TESTPROJ) $(PROJEXE) testvarious-check: - PROJ_LIB=$(PROJ_LIB) $(TESTVARIOUS) $(CS2CSEXE) + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTVARIOUS) $(CS2CSEXE) testdatumfile-check: @if [ -f $(PROJ_LIB)/conus -a -f $(PROJ_LIB)/ntv1_can.dat -a -f $(PROJ_LIB)/MD -a -f $(PROJ_LIB)/ntf_r93.gsb -a -f $(PROJ_LIB)/egm96_15.gtx ]; then \ - PROJ_LIB=$(PROJ_LIB) $(TESTDATUMFILE) $(CS2CSEXE) ; \ + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTDATUMFILE) $(CS2CSEXE) ; \ fi testign-check: @if [ -f $(PROJ_LIB)/ntf_r93.gsb ] ; then \ - PROJ_LIB=$(PROJ_LIB) $(TESTIGN) $(CS2CSEXE) ; \ + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTIGN) $(CS2CSEXE) ; \ fi testntv2-check: @if [ -f $(PROJ_LIB)/ntv2_0.gsb -a -f $(PROJ_LIB)/conus -a -f $(PROJ_LIB)/ntv1_can.dat ] ; then \ - PROJ_LIB=$(PROJ_LIB) $(TESTNTV2) $(CS2CSEXE) ; \ + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTNTV2) $(CS2CSEXE) ; \ fi testcct-check: - PROJ_LIB=$(PROJ_LIB) $(TESTCCT) $(CCTEXE) + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTCCT) $(CCTEXE) check-local: testprojinfo-check test27-check test83-check testproj-check testvarious-check testdatumfile-check testign-check testntv2-check testcct-check diff --git a/test/cli/ntv2_out.dist b/test/cli/ntv2_out.dist index 531db7be..d90fdc37 100644 --- a/test/cli/ntv2_out.dist +++ b/test/cli/ntv2_out.dist @@ -13,6 +13,14 @@ Try with NTv2 and NTv1 together ... falls back to NTv1 Switching between NTv2 subgrids -112.5839956 49.4914451 0 -112.58307487 49.49145197 0.00000000 ############################################################## +Interpolating very close (and sometimes a bit outside) to the edges a NTv2 subgrid (#209) +-115.58333333 51.25000000 0 -115.58228512 51.24997866 0.00000000 +-115.58333333 51.25000010 0 -115.58228512 51.24997876 0.00000000 +-115.58333334 51.25000000 0 -115.58228513 51.24997866 0.00000000 +-115.49166667 51.07500000 0 -115.49062909 51.07497666 0.00000000 +-115.49166668 51.07500000 0 -115.49062910 51.07497666 0.00000000 +-115.49166667 51.07499990 0 -115.49062909 51.07497656 0.00000000 +############################################################## Attempt first with ntv2_0.gsb and then conus -111.5 45.26 -111.50079772 45.25992835 0.00000000 ############################################################## diff --git a/test/cli/td_out.dist b/test/cli/td_out.dist index 478a0d84..cf4b8d73 100644 --- a/test/cli/td_out.dist +++ b/test/cli/td_out.dist @@ -21,7 +21,7 @@ edge or even a wee bit outside (#141). -5.5001 52.0 -5.500100000000 52.000000000000 0.000000000000 -5.5 52.0 -5.498893534472 52.000109529716 0.000000000000 -5.5000000000001 52.0000000000001 -5.498893534472 52.000109529717 0.000000000000 --5.4999 51.9999 -5.498793541695 52.000009529743 0.000000000000 +-5.4999 51.9999 -5.498793593803 52.000009531513 0.000000000000 -5.5001 52.0 -5.500100000000 52.000000000000 0.000000000000 ############################################################## NAD27 -> NAD83: 1st through ntv1 or ntv2, 2nd through conus diff --git a/test/cli/test27 b/test/cli/test27 index bfc1cb0a..5825ed80 100755 --- a/test/cli/test27 +++ b/test/cli/test27 @@ -34,7 +34,7 @@ echo "Running ${0} using ${EXE}:" echo "============================================" OUT=proj_out27 -INIT_FILE=${PROJ_LIB}/nad27 +INIT_FILE=nad27 # echo "doing tests into file ${OUT}, please wait" # diff --git a/test/cli/test83 b/test/cli/test83 index cfb1365e..8c1293d0 100755 --- a/test/cli/test83 +++ b/test/cli/test83 @@ -35,7 +35,7 @@ echo "Running ${0} using ${EXE}:" echo "============================================" OUT=proj_out83 -INIT_FILE=${PROJ_LIB}/nad83 +INIT_FILE=nad83 # echo "doing tests into file ${OUT}, please wait" # diff --git a/test/cli/testdatumfile b/test/cli/testdatumfile index e4b9ea2d..16e4bbc3 100755 --- a/test/cli/testdatumfile +++ b/test/cli/testdatumfile @@ -27,7 +27,11 @@ echo "Running ${0} using ${EXE}:" echo "============================================" mkdir "dir with \" space" -cp ${PROJ_LIB}/conus "dir with \" space/myconus" +if test -f "${PROJ_LIB}/conus"; then + cp "${PROJ_LIB}/conus" "dir with \" space/myconus" +else + cp "`dirname $0`/../../data/conus" "dir with \" space/myconus" +fi OUT=td_out #EXE=../src/cs2cs diff --git a/test/cli/testntv2 b/test/cli/testntv2 index 72a0f9a2..44ccac1e 100755 --- a/test/cli/testntv2 +++ b/test/cli/testntv2 @@ -61,11 +61,22 @@ $EXE +proj=latlong +datum=NAD83 +to +proj=latlong +ellps=clrk66 +nadgrids=ntv2_0 EOF echo "##############################################################" >> ${OUT} +echo "Interpolating very close (and sometimes a bit outside) to the edges a NTv2 subgrid (#209)" >> ${OUT} +$EXE +proj=latlong +datum=NAD83 +to +proj=latlong +ellps=clrk66 +nadgrids=ntv2_0.gsb -E -d 8 >>${OUT} <<EOF +-115.58333333 51.25000000 0 +-115.58333333 51.25000010 0 +-115.58333334 51.25000000 0 +-115.49166667 51.07500000 0 +-115.49166668 51.07500000 0 +-115.49166667 51.07499990 0 +EOF + +echo "##############################################################" >> ${OUT} echo Attempt first with ntv2_0.gsb and then conus >> ${OUT} $EXE +proj=longlat +datum=NAD27 +to +proj=longlat +datum=WGS84 -E -d 8 >>${OUT} <<EOF -111.5 45.26 EOF -# + echo "##############################################################" >> ${OUT} echo "NAD27 -> NAD83: 1st through ntv2, 2nd through conus" >> ${OUT} # diff --git a/test/gie/4D-API_cs2cs-style.gie b/test/gie/4D-API_cs2cs-style.gie index e5722b5e..3e4b9d2c 100644 --- a/test/gie/4D-API_cs2cs-style.gie +++ b/test/gie/4D-API_cs2cs-style.gie @@ -442,12 +442,24 @@ Test bugfix of https://github.com/OSGeo/proj.4/issues/1002 ------------------------------------------------------------------------------- operation +proj=latlong +ellps=WGS84 +geoidgrids=tests/test_nodata.gtx ------------------------------------------------------------------------------- -ignore pjd_err_failed_to_load_grid accept 4.05 52.1 0 expect 4.05 52.1 -10 ------------------------------------------------------------------------------- ------------------------------------------------------------------------------- +Test null grid with vgridshift +------------------------------------------------------------------------------- +operation proj=vgridshift grids=tests/test_nodata.gtx,null ellps=GRS80 +------------------------------------------------------------------------------- +accept 4.05 52.1 0 +expect 4.05 52.1 -10 + +# Outside validity area of test_nodata.gtx. Fallback on null +accept 4.05 -52.1 0 +expect 4.05 -52.1 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- Test bug fix of https://github.com/OSGeo/proj.4/issues/1025. Using geocent in the new API with a custom ellipsoid should return coordinates that correspond to that particular ellipsoid and not WGS84 as demonstrated in diff --git a/test/gie/Makefile.am b/test/gie/Makefile.am index 058b1fe9..560fea0b 100644 --- a/test/gie/Makefile.am +++ b/test/gie/Makefile.am @@ -9,35 +9,39 @@ EXTRA_DIST = 4D-API_cs2cs-style.gie \ ellipsoid.gie \ more_builtins.gie \ unitconvert.gie \ - DHDN_ETRS89.gie + DHDN_ETRS89.gie \ + geotiff_grids.gie PROJ_LIB ?= ../../data/for_tests 4D-API-cs2cs-style: 4D-API_cs2cs-style.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< GDA: GDA.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< axisswap: axisswap.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< builtins: builtins.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< deformation: deformation.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< ellipsoid: ellipsoid.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< more_builtins: more_builtins.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< unitconvert: unitconvert.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< DHDN_ETRS89: DHDN_ETRS89.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< -check-local: 4D-API-cs2cs-style GDA axisswap builtins deformation ellipsoid more_builtins unitconvert DHDN_ETRS89 +geotiff_grids: geotiff_grids.gie + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + +check-local: 4D-API-cs2cs-style GDA axisswap builtins deformation ellipsoid more_builtins unitconvert DHDN_ETRS89 geotiff_grids diff --git a/test/gie/deformation.gie b/test/gie/deformation.gie index 37f62e6e..848b9e89 100644 --- a/test/gie/deformation.gie +++ b/test/gie/deformation.gie @@ -12,6 +12,35 @@ The input coordinate is located at lon=60, lam=-160 - somewhere in Alaska. <gie> ------------------------------------------------------------------------------- +Test with an extract of nkgrf03vel_realigned with ctable2+gtx +------------------------------------------------------------------------------- +operation +proj=pipeline + +step +proj=cart +ellps=GRS80 + +step +proj=deformation + +xy_grids=tests/nkgrf03vel_realigned_xy_extract.ct2 + +z_grids=tests/nkgrf03vel_realigned_z_extract.gtx +ellps=GRS80 +dt=1 + +step +proj=cart +ellps=GRS80 +inv +------------------------------------------------------------------------------- +tolerance 0.05 mm +accept 21.5 63 0 +expect 21.5000000049 62.9999999937 0.0083 +roundtrip 5 + +------------------------------------------------------------------------------- +Test with an extract of nkgrf03vel_realigned with GeoTIFF +------------------------------------------------------------------------------- +operation +proj=pipeline + +step +proj=cart +ellps=GRS80 + +step +proj=deformation + +grids=tests/nkgrf03vel_realigned_extract.tif +ellps=GRS80 +dt=1 + +step +proj=cart +ellps=GRS80 +inv +------------------------------------------------------------------------------- +tolerance 0.05 mm +accept 21.5 63 0 +expect 21.5000000049 62.9999999937 0.0083 +roundtrip 5 + +------------------------------------------------------------------------------- Test the +dt parameter ------------------------------------------------------------------------------- operation +proj=deformation +xy_grids=alaska +z_grids=egm96_15.gtx diff --git a/test/gie/geotiff_grids.gie b/test/gie/geotiff_grids.gie new file mode 100644 index 00000000..62a5b16d --- /dev/null +++ b/test/gie/geotiff_grids.gie @@ -0,0 +1,324 @@ + +------------------------------------------------------------------------------- +=============================================================================== +Test GeoTIFF grids +=============================================================================== + +<gie> + +# Those first tests using +proj=vgridshift only test the capability of reading +# correctly a value from various formulations of GeoTIFF file, hence only the +# forward path is tested (reverse path is tested in other files) + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_pixelispoint.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_pixelisarea.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_deflate.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_deflate_floatingpointpredictor.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_uint16.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_uint16_with_scale_offset.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_int16.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_int32.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_uint32.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_float64.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +# The overview should be ignored +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_with_overview.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_in_second_channel.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bigtiff.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bigendian.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bigendian_bigtiff.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bottomup_with_scale.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bottomup_with_matrix.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_with_subgrid.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 + +# In subgrid +accept 5.5 53.5 0 +expect 5.5 53.5 110.0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_nodata.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.05 52.1 0 +expect 4.05 52.1 10 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_invalid_channel_type.tif +multiplier=1 +------------------------------------------------------------------------------- +expect failure errno failed_to_load_grid +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_unsupported_byte.tif +multiplier=1 +------------------------------------------------------------------------------- +expect failure errno failed_to_load_grid +------------------------------------------------------------------------------- + + + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_separate.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_strip.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_tiled.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_tiled_separate.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_positive_west.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_lon_shift_first.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_radian.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_degree.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +# The overview should be ignored +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_with_overview.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_extra_ifd_with_other_info.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +# Subset of NTv2_0.gsb +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_with_subgrid.tif +------------------------------------------------------------------------------- +# In subgrid ALbanff, of parent CAwest +accept -115.5416667 51.1666667 0 +expect -115.5427092888 51.1666899972 0 + +# In subgrid ONtronto, of parent CAeast +accept -80.5041667 44.5458333 0 +expect -80.50401615833 44.5458827236 0 +------------------------------------------------------------------------------- + +# Subset of NTv2_0.gsb +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_with_subgrid_no_grid_name.tif +------------------------------------------------------------------------------- +# In subgrid ALbanff, of parent CAwest +accept -115.5416667 51.1666667 0 +expect -115.5427092888 51.1666899972 0 + +# In subgrid ONtronto, of parent CAeast +accept -80.5041667 44.5458333 0 +expect -80.50401615833 44.5458827236 0 +------------------------------------------------------------------------------- + +# Check a nested grid of a nested grid only based on spatial extent analysis +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_with_two_level_of_subgrids_no_grid_name.tif +------------------------------------------------------------------------------- +accept -45.0 22.5 +accept -44.9983333334 22.5013888889 + +# Check a nested grid of a nested grid only based on spatial extent analysis +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_hgrid_with_two_level_of_subgrids_no_grid_name.tif +multiplier=1 +------------------------------------------------------------------------------- +accept -45.0 22.5 0 +accept -45.0 22.5 5 + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_vgrid.tif +------------------------------------------------------------------------------- +expect failure errno failed_to_load_grid +------------------------------------------------------------------------------- + + +# IGNF:LAMBE to IGNF:LAMB93 using xyzgridshift operation +------------------------------------------------------------------------------- +operation +proj=pipeline + +step +inv +proj=lcc +lat_1=46.8 +lat_0=46.8 +lon_0=0 + +k_0=0.99987742 +x_0=600000 +y_0=2200000 +ellps=clrk80ign +pm=paris + +step +proj=push +v_3 + +step +proj=cart +ellps=clrk80ign + +step +proj=xyzgridshift +grids=tests/subset_of_gr3df97a.tif +grid_ref=output_crs +ellps=GRS80 + +step +proj=cart +ellps=GRS80 +inv + +step +proj=pop +v_3 + +step +proj=lcc +lat_0=46.5 +lon_0=3 +lat_1=49 +lat_2=44 + +x_0=700000 +y_0=6600000 +ellps=GRS80 +------------------------------------------------------------------------------- +tolerance 1 mm + +accept 814149.529 1887019.768 0 +expect 860690.804 6319036.849 0 +# If using ntf_r93.gsb, one gets: 860690.805 6319036.850 + +roundtrip 1 +------------------------------------------------------------------------------- + + +</gie> diff --git a/test/gigs/Makefile.am b/test/gigs/Makefile.am index a66052db..436a6e89 100644 --- a/test/gigs/Makefile.am +++ b/test/gigs/Makefile.am @@ -23,54 +23,54 @@ EXTRA_DIST = \ PROJ_LIB ?= ../../data/for_tests 5101.1: 5101.1-jhs.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5101.2: 5101.2-jhs.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5101.3: 5101.3-jhs.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5101.4: 5101.4-jhs-etmerc.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5102.1: 5102.1.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5103.1: 5103.1.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5103.2: 5103.2.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5103.3: 5103.3.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5105.2: 5105.2.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5106: 5106.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5107: 5107.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5109: 5109.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5111.1: 5111.1.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5112: 5112.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5113: 5113.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5201: 5201.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5208: 5208.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< check-local: 5101.1 5101.2 5101.3 5101.4 5102.1 5103.1 5103.2 5103.3 5105.2 5106 5107 5109 5111.1 5112 5113 5201 5208 diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 841d72b3..845d07e5 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -71,7 +71,7 @@ target_link_libraries(proj_pj_transform_test ${PROJ_LIBRARIES}) add_test(NAME proj_pj_transform_test COMMAND proj_pj_transform_test) set_property(TEST proj_pj_transform_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests") add_executable(proj_errno_string_test @@ -82,7 +82,7 @@ target_link_libraries(proj_errno_string_test ${PROJ_LIBRARIES}) add_test(NAME proj_errno_string_test COMMAND proj_errno_string_test) set_property(TEST proj_errno_string_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests") add_executable(proj_angular_io_test main.cpp @@ -92,7 +92,7 @@ target_link_libraries(proj_angular_io_test ${PROJ_LIBRARIES}) add_test(NAME proj_angular_io_test COMMAND proj_angular_io_test) set_property(TEST proj_angular_io_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests") add_executable(proj_context_test main.cpp @@ -102,7 +102,7 @@ target_link_libraries(proj_context_test ${PROJ_LIBRARIES}) add_test(NAME proj_context_test COMMAND proj_context_test) set_property(TEST proj_context_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests") if(MSVC AND BUILD_LIBPROJ_SHARED) # ph_phi2_test not compatible of a .dll build @@ -115,7 +115,7 @@ else() ${PROJ_LIBRARIES}) add_test(NAME pj_phi2_test COMMAND pj_phi2_test) set_property(TEST pj_phi2_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests") endif() add_executable(proj_test_cpp_api @@ -128,14 +128,15 @@ add_executable(proj_test_cpp_api test_operation.cpp test_datum.cpp test_factory.cpp - test_c_api.cpp) + test_c_api.cpp + test_grids.cpp) target_link_libraries(proj_test_cpp_api GTest::gtest ${PROJ_LIBRARIES} ${SQLITE3_LIBRARY}) add_test(NAME proj_test_cpp_api COMMAND proj_test_cpp_api) set_property(TEST proj_test_cpp_api - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests") add_executable(gie_self_tests @@ -146,4 +147,20 @@ target_link_libraries(gie_self_tests ${PROJ_LIBRARIES}) add_test(NAME gie_self_tests COMMAND gie_self_tests) set_property(TEST gie_self_tests - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests") + + +add_executable(test_network + main.cpp + test_network.cpp) +if(CURL_FOUND) + include_directories(${CURL_INCLUDE_DIR}) + target_link_libraries(test_network ${CURL_LIBRARY}) +endif() +target_link_libraries(test_network + GTest::gtest + ${PROJ_LIBRARIES} + ${SQLITE3_LIBRARY}) +add_test(NAME test_network COMMAND test_network) +set_property(TEST test_network + PROPERTY ENVIRONMENT "PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") diff --git a/test/unit/Makefile.am b/test/unit/Makefile.am index b7196828..11a6473c 100644 --- a/test/unit/Makefile.am +++ b/test/unit/Makefile.am @@ -17,12 +17,13 @@ noinst_PROGRAMS += proj_context_test noinst_PROGRAMS += test_cpp_api noinst_PROGRAMS += gie_self_tests noinst_PROGRAMS += include_proj_h_from_c +noinst_PROGRAMS += test_network pj_transform_test_SOURCES = pj_transform_test.cpp main.cpp pj_transform_test_LDADD = ../../src/libproj.la @GTEST_LIBS@ pj_transform_test-check: pj_transform_test - PROJ_LIB=$(PROJ_LIB) ./pj_transform_test + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) ./pj_transform_test pj_phi2_test_SOURCES = pj_phi2_test.cpp main.cpp pj_phi2_test_LDADD = ../../src/libproj.la @GTEST_LIBS@ @@ -46,20 +47,27 @@ proj_context_test_SOURCES = proj_context_test.cpp main.cpp proj_context_test_LDADD = ../../src/libproj.la @GTEST_LIBS@ proj_context_test-check: proj_context_test - ./proj_context_test + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES ./proj_context_test -test_cpp_api_SOURCES = test_util.cpp test_common.cpp test_crs.cpp test_metadata.cpp test_io.cpp test_operation.cpp test_datum.cpp test_factory.cpp test_c_api.cpp main.cpp +test_cpp_api_SOURCES = test_util.cpp test_common.cpp test_crs.cpp test_metadata.cpp test_io.cpp test_operation.cpp test_datum.cpp test_factory.cpp test_c_api.cpp test_grids.cpp main.cpp test_cpp_api_LDADD = ../../src/libproj.la @GTEST_LIBS@ @SQLITE3_LIBS@ test_cpp_api-check: test_cpp_api - PROJ_LIB=$(PROJ_LIB) ./test_cpp_api + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) ./test_cpp_api gie_self_tests_SOURCES = gie_self_tests.cpp main.cpp gie_self_tests_LDADD = ../../src/libproj.la @GTEST_LIBS@ @SQLITE3_LIBS@ gie_self_tests-check: gie_self_tests - PROJ_LIB=$(PROJ_LIB) ./gie_self_tests + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) ./gie_self_tests include_proj_h_from_c_SOURCES = include_proj_h_from_c.c -check-local: pj_transform_test-check pj_phi2_test-check proj_errno_string_test-check proj_angular_io_test-check proj_context_test-check test_cpp_api-check gie_self_tests-check +test_network_SOURCES = test_network.cpp main.cpp +test_network_CXXFLAGS = @CURL_CFLAGS@ @CURL_ENABLED_FLAGS@ +test_network_LDADD = ../../src/libproj.la @GTEST_LIBS@ @SQLITE3_LIBS@ @CURL_LIBS@ + +test_network-check: test_network + PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) PROJ_SOURCE_DATA=$(PROJ_LIB) ./test_network + +check-local: pj_transform_test-check pj_phi2_test-check proj_errno_string_test-check proj_angular_io_test-check proj_context_test-check test_cpp_api-check gie_self_tests-check test_network-check diff --git a/test/unit/gie_self_tests.cpp b/test/unit/gie_self_tests.cpp index a738db75..bcf31139 100644 --- a/test/unit/gie_self_tests.cpp +++ b/test/unit/gie_self_tests.cpp @@ -348,7 +348,9 @@ TEST(gie, info_functions) { /* proj_info() */ /* this one is difficult to test, since the output changes with the setup */ + putenv(const_cast<char *>("PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=")); info = proj_info(); + putenv(const_cast<char *>("PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES")); if (info.version[0] != '\0') { char tmpstr[64]; @@ -360,6 +362,9 @@ TEST(gie, info_functions) { ASSERT_NE(std::string(info.searchpath), std::string()); } + ASSERT_TRUE(std::string(info.searchpath).find("/proj") != + std::string::npos); + /* proj_pj_info() */ { P = proj_create(PJ_DEFAULT_CTX, diff --git a/test/unit/pj_transform_test.cpp b/test/unit/pj_transform_test.cpp index 1f4473c1..ddb054f0 100644 --- a/test/unit/pj_transform_test.cpp +++ b/test/unit/pj_transform_test.cpp @@ -616,6 +616,22 @@ TEST(proj_api_h, pj_set_finder) { // --------------------------------------------------------------------------- +TEST(proj_api_h, default_fileapi) { + auto ctx = pj_ctx_alloc(); + auto fid = pj_open_lib(ctx, "proj.db", "rb"); + ASSERT_NE(fid, nullptr); + char header[6]; + ASSERT_EQ(pj_ctx_fread(ctx, header, 1, 6, fid), 6U); + ASSERT_TRUE(memcmp(header, "SQLite", 6) == 0); + ASSERT_EQ(pj_ctx_ftell(ctx, fid), 6); + ASSERT_EQ(pj_ctx_fseek(ctx, fid, 0, SEEK_SET), 0); + ASSERT_EQ(pj_ctx_ftell(ctx, fid), 0); + pj_ctx_fclose(ctx, fid); + pj_ctx_free(ctx); +} + +// --------------------------------------------------------------------------- + TEST(pj_transform_test, ob_tran_to_meter_as_dest) { auto src = pj_init_plus( "+ellps=WGS84 +a=57.29577951308232 +proj=eqc +lon_0=0.0 +no_defs"); @@ -633,6 +649,79 @@ TEST(pj_transform_test, ob_tran_to_meter_as_dest) { // --------------------------------------------------------------------------- +struct Spy { + bool gotInMyFOpen = false; + bool gotInMyFRead = false; + bool gotInMyFSeek = false; + bool gotInMyFTell = false; + bool gotInMyFClose = false; +}; + +struct MyFile { + FILE *fp; + Spy *spy; +}; + +static PAFile myFOpen(projCtx ctx, const char *filename, const char *access) { + FILE *fp = fopen(filename, access); + if (!fp) + return nullptr; + MyFile *myF = new MyFile; + myF->spy = (Spy *)pj_ctx_get_app_data(ctx); + myF->spy->gotInMyFOpen = true; + myF->fp = fp; + return reinterpret_cast<PAFile>(myF); +} + +static size_t myFRead(void *buffer, size_t size, size_t nmemb, PAFile file) { + MyFile *myF = reinterpret_cast<MyFile *>(file); + myF->spy->gotInMyFRead = true; + return fread(buffer, size, nmemb, myF->fp); +} + +static int myFSeek(PAFile file, long offset, int whence) { + MyFile *myF = reinterpret_cast<MyFile *>(file); + myF->spy->gotInMyFSeek = true; + return fseek(myF->fp, offset, whence); +} + +static long myFTell(PAFile file) { + MyFile *myF = reinterpret_cast<MyFile *>(file); + myF->spy->gotInMyFTell = true; + return ftell(myF->fp); +} + +static void myFClose(PAFile file) { + MyFile *myF = reinterpret_cast<MyFile *>(file); + myF->spy->gotInMyFClose = true; + fclose(myF->fp); + delete myF; +} + +TEST(proj_api_h, custom_fileapi) { + auto ctx = pj_ctx_alloc(); + Spy spy; + pj_ctx_set_app_data(ctx, &spy); + projFileAPI myAPI = {myFOpen, myFRead, myFSeek, myFTell, myFClose}; + pj_ctx_set_fileapi(ctx, &myAPI); + EXPECT_EQ(pj_ctx_get_fileapi(ctx), &myAPI); + auto fid = pj_open_lib(ctx, "proj.db", "rb"); + ASSERT_NE(fid, nullptr); + char header[6]; + ASSERT_EQ(pj_ctx_fread(ctx, header, 1, 6, fid), 6U); + ASSERT_TRUE(memcmp(header, "SQLite", 6) == 0); + ASSERT_EQ(pj_ctx_ftell(ctx, fid), 6); + ASSERT_EQ(pj_ctx_fseek(ctx, fid, 0, SEEK_SET), 0); + ASSERT_EQ(pj_ctx_ftell(ctx, fid), 0); + pj_ctx_fclose(ctx, fid); + pj_ctx_free(ctx); + EXPECT_TRUE(spy.gotInMyFOpen); + EXPECT_TRUE(spy.gotInMyFRead); + EXPECT_TRUE(spy.gotInMyFSeek); + EXPECT_TRUE(spy.gotInMyFTell); + EXPECT_TRUE(spy.gotInMyFClose); +} + TEST(pj_transform_test, ob_tran_to_meter_as_srouce) { auto src = pj_init_plus("+ellps=WGS84 +proj=ob_tran +o_proj=latlon " "+o_lon_p=0.0 +o_lat_p=90.0 +lon_0=360.0 " diff --git a/test/unit/proj_context_test.cpp b/test/unit/proj_context_test.cpp index 23c46f29..92f251cd 100644 --- a/test/unit/proj_context_test.cpp +++ b/test/unit/proj_context_test.cpp @@ -40,7 +40,20 @@ namespace { -static std::string createTempDict(std::string &dirname) { +static bool createTmpFile(const std::string &filename) { + FILE *f = fopen(filename.c_str(), "wt"); + if (!f) + return false; + fprintf( + f, + "<MY_PIPELINE> +proj=pipeline +step +proj=utm +zone=31 +ellps=GRS80\n"); + fclose(f); + return true; +} + +// --------------------------------------------------------------------------- + +static std::string createTempDict(std::string &dirname, const char *filename) { const char *temp_dir = getenv("TEMP"); if (!temp_dir) { temp_dir = getenv("TMP"); @@ -58,16 +71,9 @@ static std::string createTempDict(std::string &dirname) { std::string tmpFilename; tmpFilename = temp_dir; tmpFilename += DIR_CHAR; - tmpFilename += "temp_proj_dic"; + tmpFilename += filename; - FILE *f = fopen(tmpFilename.c_str(), "wt"); - if (!f) - return std::string(); - fprintf( - f, - "<MY_PIPELINE> +proj=pipeline +step +proj=utm +zone=31 +ellps=GRS80\n"); - fclose(f); - return tmpFilename; + return createTmpFile(tmpFilename) ? tmpFilename : std::string(); } // --------------------------------------------------------------------------- @@ -85,7 +91,7 @@ static int MyUnlink(const std::string &filename) { TEST(proj_context, proj_context_set_file_finder) { std::string dirname; - auto filename = createTempDict(dirname); + auto filename = createTempDict(dirname, "temp_proj_dic1"); if (filename.empty()) return; @@ -111,7 +117,7 @@ TEST(proj_context, proj_context_set_file_finder) { finderData.dirname = dirname; proj_context_set_file_finder(ctx, finder, &finderData); - auto P = proj_create(ctx, "+init=temp_proj_dic:MY_PIPELINE"); + auto P = proj_create(ctx, "+init=temp_proj_dic1:MY_PIPELINE"); EXPECT_NE(P, nullptr); proj_destroy(P); @@ -125,7 +131,7 @@ TEST(proj_context, proj_context_set_file_finder) { TEST(proj_context, proj_context_set_search_paths) { std::string dirname; - auto filename = createTempDict(dirname); + auto filename = createTempDict(dirname, "temp_proj_dic2"); if (filename.empty()) return; @@ -134,7 +140,7 @@ TEST(proj_context, proj_context_set_search_paths) { const char *path = dirname.c_str(); proj_context_set_search_paths(ctx, 1, &path); - auto P = proj_create(ctx, "+init=temp_proj_dic:MY_PIPELINE"); + auto P = proj_create(ctx, "+init=temp_proj_dic2:MY_PIPELINE"); EXPECT_NE(P, nullptr); proj_destroy(P); @@ -143,4 +149,33 @@ TEST(proj_context, proj_context_set_search_paths) { MyUnlink(filename); } +// --------------------------------------------------------------------------- + +TEST(proj_context, read_grid_from_user_writable_directory) { + + auto ctx = proj_context_create(); + auto path = pj_context_get_user_writable_directory(ctx, true); + EXPECT_TRUE(!path.empty()); + auto filename = path + DIR_CHAR + "temp_proj_dic3"; + EXPECT_TRUE(createTmpFile(filename)); + { + // Check that with PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES (set by + // calling script), we cannot find the file + auto P = proj_create(ctx, "+init=temp_proj_dic3:MY_PIPELINE"); + EXPECT_EQ(P, nullptr); + proj_destroy(P); + } + { + // Cancel the effect of PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY + putenv(const_cast<char *>("PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=")); + auto P = proj_create(ctx, "+init=temp_proj_dic3:MY_PIPELINE"); + EXPECT_NE(P, nullptr); + putenv( + const_cast<char *>("PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES")); + proj_destroy(P); + } + proj_context_destroy(ctx); + MyUnlink(filename); +} + } // namespace diff --git a/test/unit/test_c_api.cpp b/test/unit/test_c_api.cpp index a657c61e..5c6dabba 100644 --- a/test/unit/test_c_api.cpp +++ b/test/unit/test_c_api.cpp @@ -4467,6 +4467,27 @@ TEST_F(CApi, proj_create_derived_geographic_crs) { // --------------------------------------------------------------------------- +TEST_F(CApi, proj_context_set_sqlite3_vfs_name) { + + PJ_CONTEXT *ctx = proj_context_create(); + proj_log_func(ctx, nullptr, [](void *, int, const char *) -> void {}); + + // Set a dummy VFS and check it is taken into account + // (failure to open proj.db) + proj_context_set_sqlite3_vfs_name(ctx, "dummy_vfs_name"); + ASSERT_EQ(proj_create(ctx, "EPSG:4326"), nullptr); + + // Restore default VFS + proj_context_set_sqlite3_vfs_name(ctx, nullptr); + PJ *crs_4326 = proj_create(ctx, "EPSG:4326"); + ASSERT_NE(crs_4326, nullptr); + proj_destroy(crs_4326); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + TEST_F(CApi, proj_is_equivalent_to_with_ctx) { auto from_epsg = proj_create_from_database(m_ctxt, "EPSG", "7844", PJ_CATEGORY_CRS, false, nullptr); diff --git a/test/unit/test_factory.cpp b/test/unit/test_factory.cpp index 8ae58c96..6a88e2b6 100644 --- a/test/unit/test_factory.cpp +++ b/test/unit/test_factory.cpp @@ -1617,34 +1617,34 @@ class FactoryWithTmpDatabase : public ::testing::Test { DatabaseContext::create(m_ctxt), "OTHER"); auto res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, {}); + false, false, {}); EXPECT_EQ(res.size(), 1U); EXPECT_TRUE(res.empty() || nn_dynamic_pointer_cast<ConcatenatedOperation>(res[0])); res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, {std::make_pair(std::string("NS_PIVOT"), - std::string("PIVOT"))}); + false, false, {std::make_pair(std::string("NS_PIVOT"), + std::string("PIVOT"))}); EXPECT_EQ(res.size(), 1U); EXPECT_TRUE(res.empty() || nn_dynamic_pointer_cast<ConcatenatedOperation>(res[0])); res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, {std::make_pair(std::string("NS_PIVOT"), - std::string("NOT_EXISTING"))}); + false, false, {std::make_pair(std::string("NS_PIVOT"), + std::string("NOT_EXISTING"))}); EXPECT_EQ(res.size(), 0U); res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, + false, false, {std::make_pair(std::string("BAD_NS"), std::string("PIVOT"))}); EXPECT_EQ(res.size(), 0U); res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_TARGET", "TARGET", "NS_SOURCE", "SOURCE", false, false, - false, {}); + false, false, {}); EXPECT_EQ(res.size(), 1U); EXPECT_TRUE(res.empty() || nn_dynamic_pointer_cast<ConcatenatedOperation>(res[0])); @@ -1654,7 +1654,7 @@ class FactoryWithTmpDatabase : public ::testing::Test { DatabaseContext::create(m_ctxt), std::string()); auto res = factory->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, {}); + false, false, {}); EXPECT_EQ(res.size(), 1U); EXPECT_TRUE(res.empty() || nn_dynamic_pointer_cast<ConcatenatedOperation>(res[0])); @@ -1833,7 +1833,7 @@ TEST(factory, AuthorityFactory_createFromCoordinateReferenceSystemCodes) { { // Test removal of superseded transform auto list = factory->createFromCoordinateReferenceSystemCodes( - "EPSG", "4179", "EPSG", "4258", false, false, true); + "EPSG", "4179", "EPSG", "4258", false, false, false, true); ASSERT_EQ(list.size(), 2U); // Romania has a larger area than Poland (given our approx formula) EXPECT_EQ(list[0]->getEPSGCode(), 15994); // Romania - 3m @@ -1851,12 +1851,12 @@ TEST( { auto res = factory->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "32631", false, false, false); + "EPSG", "4326", "EPSG", "32631", false, false, false, false); ASSERT_EQ(res.size(), 1U); } { auto res = factory->createFromCoordinateReferenceSystemCodes( - "EPSG", "4209", "EPSG", "4326", false, false, false); + "EPSG", "4209", "EPSG", "4326", false, false, false, false); EXPECT_TRUE(!res.empty()); for (const auto &conv : res) { EXPECT_TRUE(conv->sourceCRS()->getEPSGCode() == 4209); @@ -1889,7 +1889,8 @@ TEST_F(FactoryWithTmpDatabase, DatabaseContext::create(m_ctxt), std::string()); { auto res = factoryGeneral->createFromCoordinateReferenceSystemCodes( - "OTHER", "OTHER_4326", "OTHER", "OTHER_32631", false, false, false); + "OTHER", "OTHER_4326", "OTHER", "OTHER_32631", false, false, false, + false); ASSERT_EQ(res.size(), 1U); } @@ -1897,7 +1898,8 @@ TEST_F(FactoryWithTmpDatabase, AuthorityFactory::create(DatabaseContext::create(m_ctxt), "EPSG"); { auto res = factoryEPSG->createFromCoordinateReferenceSystemCodes( - "OTHER", "OTHER_4326", "OTHER", "OTHER_32631", false, false, false); + "OTHER", "OTHER_4326", "OTHER", "OTHER_32631", false, false, false, + false); ASSERT_EQ(res.size(), 1U); } @@ -1919,17 +1921,17 @@ TEST_F(FactoryWithTmpDatabase, << last_error(); { auto res = factoryGeneral->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false); + "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false, false); ASSERT_EQ(res.size(), 1U); } { auto res = factoryEPSG->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false); + "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false, false); ASSERT_EQ(res.size(), 0U); } { auto res = factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false); + "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false, false); ASSERT_EQ(res.size(), 1U); } } @@ -1982,7 +1984,7 @@ TEST_F(FactoryWithTmpDatabase, auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); auto res = factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false); + "EPSG", "4326", "EPSG", "4326", false, false, false, false); ASSERT_EQ(res.size(), 3U); EXPECT_EQ(*(res[0]->name()->description()), "TRANSFORMATION_1M"); EXPECT_EQ(*(res[1]->name()->description()), "TRANSFORMATION_10M"); @@ -2001,7 +2003,7 @@ TEST_F( auto factory = AuthorityFactory::create(DatabaseContext::create(m_ctxt), std::string()); auto res = factory->createFromCRSCodesWithIntermediates( - "EPSG", "4326", "EPSG", "4326", false, false, false, {}); + "EPSG", "4326", "EPSG", "4326", false, false, false, false, {}); EXPECT_EQ(res.size(), 0U); } @@ -2085,7 +2087,7 @@ TEST_F(FactoryWithTmpDatabase, AuthorityFactory_proj_based_transformation) { auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); auto res = factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false); + "EPSG", "4326", "EPSG", "4326", false, false, false, false); ASSERT_EQ(res.size(), 1U); EXPECT_EQ(res[0]->nameStr(), "My PROJ string based op"); EXPECT_EQ(res[0]->exportToPROJString(PROJStringFormatter::create().get()), @@ -2146,7 +2148,7 @@ TEST_F(FactoryWithTmpDatabase, AuthorityFactory_wkt_based_transformation) { auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); auto res = factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false); + "EPSG", "4326", "EPSG", "4326", false, false, false, false); ASSERT_EQ(res.size(), 1U); EXPECT_EQ(res[0]->nameStr(), "My WKT string based op"); EXPECT_EQ(res[0]->exportToPROJString(PROJStringFormatter::create().get()), @@ -2180,9 +2182,10 @@ TEST_F(FactoryWithTmpDatabase, auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); - EXPECT_THROW(factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false), - FactoryException); + EXPECT_THROW( + factoryOTHER->createFromCoordinateReferenceSystemCodes( + "EPSG", "4326", "EPSG", "4326", false, false, false, false), + FactoryException); } // --------------------------------------------------------------------------- @@ -2207,9 +2210,10 @@ TEST_F(FactoryWithTmpDatabase, auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); - EXPECT_THROW(factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false), - FactoryException); + EXPECT_THROW( + factoryOTHER->createFromCoordinateReferenceSystemCodes( + "EPSG", "4326", "EPSG", "4326", false, false, false, false), + FactoryException); } // --------------------------------------------------------------------------- @@ -2262,7 +2266,7 @@ TEST_F(FactoryWithTmpDatabase, lookForGridInfo) { bool openLicense = false; bool gridAvailable = false; EXPECT_TRUE(DatabaseContext::create(m_ctxt)->lookForGridInfo( - "PROJ_fake_grid", fullFilename, packageName, url, directDownload, + "PROJ_fake_grid", false, fullFilename, packageName, url, directDownload, openLicense, gridAvailable)); EXPECT_TRUE(fullFilename.empty()); EXPECT_TRUE(packageName.empty()); diff --git a/test/unit/test_grids.cpp b/test/unit/test_grids.cpp new file mode 100644 index 00000000..5240949e --- /dev/null +++ b/test/unit/test_grids.cpp @@ -0,0 +1,227 @@ +/****************************************************************************** + * + * Project: PROJ + * Purpose: Test grids.hpp + * Author: Even Rouault <even dot rouault at spatialys dot com> + * + ****************************************************************************** + * Copyright (c) 2020, Even Rouault <even dot rouault at spatialys dot com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "gtest_include.h" + +#include "grids.hpp" + +#include "proj_internal.h" // M_PI + +namespace { + +// --------------------------------------------------------------------------- + +class GridTest : public ::testing::Test { + + static void DummyLogFunction(void *, int, const char *) {} + + protected: + void SetUp() override { + m_ctxt = proj_context_create(); + proj_log_func(m_ctxt, nullptr, DummyLogFunction); + m_ctxt2 = proj_context_create(); + proj_log_func(m_ctxt2, nullptr, DummyLogFunction); + } + + void TearDown() override { + proj_context_destroy(m_ctxt); + proj_context_destroy(m_ctxt2); + } + + PJ_CONTEXT *m_ctxt = nullptr; + PJ_CONTEXT *m_ctxt2 = nullptr; +}; + +// --------------------------------------------------------------------------- + +TEST_F(GridTest, VerticalShiftGridSet_null) { + auto gridSet = NS_PROJ::VerticalShiftGridSet::open(m_ctxt, "null"); + ASSERT_NE(gridSet, nullptr); + auto grid = gridSet->gridAt(0.0, 0.0); + ASSERT_NE(grid, nullptr); + EXPECT_EQ(grid->width(), 3); + EXPECT_EQ(grid->height(), 3); + EXPECT_EQ(grid->extentAndRes().westLon, -M_PI); + EXPECT_TRUE(grid->isNullGrid()); + EXPECT_FALSE(grid->hasChanged()); + float out = -1.0f; + EXPECT_TRUE(grid->valueAt(0, 0, out)); + EXPECT_EQ(out, 0.0f); + EXPECT_FALSE(grid->isNodata(0.0f, 0.0)); + gridSet->reassign_context(m_ctxt2); + gridSet->reopen(m_ctxt2); +} + +// --------------------------------------------------------------------------- + +TEST_F(GridTest, VerticalShiftGridSet_gtx) { + ASSERT_EQ(NS_PROJ::VerticalShiftGridSet::open(m_ctxt, "foobar"), nullptr); + auto gridSet = + NS_PROJ::VerticalShiftGridSet::open(m_ctxt, "tests/test_nodata.gtx"); + ASSERT_NE(gridSet, nullptr); + ASSERT_EQ(gridSet->gridAt(-100, -100), nullptr); + auto grid = gridSet->gridAt(4.15 / 180 * M_PI, 52.15 / 180 * M_PI); + ASSERT_NE(grid, nullptr); + EXPECT_TRUE(grid->isNodata(-88.8888f, 1.0)); + gridSet->reassign_context(m_ctxt2); + gridSet->reopen(m_ctxt2); + grid = gridSet->gridAt(4.15 / 180 * M_PI, 52.15 / 180 * M_PI); + EXPECT_NE(grid, nullptr); +} + +// --------------------------------------------------------------------------- + +TEST_F(GridTest, HorizontalShiftGridSet_null) { + auto gridSet = NS_PROJ::HorizontalShiftGridSet::open(m_ctxt, "null"); + ASSERT_NE(gridSet, nullptr); + auto grid = gridSet->gridAt(0.0, 0.0); + ASSERT_NE(grid, nullptr); + EXPECT_EQ(grid->width(), 3); + EXPECT_EQ(grid->height(), 3); + EXPECT_EQ(grid->extentAndRes().westLon, -M_PI); + EXPECT_TRUE(grid->isNullGrid()); + EXPECT_FALSE(grid->hasChanged()); + float out1 = -1.0f; + float out2 = -1.0f; + EXPECT_TRUE(grid->valueAt(0, 0, false, out1, out2)); + EXPECT_EQ(out1, 0.0f); + EXPECT_EQ(out2, 0.0f); + gridSet->reassign_context(m_ctxt2); + gridSet->reopen(m_ctxt2); +} + +// --------------------------------------------------------------------------- + +TEST_F(GridTest, HorizontalShiftGridSet_gtiff) { + auto gridSet = + NS_PROJ::HorizontalShiftGridSet::open(m_ctxt, "tests/test_hgrid.tif"); + ASSERT_NE(gridSet, nullptr); + EXPECT_EQ(gridSet->format(), "gtiff"); + EXPECT_TRUE(gridSet->name().find("tests/test_hgrid.tif") != + std::string::npos) + << gridSet->name(); + EXPECT_EQ(gridSet->grids().size(), 1U); + ASSERT_EQ(gridSet->gridAt(-100, -100), nullptr); + auto grid = gridSet->gridAt(5.5 / 180 * M_PI, 53.5 / 180 * M_PI); + ASSERT_NE(grid, nullptr); + EXPECT_EQ(grid->width(), 4); + EXPECT_EQ(grid->height(), 4); + EXPECT_EQ(grid->extentAndRes().westLon, 4.0 / 180 * M_PI); + EXPECT_FALSE(grid->isNullGrid()); + EXPECT_FALSE(grid->hasChanged()); + float out1 = -1.0f; + float out2 = -1.0f; + EXPECT_TRUE(grid->valueAt(0, 3, false, out1, out2)); + EXPECT_EQ(out1, static_cast<float>(14400.0 / 3600. / 180 * M_PI)); + EXPECT_EQ(out2, static_cast<float>(900.0 / 3600. / 180 * M_PI)); + gridSet->reassign_context(m_ctxt2); + gridSet->reopen(m_ctxt2); + grid = gridSet->gridAt(5.5 / 180 * M_PI, 53.5 / 180 * M_PI); + EXPECT_NE(grid, nullptr); +} + +// --------------------------------------------------------------------------- + +TEST_F(GridTest, GenericShiftGridSet_null) { + auto gridSet = NS_PROJ::GenericShiftGridSet::open(m_ctxt, "null"); + ASSERT_NE(gridSet, nullptr); + auto grid = gridSet->gridAt(0.0, 0.0); + ASSERT_NE(grid, nullptr); + EXPECT_EQ(grid->width(), 3); + EXPECT_EQ(grid->height(), 3); + EXPECT_EQ(grid->extentAndRes().westLon, -M_PI); + EXPECT_TRUE(grid->isNullGrid()); + EXPECT_FALSE(grid->hasChanged()); + float out = -1.0f; + EXPECT_TRUE(grid->valueAt(0, 0, 0, out)); + EXPECT_EQ(out, 0.0f); + EXPECT_EQ(grid->unit(0), ""); + EXPECT_EQ(grid->description(0), ""); + EXPECT_EQ(grid->metadataItem("foo"), ""); + EXPECT_EQ(grid->samplesPerPixel(), 0); + gridSet->reassign_context(m_ctxt2); + gridSet->reopen(m_ctxt2); +} + +// --------------------------------------------------------------------------- + +TEST_F(GridTest, GenericShiftGridSet_gtiff) { + ASSERT_EQ(NS_PROJ::GenericShiftGridSet::open(m_ctxt, "foobar"), nullptr); + auto gridSet = NS_PROJ::GenericShiftGridSet::open( + m_ctxt, "tests/nkgrf03vel_realigned_extract.tif"); + ASSERT_NE(gridSet, nullptr); + ASSERT_EQ(gridSet->gridAt(-100, -100), nullptr); + auto grid = gridSet->gridAt(21.3333333 / 180 * M_PI, 63.0 / 180 * M_PI); + ASSERT_NE(grid, nullptr); + EXPECT_EQ(grid->width(), 5); + EXPECT_EQ(grid->height(), 5); + EXPECT_EQ(grid->extentAndRes().westLon, 21.0 / 180 * M_PI); + EXPECT_FALSE(grid->isNullGrid()); + EXPECT_FALSE(grid->hasChanged()); + float out = -1.0f; + EXPECT_FALSE(grid->valueAt(0, 0, grid->samplesPerPixel(), out)); + EXPECT_EQ(grid->metadataItem("area_of_use"), "Nordic and Baltic countries"); + EXPECT_EQ(grid->metadataItem("non_existing"), std::string()); + EXPECT_EQ(grid->metadataItem("non_existing", 1), std::string()); + EXPECT_EQ(grid->metadataItem("non_existing", 10), std::string()); + gridSet->reassign_context(m_ctxt2); + gridSet->reopen(m_ctxt2); + grid = gridSet->gridAt(21.3333333 / 180 * M_PI, 63.0 / 180 * M_PI); + EXPECT_NE(grid, nullptr); +} + +// --------------------------------------------------------------------------- + +TEST_F(GridTest, GenericShiftGridSet_gtiff_with_subgrid) { + auto gridSet = NS_PROJ::GenericShiftGridSet::open( + m_ctxt, "tests/test_hgrid_with_subgrid.tif"); + ASSERT_NE(gridSet, nullptr); + ASSERT_EQ(gridSet->gridAt(-100, -100), nullptr); + auto grid = + gridSet->gridAt(-115.5416667 / 180 * M_PI, 51.1666667 / 180 * M_PI); + ASSERT_NE(grid, nullptr); + EXPECT_EQ(grid->width(), 11); + EXPECT_EQ(grid->height(), 21); + EXPECT_EQ(grid->metadataItem("grid_name"), "ALbanff"); +} + +// --------------------------------------------------------------------------- + +TEST_F(GridTest, + GenericShiftGridSet_gtiff_with_two_level_of_subgrids_no_grid_name) { + auto gridSet = NS_PROJ::GenericShiftGridSet::open( + m_ctxt, "tests/test_hgrid_with_two_level_of_subgrids_no_grid_name.tif"); + ASSERT_NE(gridSet, nullptr); + ASSERT_EQ(gridSet->gridAt(-100, -100), nullptr); + auto grid = gridSet->gridAt(-45.5 / 180 * M_PI, 22.5 / 180 * M_PI); + ASSERT_NE(grid, nullptr); + EXPECT_EQ(grid->width(), 8); + EXPECT_EQ(grid->height(), 8); +} + +} // namespace diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp new file mode 100644 index 00000000..241e6bbc --- /dev/null +++ b/test/unit/test_network.cpp @@ -0,0 +1,1856 @@ +/****************************************************************************** + * + * Project: PROJ + * Purpose: Test networking + * Author: Even Rouault <even dot rouault at spatialys dot com> + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault <even dot rouault at spatialys dot com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "gtest_include.h" + +#include <memory> +#include <stdio.h> +#include <stdlib.h> + +#include "proj_internal.h" +#include <proj.h> + +#include <sqlite3.h> +#include <time.h> + +#ifdef _WIN32 +#include <windows.h> +#else +#include <unistd.h> +#endif + +#ifdef CURL_ENABLED +#include <curl/curl.h> +#endif + +namespace { + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +static bool networkAccessOK = false; + +static size_t noop_curl_write_func(void *, size_t, size_t nmemb, void *) { + return nmemb; +} + +TEST(networking, initial_check) { + CURL *hCurlHandle = curl_easy_init(); + if (!hCurlHandle) + return; + curl_easy_setopt(hCurlHandle, CURLOPT_URL, + "https://cdn.proj.org/ntf_r93.tif"); + + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, "0-1"); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, noop_curl_write_func); + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_cleanup(hCurlHandle); + + networkAccessOK = (response_code == 206); + if (!networkAccessOK) { + fprintf(stderr, "network access not working"); + } +} + +#endif + +// --------------------------------------------------------------------------- + +static void silent_logger(void *, int, const char *) {} + +// --------------------------------------------------------------------------- + +TEST(networking, basic) { + const char *pipeline = + "+proj=pipeline " + "+step +proj=unitconvert +xy_in=deg +xy_out=rad " + "+step +proj=hgridshift +grids=https://cdn.proj.org/ntf_r93.tif " + "+step +proj=unitconvert +xy_in=rad +xy_out=deg"; + + // network access disabled by default + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_log_func(ctx, nullptr, silent_logger); + auto P = proj_create(ctx, pipeline); + ASSERT_EQ(P, nullptr); + proj_context_destroy(ctx); + +#ifdef CURL_ENABLED + // enable through env variable + ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + putenv(const_cast<char *>("PROJ_NETWORK=ON")); + P = proj_create(ctx, pipeline); + if (networkAccessOK) { + ASSERT_NE(P, nullptr); + } + proj_destroy(P); + proj_context_destroy(ctx); + putenv(const_cast<char *>("PROJ_NETWORK=")); +#endif + + // still disabled + ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_log_func(ctx, nullptr, silent_logger); + P = proj_create(ctx, pipeline); + ASSERT_EQ(P, nullptr); + proj_context_destroy(ctx); + + // enable through API + ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + P = proj_create(ctx, pipeline); +#ifdef CURL_ENABLED + if (networkAccessOK) { + ASSERT_NE(P, nullptr); + } else { + ASSERT_EQ(P, nullptr); + proj_context_destroy(ctx); + return; + } + double lon = 2; + double lat = 49; + proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, sizeof(double), + 1, nullptr, 0, 0, nullptr, 0, 0); + EXPECT_NEAR(lon, 1.9992776848, 1e-10); + EXPECT_NEAR(lat, 48.9999322600, 1e-10); + + proj_destroy(P); +#else + ASSERT_EQ(P, nullptr); +#endif + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, curl_invalid_resource) { + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + proj_log_func(ctx, nullptr, silent_logger); + auto P = proj_create( + ctx, "+proj=hgridshift +grids=https://i_do_not.exist/my.tif"); + proj_context_destroy(ctx); + ASSERT_EQ(P, nullptr); +} +#endif + +// --------------------------------------------------------------------------- + +struct Event { + virtual ~Event(); + std::string type{}; + PJ_CONTEXT *ctx = nullptr; +}; + +Event::~Event() = default; + +struct OpenEvent : public Event { + OpenEvent() { type = "OpenEvent"; } + + std::string url{}; + unsigned long long offset = 0; + size_t size_to_read = 0; + std::vector<unsigned char> response{}; + std::string errorMsg{}; + int file_id = 0; +}; + +struct CloseEvent : public Event { + CloseEvent() { type = "CloseEvent"; } + + int file_id = 0; +}; + +struct GetHeaderValueEvent : public Event { + GetHeaderValueEvent() { type = "GetHeaderValueEvent"; } + + int file_id = 0; + std::string key{}; + std::string value{}; +}; + +struct ReadRangeEvent : public Event { + ReadRangeEvent() { type = "ReadRangeEvent"; } + + unsigned long long offset = 0; + size_t size_to_read = 0; + std::vector<unsigned char> response{}; + std::string errorMsg{}; + int file_id = 0; +}; + +struct File {}; + +struct ExchangeWithCallback { + std::vector<std::unique_ptr<Event>> events{}; + size_t nextEvent = 0; + bool error = false; + std::map<int, PROJ_NETWORK_HANDLE *> mapIdToHandle{}; + + bool allConsumedAndNoError() const { + return nextEvent == events.size() && !error; + } +}; + +static PROJ_NETWORK_HANDLE *open_cbk(PJ_CONTEXT *ctx, const char *url, + unsigned long long offset, + size_t size_to_read, void *buffer, + size_t *out_size_read, + size_t error_string_max_size, + char *out_error_string, void *user_data) { + auto exchange = static_cast<ExchangeWithCallback *>(user_data); + if (exchange->error) + return nullptr; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to open(%s, %ld, %ld)\n", url, + (long)offset, (long)size_to_read); + exchange->error = true; + return nullptr; + } + auto openEvent = + dynamic_cast<OpenEvent *>(exchange->events[exchange->nextEvent].get()); + if (!openEvent) { + fprintf(stderr, "unexpected call to open(%s, %ld, %ld). " + "Was expecting a %s event\n", + url, (long)offset, (long)size_to_read, + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return nullptr; + } + exchange->nextEvent++; + if (openEvent->ctx != ctx || openEvent->url != url || + openEvent->offset != offset || + openEvent->size_to_read != size_to_read) { + fprintf(stderr, "wrong call to open(%s, %ld, %ld). Was expecting " + "open(%s, %ld, %ld)\n", + url, (long)offset, (long)size_to_read, openEvent->url.c_str(), + (long)openEvent->offset, (long)openEvent->size_to_read); + exchange->error = true; + return nullptr; + } + if (!openEvent->errorMsg.empty()) { + snprintf(out_error_string, error_string_max_size, "%s", + openEvent->errorMsg.c_str()); + return nullptr; + } + + memcpy(buffer, openEvent->response.data(), openEvent->response.size()); + *out_size_read = openEvent->response.size(); + auto handle = reinterpret_cast<PROJ_NETWORK_HANDLE *>(new File()); + exchange->mapIdToHandle[openEvent->file_id] = handle; + return handle; +} + +static void close_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle, + void *user_data) { + auto exchange = static_cast<ExchangeWithCallback *>(user_data); + if (exchange->error) + return; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to close()\n"); + exchange->error = true; + return; + } + auto closeEvent = + dynamic_cast<CloseEvent *>(exchange->events[exchange->nextEvent].get()); + if (!closeEvent) { + fprintf(stderr, "unexpected call to close(). " + "Was expecting a %s event\n", + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return; + } + if (closeEvent->ctx != ctx) { + fprintf(stderr, "close() called with bad context\n"); + exchange->error = true; + return; + } + if (exchange->mapIdToHandle[closeEvent->file_id] != handle) { + fprintf(stderr, "close() called with bad handle\n"); + exchange->error = true; + return; + } + exchange->nextEvent++; + delete reinterpret_cast<File *>(handle); +} + +static const char *get_header_value_cbk(PJ_CONTEXT *ctx, + PROJ_NETWORK_HANDLE *handle, + const char *header_name, + void *user_data) { + auto exchange = static_cast<ExchangeWithCallback *>(user_data); + if (exchange->error) + return nullptr; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to get_header_value()\n"); + exchange->error = true; + return nullptr; + } + auto getHeaderValueEvent = dynamic_cast<GetHeaderValueEvent *>( + exchange->events[exchange->nextEvent].get()); + if (!getHeaderValueEvent) { + fprintf(stderr, "unexpected call to get_header_value(). " + "Was expecting a %s event\n", + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return nullptr; + } + if (getHeaderValueEvent->ctx != ctx) { + fprintf(stderr, "get_header_value() called with bad context\n"); + exchange->error = true; + return nullptr; + } + if (getHeaderValueEvent->key != header_name) { + fprintf(stderr, "wrong call to get_header_value(%s). Was expecting " + "get_header_value(%s)\n", + header_name, getHeaderValueEvent->key.c_str()); + exchange->error = true; + return nullptr; + } + if (exchange->mapIdToHandle[getHeaderValueEvent->file_id] != handle) { + fprintf(stderr, "get_header_value() called with bad handle\n"); + exchange->error = true; + return nullptr; + } + exchange->nextEvent++; + return getHeaderValueEvent->value.c_str(); +} + +static size_t read_range_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle, + unsigned long long offset, size_t size_to_read, + void *buffer, size_t error_string_max_size, + char *out_error_string, void *user_data) { + auto exchange = static_cast<ExchangeWithCallback *>(user_data); + if (exchange->error) + return 0; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to read_range(%ld, %ld)\n", + (long)offset, (long)size_to_read); + exchange->error = true; + return 0; + } + auto readRangeEvent = dynamic_cast<ReadRangeEvent *>( + exchange->events[exchange->nextEvent].get()); + if (!readRangeEvent) { + fprintf(stderr, "unexpected call to read_range(). " + "Was expecting a %s event\n", + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return 0; + } + if (exchange->mapIdToHandle[readRangeEvent->file_id] != handle) { + fprintf(stderr, "read_range() called with bad handle\n"); + exchange->error = true; + return 0; + } + if (readRangeEvent->ctx != ctx || readRangeEvent->offset != offset || + readRangeEvent->size_to_read != size_to_read) { + fprintf(stderr, "wrong call to read_range(%ld, %ld). Was expecting " + "read_range(%ld, %ld)\n", + (long)offset, (long)size_to_read, (long)readRangeEvent->offset, + (long)readRangeEvent->size_to_read); + exchange->error = true; + return 0; + } + exchange->nextEvent++; + if (!readRangeEvent->errorMsg.empty()) { + snprintf(out_error_string, error_string_max_size, "%s", + readRangeEvent->errorMsg.c_str()); + return 0; + } + memcpy(buffer, readRangeEvent->response.data(), + readRangeEvent->response.size()); + return readRangeEvent->response.size(); +} + +TEST(networking, custom) { + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/my.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create( + ctx, "+proj=vgridshift +grids=https://foo/my.tif +multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/my.tif"; + event->offset = 524288; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 1.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr<ReadRangeEvent> event(new ReadRangeEvent()); + event->ctx = ctx; + event->offset = 3670016; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 2.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = -49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 2.25); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + // Once again ! No network access + + P = proj_create(ctx, + "+proj=vgridshift +grids=https://foo/my.tif +multiplier=1"); + ASSERT_NE(P, nullptr); + + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, getfilesize) { + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/getfilesize.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/test_vgrid_single_strip_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 550, f), 550U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes 0-16383/4153510"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create( + ctx, + "+proj=vgridshift +grids=https://foo/getfilesize.tif +multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_destroy(P); + + P = proj_create( + ctx, + "+proj=vgridshift +grids=https://foo/getfilesize.tif +multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_destroy(P); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, simul_open_error) { + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_log_func(ctx, nullptr, silent_logger); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/open_error.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->errorMsg = "Cannot open file"; + event->file_id = 1; + + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create( + ctx, + "+proj=vgridshift +grids=https://foo/open_error.tif +multiplier=1"); + + ASSERT_EQ(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, simul_read_range_error) { + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/read_range_error.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create(ctx, "+proj=vgridshift " + "+grids=https://foo/read_range_error.tif " + "+multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/read_range_error.tif"; + event->offset = 524288; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 1.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr<ReadRangeEvent> event(new ReadRangeEvent()); + event->ctx = ctx; + event->offset = 3670016; + event->size_to_read = 278528; + event->errorMsg = "read range error"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = -49 / 180. * M_PI; + double z = 0; + proj_log_func(ctx, nullptr, silent_logger); + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, HUGE_VAL); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, simul_file_change_while_opened) { + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/file_change_while_opened.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create(ctx, "+proj=vgridshift " + "+grids=https://foo/file_change_while_opened.tif " + "+multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/file_change_while_opened.tif"; + event->offset = 524288; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 1.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date CHANGED!!!!"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/file_change_while_opened.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 3; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 3; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date CHANGED!!!!"; + event->file_id = 3; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 3; + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 3; + exchange.events.emplace_back(std::move(event)); + } + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, curl_hgridshift) { + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + + // NAD83 to NAD83(HARN) in West-Virginia. Using wvhpgn.tif + auto P = proj_create_crs_to_crs(ctx, "EPSG:4269", "EPSG:4152", nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = 40; // lat + c.xyz.y = -80; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + + proj_assign_context(P, ctx); // (dummy) test context reassignment + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_NEAR(c.xyz.x, 39.99999839, 1e-8); + EXPECT_NEAR(c.xyz.y, -79.99999807, 1e-8); + EXPECT_NEAR(c.xyz.z, 0, 1e-2); +} + +#endif + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, curl_vgridshift) { + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + + // WGS84 to EGM2008 height. Using egm08_25.tif + auto P = + proj_create_crs_to_crs(ctx, "EPSG:4326", "EPSG:4326+3855", nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = -30; // lat + c.xyz.y = 150; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + + proj_assign_context(P, ctx); // (dummy) test context reassignment + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_NEAR(c.xyz.x, -30, 1e-8); + EXPECT_NEAR(c.xyz.y, 150, 1e-8); + EXPECT_NEAR(c.xyz.z, -31.89, 1e-2); +} + +#endif + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, curl_vgridshift_vertcon) { + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + + // NGVD29 to NAVD88 height. Using vertcone.tif + auto P = proj_create_crs_to_crs(ctx, "EPSG:4269+7968", "EPSG:4269+5703", + nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = 40; // lat + c.xyz.y = -80; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_NEAR(c.xyz.x, 40, 1e-8); + EXPECT_NEAR(c.xyz.y, -80, 1e-8); + EXPECT_NEAR(c.xyz.z, -0.15, 1e-2); +} + +#endif + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, network_endpoint_env_variable) { + putenv(const_cast<char *>("PROJ_NETWORK_ENDPOINT=http://0.0.0.0/")); + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + + // NAD83 to NAD83(HARN) in West-Virginia. Using wvhpgn.tif + auto P = proj_create_crs_to_crs(ctx, "EPSG:4269", "EPSG:4152", nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = 40; // lat + c.xyz.y = -80; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + putenv(const_cast<char *>("PROJ_NETWORK_ENDPOINT=")); + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_EQ(c.xyz.x, HUGE_VAL); +} + +#endif + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, network_endpoint_api) { + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + proj_context_set_url_endpoint(ctx, "http://0.0.0.0"); + + // NAD83 to NAD83(HARN) in West-Virginia. Using wvhpgn.tif + auto P = proj_create_crs_to_crs(ctx, "EPSG:4269", "EPSG:4152", nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = 40; // lat + c.xyz.y = -80; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_EQ(c.xyz.x, HUGE_VAL); +} + +#endif + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +static PROJ_NETWORK_HANDLE *dummy_open_cbk(PJ_CONTEXT *, const char *, + unsigned long long, size_t, void *, + size_t *, size_t, char *, void *) { + assert(false); + return nullptr; +} + +static void dummy_close_cbk(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, void *) { + assert(false); +} + +static const char *dummy_get_header_value_cbk(PJ_CONTEXT *, + PROJ_NETWORK_HANDLE *, + const char *, void *) { + assert(false); + return nullptr; +} + +static size_t dummy_read_range_cbk(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, + unsigned long long, size_t, void *, size_t, + char *, void *) { + assert(false); + return 0; +} + +TEST(networking, cache_basic) { + if (!networkAccessOK) { + return; + } + + proj_cleanup(); + + const char *pipeline = + "+proj=pipeline " + "+step +proj=unitconvert +xy_in=deg +xy_out=rad " + "+step +proj=hgridshift +grids=https://cdn.proj.org/ntf_r93.tif " + "+step +proj=unitconvert +xy_in=rad +xy_out=deg"; + + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + auto P = proj_create(ctx, pipeline); + ASSERT_NE(P, nullptr); + proj_destroy(P); + + EXPECT_TRUE(!pj_context_get_grid_cache_filename(ctx).empty()); + + sqlite3 *hDB = nullptr; + sqlite3_open_v2(pj_context_get_grid_cache_filename(ctx).c_str(), &hDB, + SQLITE_OPEN_READONLY, nullptr); + ASSERT_NE(hDB, nullptr); + sqlite3_stmt *hStmt = nullptr; + sqlite3_prepare_v2(hDB, "SELECT url, offset FROM chunks WHERE id = (" + "SELECT chunk_id FROM linked_chunks WHERE id = (" + "SELECT head FROM linked_chunks_head_tail))", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_ROW); + const char *url = + reinterpret_cast<const char *>(sqlite3_column_text(hStmt, 0)); + ASSERT_NE(url, nullptr); + ASSERT_EQ(std::string(url), "https://cdn.proj.org/ntf_r93.tif"); + ASSERT_EQ(sqlite3_column_int64(hStmt, 1), 0); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + + proj_cleanup(); + + // Check that a second access doesn't trigger any network activity + ASSERT_TRUE(proj_context_set_network_callbacks( + ctx, dummy_open_cbk, dummy_close_cbk, dummy_get_header_value_cbk, + dummy_read_range_cbk, nullptr)); + P = proj_create(ctx, pipeline); + ASSERT_NE(P, nullptr); + proj_destroy(P); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, proj_grid_cache_clear) { + if (!networkAccessOK) { + return; + } + const char *pipeline = + "+proj=pipeline " + "+step +proj=unitconvert +xy_in=deg +xy_out=rad " + "+step +proj=hgridshift +grids=https://cdn.proj.org/ntf_r93.tif " + "+step +proj=unitconvert +xy_in=rad +xy_out=deg"; + + proj_cleanup(); + + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + proj_grid_cache_set_filename(ctx, "tmp_proj_db_cache.db"); + EXPECT_EQ(pj_context_get_grid_cache_filename(ctx), + std::string("tmp_proj_db_cache.db")); + + proj_grid_cache_clear(ctx); + + auto P = proj_create(ctx, pipeline); + ASSERT_NE(P, nullptr); + proj_destroy(P); + + // Check that the file exists + { + sqlite3 *hDB = nullptr; + ASSERT_EQ( + sqlite3_open_v2(pj_context_get_grid_cache_filename(ctx).c_str(), + &hDB, SQLITE_OPEN_READONLY, nullptr), + SQLITE_OK); + sqlite3_close(hDB); + } + + proj_grid_cache_clear(ctx); + + // Check that the file no longer exists + { + sqlite3 *hDB = nullptr; + ASSERT_NE( + sqlite3_open_v2(pj_context_get_grid_cache_filename(ctx).c_str(), + &hDB, SQLITE_OPEN_READONLY, nullptr), + SQLITE_OK); + sqlite3_close(hDB); + } + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, cache_saturation) { + if (!networkAccessOK) { + return; + } + const char *pipeline = + "+proj=pipeline " + "+step +proj=unitconvert +xy_in=deg +xy_out=rad " + "+step +proj=hgridshift +grids=https://cdn.proj.org/ntf_r93.tif " + "+step +proj=unitconvert +xy_in=rad +xy_out=deg"; + + proj_cleanup(); + + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + proj_grid_cache_set_filename(ctx, "tmp_proj_db_cache.db"); + + proj_grid_cache_clear(ctx); + + // Limit to two chunks + putenv(const_cast<char *>("PROJ_GRID_CACHE_MAX_SIZE_BYTES=32768")); + proj_grid_cache_set_max_size(ctx, 0); + putenv(const_cast<char *>("PROJ_GRID_CACHE_MAX_SIZE_BYTES=")); + + auto P = proj_create(ctx, pipeline); + ASSERT_NE(P, nullptr); + + double lon = 2; + double lat = 49; + proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, sizeof(double), + 1, nullptr, 0, 0, nullptr, 0, 0); + EXPECT_NEAR(lon, 1.9992776848, 1e-10); + EXPECT_NEAR(lat, 48.9999322600, 1e-10); + + proj_destroy(P); + + sqlite3 *hDB = nullptr; + sqlite3_open_v2(pj_context_get_grid_cache_filename(ctx).c_str(), &hDB, + SQLITE_OPEN_READONLY, nullptr); + ASSERT_NE(hDB, nullptr); + + sqlite3_stmt *hStmt = nullptr; + sqlite3_prepare_v2(hDB, "SELECT COUNT(*) FROM chunk_data UNION ALL " + "SELECT COUNT(*) FROM chunks UNION ALL " + "SELECT COUNT(*) FROM linked_chunks", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(hStmt, 0), 2); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(hStmt, 0), 2); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(hStmt, 0), 2); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + + proj_grid_cache_clear(ctx); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, cache_ttl) { + if (!networkAccessOK) { + return; + } + const char *pipeline = + "+proj=pipeline " + "+step +proj=unitconvert +xy_in=deg +xy_out=rad " + "+step +proj=hgridshift +grids=https://cdn.proj.org/ntf_r93.tif " + "+step +proj=unitconvert +xy_in=rad +xy_out=deg"; + + proj_cleanup(); + + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + proj_grid_cache_set_filename(ctx, "tmp_proj_db_cache.db"); + + proj_grid_cache_clear(ctx); + + auto P = proj_create(ctx, pipeline); + ASSERT_NE(P, nullptr); + + double lon = 2; + double lat = 49; + proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, sizeof(double), + 1, nullptr, 0, 0, nullptr, 0, 0); + EXPECT_NEAR(lon, 1.9992776848, 1e-10); + EXPECT_NEAR(lat, 48.9999322600, 1e-10); + + proj_destroy(P); + + sqlite3 *hDB = nullptr; + sqlite3_open_v2(pj_context_get_grid_cache_filename(ctx).c_str(), &hDB, + SQLITE_OPEN_READWRITE, nullptr); + ASSERT_NE(hDB, nullptr); + + // Force lastChecked to the Epoch so that data is expired. + sqlite3_stmt *hStmt = nullptr; + sqlite3_prepare_v2(hDB, "UPDATE properties SET lastChecked = 0, " + "lastModified = 'foo', etag = 'bar'", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_DONE); + sqlite3_finalize(hStmt); + + // Put junk in already cached data to check that we will refresh it. + hStmt = nullptr; + sqlite3_prepare_v2(hDB, "UPDATE chunk_data SET data = zeroblob(16384)", -1, + &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_DONE); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + + proj_cleanup(); + + // Set a never expire ttl + proj_grid_cache_set_ttl(ctx, -1); + + // We'll get junk data, hence the pipeline initialization fails + proj_log_func(ctx, nullptr, silent_logger); + P = proj_create(ctx, pipeline); + ASSERT_EQ(P, nullptr); + proj_destroy(P); + + proj_cleanup(); + + // Set a normal ttl + proj_grid_cache_set_ttl(ctx, 86400); + + // Pipeline creation succeeds + P = proj_create(ctx, pipeline); + ASSERT_NE(P, nullptr); + proj_destroy(P); + + hDB = nullptr; + sqlite3_open_v2(pj_context_get_grid_cache_filename(ctx).c_str(), &hDB, + SQLITE_OPEN_READWRITE, nullptr); + ASSERT_NE(hDB, nullptr); + hStmt = nullptr; + sqlite3_prepare_v2(hDB, + "SELECT lastChecked, lastModified, etag FROM properties", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_ROW); + ASSERT_NE(sqlite3_column_int64(hStmt, 0), 0); + ASSERT_NE(sqlite3_column_text(hStmt, 1), nullptr); + ASSERT_NE(std::string(reinterpret_cast<const char *>( + sqlite3_column_text(hStmt, 1))), + "foo"); + ASSERT_NE(sqlite3_column_text(hStmt, 2), nullptr); + ASSERT_NE(std::string(reinterpret_cast<const char *>( + sqlite3_column_text(hStmt, 2))), + "bar"); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + + proj_grid_cache_clear(ctx); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, cache_lock) { + if (!networkAccessOK) { + return; + } + const char *pipeline = + "+proj=pipeline " + "+step +proj=unitconvert +xy_in=deg +xy_out=rad " + "+step +proj=hgridshift +grids=https://cdn.proj.org/ntf_r93.tif " + "+step +proj=unitconvert +xy_in=rad +xy_out=deg"; + + proj_cleanup(); + + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + proj_grid_cache_set_filename(ctx, "tmp_proj_db_cache.db"); + + proj_grid_cache_clear(ctx); + + auto P = proj_create(ctx, pipeline); + ASSERT_NE(P, nullptr); + + double lon = 2; + double lat = 49; + proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, sizeof(double), + 1, nullptr, 0, 0, nullptr, 0, 0); + EXPECT_NEAR(lon, 1.9992776848, 1e-10); + EXPECT_NEAR(lat, 48.9999322600, 1e-10); + + proj_destroy(P); + + // Take a lock + sqlite3 *hDB = nullptr; + sqlite3_open_v2(pj_context_get_grid_cache_filename(ctx).c_str(), &hDB, + SQLITE_OPEN_READWRITE, nullptr); + ASSERT_NE(hDB, nullptr); + sqlite3_stmt *hStmt = nullptr; + sqlite3_prepare_v2(hDB, "BEGIN EXCLUSIVE", -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_DONE); + sqlite3_finalize(hStmt); + + proj_cleanup(); + + time_t start; + time(&start); + // 2 lock attempts, so we must sleep for each at least 0.5 ms + putenv(const_cast<char *>("PROJ_LOCK_MAX_ITERS=25")); + P = proj_create(ctx, pipeline); + putenv(const_cast<char *>("PROJ_LOCK_MAX_ITERS=")); + ASSERT_NE(P, nullptr); + proj_destroy(P); + + // Check that we have spend more than 1 sec + time_t end; + time(&end); + ASSERT_GE(end - start, 1U); + + sqlite3_close(hDB); + + proj_grid_cache_clear(ctx); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, download_whole_files) { + if (!networkAccessOK) { + return; + } + + proj_cleanup(); + unlink("proj_test_tmp/cache.db"); + unlink("proj_test_tmp/dvr90.tif"); + rmdir("proj_test_tmp"); + + putenv(const_cast<char *>("PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=")); + putenv(const_cast<char *>("PROJ_USER_WRITABLE_DIRECTORY=./proj_test_tmp")); + putenv(const_cast<char *>("PROJ_FULL_FILE_CHUNK_SIZE=100000")); + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + ASSERT_TRUE(proj_is_download_needed(ctx, "dvr90.gtx", false)); + + ASSERT_TRUE(proj_download_file(ctx, "dvr90.gtx", false, nullptr, nullptr)); + + FILE *f = fopen("proj_test_tmp/dvr90.tif", "rb"); + ASSERT_NE(f, nullptr); + fclose(f); + + proj_context_set_enable_network(ctx, false); + + const char *pipeline = + "+proj=pipeline " + "+step +proj=unitconvert +xy_in=deg +xy_out=rad " + "+step +proj=vgridshift +grids=dvr90.gtx +multiplier=1 " + "+step +proj=unitconvert +xy_in=rad +xy_out=deg"; + + auto P = proj_create(ctx, pipeline); + ASSERT_NE(P, nullptr); + + double lon = 12; + double lat = 56; + double z = 0; + proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, sizeof(double), + 1, &z, sizeof(double), 1, nullptr, 0, 0); + EXPECT_NEAR(z, 36.5909996032715, 1e-10); + proj_destroy(P); + + proj_context_set_enable_network(ctx, true); + + ASSERT_FALSE(proj_is_download_needed(ctx, "dvr90.gtx", false)); + + { + sqlite3 *hDB = nullptr; + sqlite3_open_v2("proj_test_tmp/cache.db", &hDB, SQLITE_OPEN_READWRITE, + nullptr); + ASSERT_NE(hDB, nullptr); + // Force lastChecked to the Epoch so that data is expired. + sqlite3_stmt *hStmt = nullptr; + sqlite3_prepare_v2( + hDB, "UPDATE downloaded_file_properties SET lastChecked = 0", -1, + &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_DONE); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + } + + // If we ignore TTL settings, then no network access will be done + ASSERT_FALSE(proj_is_download_needed(ctx, "dvr90.gtx", true)); + + { + sqlite3 *hDB = nullptr; + sqlite3_open_v2("proj_test_tmp/cache.db", &hDB, SQLITE_OPEN_READWRITE, + nullptr); + ASSERT_NE(hDB, nullptr); + // Check that the lastChecked timestamp is still 0 + sqlite3_stmt *hStmt = nullptr; + sqlite3_prepare_v2(hDB, + "SELECT lastChecked FROM downloaded_file_properties", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(hStmt, 0), 0); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + } + + // Should recheck from the CDN, update last_checked and do nothing + ASSERT_FALSE(proj_is_download_needed(ctx, "dvr90.gtx", false)); + + { + sqlite3 *hDB = nullptr; + sqlite3_open_v2("proj_test_tmp/cache.db", &hDB, SQLITE_OPEN_READWRITE, + nullptr); + ASSERT_NE(hDB, nullptr); + sqlite3_stmt *hStmt = nullptr; + // Check that the lastChecked timestamp has been updated + sqlite3_prepare_v2(hDB, + "SELECT lastChecked FROM downloaded_file_properties", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_ROW); + ASSERT_NE(sqlite3_column_int64(hStmt, 0), 0); + sqlite3_finalize(hStmt); + hStmt = nullptr; + + // Now invalid lastModified. This should trigger a new download + sqlite3_prepare_v2( + hDB, "UPDATE downloaded_file_properties SET lastChecked = 0, " + "lastModified = 'foo'", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_DONE); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + } + + ASSERT_TRUE(proj_is_download_needed(ctx, "dvr90.gtx", false)); + + // Redo download with a progress callback this time. + unlink("proj_test_tmp/dvr90.tif"); + + const auto cbk = [](PJ_CONTEXT *l_ctx, double pct, void *user_data) -> int { + auto vect = static_cast<std::vector<std::pair<PJ_CONTEXT *, double>> *>( + user_data); + vect->push_back(std::pair<PJ_CONTEXT *, double>(l_ctx, pct)); + return true; + }; + + std::vector<std::pair<PJ_CONTEXT *, double>> vectPct; + ASSERT_TRUE(proj_download_file(ctx, "dvr90.gtx", false, cbk, &vectPct)); + ASSERT_EQ(vectPct.size(), 3U); + ASSERT_EQ(vectPct.back().first, ctx); + ASSERT_EQ(vectPct.back().second, 1.0); + + proj_context_destroy(ctx); + putenv(const_cast<char *>("PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES")); + putenv(const_cast<char *>("PROJ_USER_WRITABLE_DIRECTORY=")); + putenv(const_cast<char *>("PROJ_FULL_FILE_CHUNK_SIZE=")); + unlink("proj_test_tmp/cache.db"); + unlink("proj_test_tmp/dvr90.tif"); + rmdir("proj_test_tmp"); +} + +// --------------------------------------------------------------------------- + +TEST(networking, file_api) { + if (!networkAccessOK) { + return; + } + + proj_cleanup(); + unlink("proj_test_tmp/cache.db"); + unlink("proj_test_tmp/dvr90.tif"); + rmdir("proj_test_tmp"); + + putenv(const_cast<char *>("PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=")); + putenv(const_cast<char *>("PROJ_USER_WRITABLE_DIRECTORY=./proj_test_tmp")); + putenv(const_cast<char *>("PROJ_FULL_FILE_CHUNK_SIZE=30000")); + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + struct UserData { + bool in_open = false; + bool in_read = false; + bool in_write = false; + bool in_seek = false; + bool in_tell = false; + bool in_close = false; + bool in_exists = false; + bool in_mkdir = false; + bool in_unlink = false; + bool in_rename = false; + }; + + struct PROJ_FILE_API api; + api.version = 1; + api.open_cbk = [](PJ_CONTEXT *, const char *filename, + PROJ_OPEN_ACCESS access, + void *user_data) -> PROJ_FILE_HANDLE * { + static_cast<UserData *>(user_data)->in_open = true; + return reinterpret_cast<PROJ_FILE_HANDLE *>(fopen( + filename, + access == PROJ_OPEN_ACCESS_READ_ONLY + ? "rb" + : access == PROJ_OPEN_ACCESS_READ_UPDATE ? "r+b" : "w+b")); + }; + api.read_cbk = [](PJ_CONTEXT *, PROJ_FILE_HANDLE *handle, void *buffer, + size_t sizeBytes, void *user_data) -> size_t { + static_cast<UserData *>(user_data)->in_read = true; + return fread(buffer, 1, sizeBytes, reinterpret_cast<FILE *>(handle)); + }; + api.write_cbk = [](PJ_CONTEXT *, PROJ_FILE_HANDLE *handle, + const void *buffer, size_t sizeBytes, + void *user_data) -> size_t { + static_cast<UserData *>(user_data)->in_write = true; + return fwrite(buffer, 1, sizeBytes, reinterpret_cast<FILE *>(handle)); + }; + api.seek_cbk = [](PJ_CONTEXT *, PROJ_FILE_HANDLE *handle, long long offset, + int whence, void *user_data) -> int { + static_cast<UserData *>(user_data)->in_seek = true; + return fseek(reinterpret_cast<FILE *>(handle), + static_cast<long>(offset), whence) == 0; + }; + api.tell_cbk = [](PJ_CONTEXT *, PROJ_FILE_HANDLE *handle, + void *user_data) -> unsigned long long { + static_cast<UserData *>(user_data)->in_tell = true; + return ftell(reinterpret_cast<FILE *>(handle)); + }; + api.close_cbk = [](PJ_CONTEXT *, PROJ_FILE_HANDLE *handle, + void *user_data) -> void { + static_cast<UserData *>(user_data)->in_close = true; + fclose(reinterpret_cast<FILE *>(handle)); + }; + api.exists_cbk = [](PJ_CONTEXT *, const char *filename, + void *user_data) -> int { + static_cast<UserData *>(user_data)->in_exists = true; + struct stat buf; + return stat(filename, &buf) == 0; + }; + api.mkdir_cbk = [](PJ_CONTEXT *, const char *filename, + void *user_data) -> int { + static_cast<UserData *>(user_data)->in_mkdir = true; +#ifdef _WIN32 + return mkdir(filename) == 0; +#else + return mkdir(filename, 0755) == 0; +#endif + }; + api.unlink_cbk = [](PJ_CONTEXT *, const char *filename, + void *user_data) -> int { + static_cast<UserData *>(user_data)->in_unlink = true; + return unlink(filename) == 0; + }; + api.rename_cbk = [](PJ_CONTEXT *, const char *oldPath, const char *newPath, + void *user_data) -> int { + static_cast<UserData *>(user_data)->in_rename = true; + return rename(oldPath, newPath) == 0; + }; + + UserData userData; + ASSERT_TRUE(proj_context_set_fileapi(ctx, &api, &userData)); + + ASSERT_TRUE(proj_is_download_needed(ctx, "dvr90.gtx", false)); + + ASSERT_TRUE(proj_download_file(ctx, "dvr90.gtx", false, nullptr, nullptr)); + + ASSERT_TRUE(userData.in_open); + ASSERT_FALSE(userData.in_read); + ASSERT_TRUE(userData.in_write); + ASSERT_TRUE(userData.in_close); + ASSERT_TRUE(userData.in_exists); + ASSERT_TRUE(userData.in_mkdir); + ASSERT_TRUE(userData.in_unlink); + ASSERT_TRUE(userData.in_rename); + + proj_context_set_enable_network(ctx, false); + + const char *pipeline = + "+proj=pipeline " + "+step +proj=unitconvert +xy_in=deg +xy_out=rad " + "+step +proj=vgridshift +grids=dvr90.gtx +multiplier=1 " + "+step +proj=unitconvert +xy_in=rad +xy_out=deg"; + + auto P = proj_create(ctx, pipeline); + ASSERT_NE(P, nullptr); + + double lon = 12; + double lat = 56; + double z = 0; + proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, sizeof(double), + 1, &z, sizeof(double), 1, nullptr, 0, 0); + EXPECT_NEAR(z, 36.5909996032715, 1e-10); + + proj_destroy(P); + + ASSERT_TRUE(userData.in_read); + ASSERT_TRUE(userData.in_seek); + + proj_context_destroy(ctx); + putenv(const_cast<char *>("PROJ_SKIP_READ_USER_WRITABLE_DIRECTORY=YES")); + putenv(const_cast<char *>("PROJ_USER_WRITABLE_DIRECTORY=")); + putenv(const_cast<char *>("PROJ_FULL_FILE_CHUNK_SIZE=")); + unlink("proj_test_tmp/cache.db"); + unlink("proj_test_tmp/dvr90.tif"); + rmdir("proj_test_tmp"); +} + +#endif + +} // namespace diff --git a/test/unit/test_operation.cpp b/test/unit/test_operation.cpp index b2c13b6c..8ee5814b 100644 --- a/test/unit/test_operation.cpp +++ b/test/unit/test_operation.cpp @@ -4832,7 +4832,7 @@ TEST(operation, geogCRS_to_geogCRS_context_concatenated_operation) { EXPECT_TRUE(nn_dynamic_pointer_cast<ConcatenatedOperation>(list[0]) != nullptr); - auto grids = list[0]->gridsNeeded(DatabaseContext::create()); + auto grids = list[0]->gridsNeeded(DatabaseContext::create(), false); EXPECT_EQ(grids.size(), 1U); } @@ -6508,7 +6508,7 @@ TEST(operation, transformation_height_to_PROJ_string) { EXPECT_EQ(transf->exportToPROJString(PROJStringFormatter::create().get()), "+proj=vgridshift +grids=egm08_25.gtx +multiplier=1"); - auto grids = transf->gridsNeeded(DatabaseContext::create()); + auto grids = transf->gridsNeeded(DatabaseContext::create(), false); ASSERT_EQ(grids.size(), 1U); auto gridDesc = *(grids.begin()); EXPECT_EQ(gridDesc.shortName, "egm08_25.gtx"); @@ -6820,7 +6820,7 @@ TEST(operation, compoundCRS_with_boundGeogCRS_and_boundVerticalCRS_to_geogCRS) { "+step +proj=unitconvert +xy_in=rad +xy_out=deg " "+step +proj=axisswap +order=2,1"); - auto grids = op->gridsNeeded(DatabaseContext::create()); + auto grids = op->gridsNeeded(DatabaseContext::create(), false); EXPECT_EQ(grids.size(), 1U); auto opInverse = CoordinateOperationFactory::create()->createOperation( @@ -8262,8 +8262,8 @@ TEST(operation, isPROJInstantiable) { auto transformation = Transformation::createGeocentricTranslations( PropertyMap(), GeographicCRS::EPSG_4269, GeographicCRS::EPSG_4326, 1.0, 2.0, 3.0, {}); - EXPECT_TRUE( - transformation->isPROJInstantiable(DatabaseContext::create())); + EXPECT_TRUE(transformation->isPROJInstantiable( + DatabaseContext::create(), false)); } // Missing grid @@ -8271,8 +8271,8 @@ TEST(operation, isPROJInstantiable) { auto transformation = Transformation::createNTv2( PropertyMap(), GeographicCRS::EPSG_4807, GeographicCRS::EPSG_4326, "foo.gsb", std::vector<PositionalAccuracyNNPtr>()); - EXPECT_FALSE( - transformation->isPROJInstantiable(DatabaseContext::create())); + EXPECT_FALSE(transformation->isPROJInstantiable( + DatabaseContext::create(), false)); } // Unsupported method @@ -8283,8 +8283,8 @@ TEST(operation, isPROJInstantiable) { PropertyMap(), std::vector<OperationParameterNNPtr>{}), std::vector<GeneralParameterValueNNPtr>{}, std::vector<PositionalAccuracyNNPtr>{}); - EXPECT_FALSE( - transformation->isPROJInstantiable(DatabaseContext::create())); + EXPECT_FALSE(transformation->isPROJInstantiable( + DatabaseContext::create(), false)); } } diff --git a/travis/csa/before_install.sh b/travis/csa/before_install.sh index c2e29064..dcb7733b 100755 --- a/travis/csa/before_install.sh +++ b/travis/csa/before_install.sh @@ -5,7 +5,7 @@ set -e ./travis/before_install_apt.sh ./travis/before_install_pip.sh -sudo apt-get install -qq sqlite3 libsqlite3-dev +sudo apt-get install -qq sqlite3 libsqlite3-dev libtiff-dev libcurl4-openssl-dev CLANG_LLVM=clang+llvm-6.0.0-x86_64-linux-gnu-ubuntu-16.04 wget http://releases.llvm.org/6.0.0/$CLANG_LLVM.tar.xz diff --git a/travis/linux_clang/before_install.sh b/travis/linux_clang/before_install.sh index 8ce465a2..c4b8acad 100755 --- a/travis/linux_clang/before_install.sh +++ b/travis/linux_clang/before_install.sh @@ -5,4 +5,4 @@ set -e ./travis/before_install_apt.sh ./travis/before_install_pip.sh -sudo apt-get install -qq sqlite3 libsqlite3-dev +sudo apt-get install -qq sqlite3 libsqlite3-dev libtiff-dev libcurl4-openssl-dev diff --git a/travis/linux_gcc/before_install.sh b/travis/linux_gcc/before_install.sh index 7725b3ef..9db2ea89 100755 --- a/travis/linux_gcc/before_install.sh +++ b/travis/linux_gcc/before_install.sh @@ -9,6 +9,7 @@ sudo apt-get install -qq \ lcov \ doxygen graphviz \ sqlite3 libsqlite3-dev \ + libtiff-dev libcurl4-openssl-dev \ cppcheck scripts/cppcheck.sh diff --git a/travis/linux_gcc7/before_install.sh b/travis/linux_gcc7/before_install.sh index a3f6c8c0..2ef2feee 100755 --- a/travis/linux_gcc7/before_install.sh +++ b/travis/linux_gcc7/before_install.sh @@ -8,7 +8,8 @@ set -e sudo apt-get install -qq \ lcov \ doxygen graphviz \ - sqlite3 libsqlite3-dev + sqlite3 libsqlite3-dev \ + libtiff-dev libcurl4-openssl-dev #scripts/cppcheck.sh #scripts/doxygen.sh diff --git a/travis/mingw32/install.sh b/travis/mingw32/install.sh index 52e4f7a6..8c756ced 100755 --- a/travis/mingw32/install.sh +++ b/travis/mingw32/install.sh @@ -2,6 +2,9 @@ set -e +unset CC +unset CXX + export CCACHE_CPP2=yes export PROJ_DB_CACHE_DIR="$HOME/.ccache" @@ -16,6 +19,22 @@ ln -s $MINGW_PREFIX/libgcc_s_seh-1.dll $WINE_SYSDIR ln -s $MINGW_PREFIX/libgcc_s_sjlj-1.dll $WINE_SYSDIR ln -s /usr/$MINGW_ARCH/lib/libwinpthread-1.dll $WINE_SYSDIR +# build zlib +wget https://github.com/madler/zlib/archive/v1.2.11.tar.gz +tar xzf v1.2.11.tar.gz +(cd zlib-1.2.11 && sudo make install -fwin32/Makefile.gcc SHARED_MODE=1 PREFIX=x86_64-w64-mingw32- DESTDIR=/usr/$MINGW_ARCH/) +sudo mkdir -p /usr/$MINGW_ARCH/include +sudo mkdir -p /usr/$MINGW_ARCH/lib +sudo cp /usr/$MINGW_ARCH/*.h /usr/$MINGW_ARCH/include +sudo cp /usr/$MINGW_ARCH/libz.* /usr/$MINGW_ARCH/lib +ln -s /usr/$MINGW_ARCH/zlib1.dll $WINE_SYSDIR + +# build libtiff +wget https://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz +tar xzf tiff-4.1.0.tar.gz +(cd tiff-4.1.0 && ./configure --host=$MINGW_ARCH --prefix=/usr/$MINGW_ARCH && make -j2 && sudo make install) +ln -s /usr/$MINGW_ARCH/bin/libtiff-5.dll $WINE_SYSDIR + # build sqlite3 wget https://sqlite.org/2018/sqlite-autoconf-3250100.tar.gz tar xzf sqlite-autoconf-3250100.tar.gz @@ -28,12 +47,13 @@ ln -s /usr/$MINGW_ARCH/bin/libsqlite3-0.dll $WINE_SYSDIR # autoconf build mkdir build_autoconf cd build_autoconf -CC="ccache $MINGW_ARCH-gcc" CXX="ccache $MINGW_ARCH-g++" LD=$MINGW_ARCH-ld ../configure --host=$MINGW_ARCH --prefix=/tmp/proj_autoconf_install +CC="ccache $MINGW_ARCH-gcc" CXX="ccache $MINGW_ARCH-g++" LD=$MINGW_ARCH-ld ../configure --host=$MINGW_ARCH --prefix=/tmp/proj_autoconf_install --without-curl make -j2 make install make dist-all find /tmp/proj_autoconf_install (cd test; make -j2) +cp -r ../data/tests /tmp/proj_autoconf_install/share/proj test/unit/test_cpp_api.exe cd .. # Now with grids @@ -44,7 +64,7 @@ cd .. # autoconf build with grids mkdir build_autoconf_grids cd build_autoconf_grids -CC="ccache $MINGW_ARCH-gcc" CXX="ccache $MINGW_ARCH-g++" LD=$MINGW_ARCH-ld ../configure --host=$MINGW_ARCH --prefix=/tmp/proj_autoconf_install_nad +CC="ccache $MINGW_ARCH-gcc" CXX="ccache $MINGW_ARCH-g++" LD=$MINGW_ARCH-ld ../configure --host=$MINGW_ARCH --prefix=/tmp/proj_autoconf_install_nad --without-curl make -j2 make install find /tmp/proj_autoconf_install_nad diff --git a/travis/osx/before_install.sh b/travis/osx/before_install.sh index de9544a5..e457ec8c 100755 --- a/travis/osx/before_install.sh +++ b/travis/osx/before_install.sh @@ -6,7 +6,8 @@ export PATH=$HOME/Library/Python/3.7/bin:$PATH brew update brew install ccache -brew install sqlite3 +#brew upgrade sqlite3 +#brew upgrade libtiff brew install doxygen #brew install md5sha1sum #brew reinstall python |