Merge RFC4 (#1865)

This commit is the result of the squashing of rfc4_dev branch in a single
commit. It implements mostly RFC 4 related work.

* Grid handling:
  - remove obsolete and presumably unfinished implementation of grid catalog functionality
  - all grid functionality is in grids.cpp/.hpp
  - vertical and horizontal grid shift: rework to no longer load whole grid into memory
  - remove hgrids and vgrids member from PJ structure, and store them in hgridshift/vgridshift/deformation structures
  - build systems: add optional libtiff dependency. Must be explicitly disabled if not desired
  - add support for horizontal and vertical grids in GeoTIFF, if libtiff is available
  - add GenericShiftGridSet and GenericShiftGrid classes, relying on TIFF grids, that can be used for generic purpose grid-based adjustment
  - add a +proj=xyzgridshift method to perform geocentric translation by grid. Used for French NTF to RGF93 transformation using gr3df97a.tif grid
  - deformation: add support for +grids= for GeoTIFF grids
  - horizontal grid shift: fix failures on points slightly outside a subgrid (fixes #209)

* File management:
  - add a filemanager.cpp/.hpp to deal with file related work
  - test for legacy proj_api.h fileapi
  - proj.h: add proj_context_set_fileapi() and proj_context_set_sqlite3_vfs_name() (fixes #866)
  - add capability to read resource files from the user writable directory

* Network access:
  - build systems: add optional curl dependency
  - add a curl-based default implementation for network related functionality
  - proj.h: add C API to control network functionality, and optionaly provide network callbacks
  - add data/proj.ini with default settings
  - add a SQLite3 local cache of downloaded chunks
  - add proj_is_download_needed() and proj_download_file()

* Use Win32 Unicode APIs and expect all strings to be UTF-8 (fixes #1765)
  For backward compatibility, if PROJ_LIB content is found to be not UTF-8 or
  pointing to a non existing directory, then an attempt at interpretating it
  in the ANSI page encoding is done.
  proj_context_set_search_paths() now assumes strings to be in UTF-8, and
  functions returning paths will also return values in UTF-8.

3 files changed, 64 insertions, 8 deletions

diff --git a/scripts/doxygen.sh b/scripts/doxygen.sh
index 48646237..380afcfa 100755
--- a/scripts/doxygen.sh
+++ b/scripts/doxygen.sh
@@ -34,7 +34,11 @@ fi
 mkdir -p docs/build/tmp_breathe
 python scripts/generate_breathe_friendly_general_doc.py
 rm -rf docs/build/xml/
-(cat Doxyfile; printf "GENERATE_HTML=NO\nGENERATE_XML=YES\nINPUT= src/iso19111 include/proj src/proj.h docs/build/tmp_breathe/general_doc.dox.reworked.h") | doxygen -  > docs/build/tmp_breathe/docs_log.txt 2>&1
+
+# Ugly hack to workaround a bug of Doxygen 1.8.17 that erroneously detect proj_network_get_header_value_cbk_type/ as a variable
+sed "s/const char\* (\*proj_network_get_header_value_cbk_type/CONST_CHAR\* (\*proj_network_get_header_value_cbk_type/" < src/proj.h > docs/build/tmp_breathe/proj.h
+
+(cat Doxyfile; printf "GENERATE_HTML=NO\nGENERATE_XML=YES\nINPUT= src/iso19111 include/proj docs/build/tmp_breathe/proj.h src/filemanager.cpp src/networkfilemanager.cpp docs/build/tmp_breathe/general_doc.dox.reworked.h") | doxygen -  > docs/build/tmp_breathe/docs_log.txt 2>&1
 if grep -i warning docs/build/tmp_breathe/docs_log.txt; then
     echo "Doxygen warnings found" && cat docs/build/tmp_breathe/docs_log.txt && /bin/false;
 else
@@ -56,5 +60,9 @@ done
 sed "s/Convention/Convention_/g" < ${TOPDIR}/docs/build/xml/classosgeo_1_1proj_1_1io_1_1WKTFormatter.xml | sed "s/WKT2_2018/_WKT2_2018/g" | sed "s/WKT2_2019/_WKT2_2019/g" | sed "s/WKT2_2015/_WKT2_2015/g" | sed "s/WKT1_GDAL/_WKT1_GDAL/g" | sed "s/WKT1_ESRI/_WKT1_ESRI/g" > ${TOPDIR}/docs/build/xml/classosgeo_1_1proj_1_1io_1_1WKTFormatter.xml.tmp
 mv ${TOPDIR}/docs/build/xml/classosgeo_1_1proj_1_1io_1_1WKTFormatter.xml.tmp  ${TOPDIR}/docs/build/xml/classosgeo_1_1proj_1_1io_1_1WKTFormatter.xml
 
+# Ugly hack to workaround a bug of Doxygen 1.8.17 that erroneously detect proj_network_get_header_value_cbk_type/ as a variable
+sed "s/CONST_CHAR/const char/"  < ${TOPDIR}/docs/build/xml/proj_8h.xml > ${TOPDIR}/docs/build/xml/proj_8h.xml.tmp
+mv ${TOPDIR}/docs/build/xml/proj_8h.xml.tmp ${TOPDIR}/docs/build/xml/proj_8h.xml
+
 popd > /dev/null || exit
 
diff --git a/scripts/reference_exported_symbols.txt b/scripts/reference_exported_symbols.txt
index 49d6e5d2..07043504 100644
--- a/scripts/reference_exported_symbols.txt
+++ b/scripts/reference_exported_symbols.txt
@@ -281,6 +281,26 @@ osgeo::proj::datum::TemporalDatum::temporalOrigin() const
 osgeo::proj::datum::VerticalReferenceFrame::create(osgeo::proj::util::PropertyMap const&, osgeo::proj::util::optional<std::string> const&, osgeo::proj::util::optional<osgeo::proj::datum::RealizationMethod> const&)
 osgeo::proj::datum::VerticalReferenceFrame::realizationMethod() const
 osgeo::proj::datum::VerticalReferenceFrame::~VerticalReferenceFrame()
+osgeo::proj::GenericShiftGrid::~GenericShiftGrid()
+osgeo::proj::GenericShiftGrid::GenericShiftGrid(std::string const&, int, int, osgeo::proj::ExtentAndRes const&)
+osgeo::proj::GenericShiftGrid::gridAt(double, double) const
+osgeo::proj::GenericShiftGridSet::~GenericShiftGridSet()
+osgeo::proj::GenericShiftGridSet::GenericShiftGridSet()
+osgeo::proj::GenericShiftGridSet::gridAt(double, double) const
+osgeo::proj::GenericShiftGridSet::open(projCtx_t*, std::string const&)
+osgeo::proj::GenericShiftGridSet::reassign_context(projCtx_t*)
+osgeo::proj::GenericShiftGridSet::reopen(projCtx_t*)
+osgeo::proj::Grid::~Grid()
+osgeo::proj::Grid::Grid(std::string const&, int, int, osgeo::proj::ExtentAndRes const&)
+osgeo::proj::HorizontalShiftGrid::gridAt(double, double) const
+osgeo::proj::HorizontalShiftGrid::~HorizontalShiftGrid()
+osgeo::proj::HorizontalShiftGrid::HorizontalShiftGrid(std::string const&, int, int, osgeo::proj::ExtentAndRes const&)
+osgeo::proj::HorizontalShiftGridSet::gridAt(double, double) const
+osgeo::proj::HorizontalShiftGridSet::~HorizontalShiftGridSet()
+osgeo::proj::HorizontalShiftGridSet::HorizontalShiftGridSet()
+osgeo::proj::HorizontalShiftGridSet::open(projCtx_t*, std::string const&)
+osgeo::proj::HorizontalShiftGridSet::reassign_context(projCtx_t*)
+osgeo::proj::HorizontalShiftGridSet::reopen(projCtx_t*)
 osgeo::proj::internal::ci_equal(std::string const&, char const*)
 osgeo::proj::internal::ci_equal(std::string const&, std::string const&)
 osgeo::proj::internal::ci_find(std::string const&, char const*)
@@ -301,8 +321,8 @@ osgeo::proj::io::AuthorityFactory::create(dropbox::oxygen::nn<std::shared_ptr<os
 osgeo::proj::io::AuthorityFactory::createEllipsoid(std::string const&) const
 osgeo::proj::io::AuthorityFactory::createExtent(std::string const&) const
 osgeo::proj::io::AuthorityFactory::createFromCoordinateReferenceSystemCodes(std::string const&, std::string const&) const
-osgeo::proj::io::AuthorityFactory::createFromCoordinateReferenceSystemCodes(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, bool, bool, std::shared_ptr<osgeo::proj::metadata::Extent> const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&) const
-osgeo::proj::io::AuthorityFactory::createFromCRSCodesWithIntermediates(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, std::vector<std::pair<std::string, std::string>, std::allocator<std::pair<std::string, std::string> > > const&, osgeo::proj::io::AuthorityFactory::ObjectType, std::vector<std::string, std::allocator<std::string> > const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&) const
+osgeo::proj::io::AuthorityFactory::createFromCoordinateReferenceSystemCodes(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, bool, bool, bool, std::shared_ptr<osgeo::proj::metadata::Extent> const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&) const
+osgeo::proj::io::AuthorityFactory::createFromCRSCodesWithIntermediates(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, bool, std::vector<std::pair<std::string, std::string>, std::allocator<std::pair<std::string, std::string> > > const&, osgeo::proj::io::AuthorityFactory::ObjectType, std::vector<std::string, std::allocator<std::string> > const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&) const
 osgeo::proj::io::AuthorityFactory::createGeodeticCRS(std::string const&) const
 osgeo::proj::io::AuthorityFactory::createGeodeticDatum(std::string const&) const
 osgeo::proj::io::AuthorityFactory::createGeographicCRS(std::string const&) const
@@ -332,7 +352,7 @@ osgeo::proj::io::DatabaseContext::getDatabaseStructure() const
 osgeo::proj::io::DatabaseContext::getMetadata(char const*) const
 osgeo::proj::io::DatabaseContext::getPath() const
 osgeo::proj::io::DatabaseContext::getSqliteHandle() const
-osgeo::proj::io::DatabaseContext::lookForGridInfo(std::string const&, std::string&, std::string&, std::string&, bool&, bool&, bool&) const
+osgeo::proj::io::DatabaseContext::lookForGridInfo(std::string const&, bool, std::string&, std::string&, std::string&, bool&, bool&, bool&) const
 osgeo::proj::io::FactoryException::~FactoryException()
 osgeo::proj::io::FactoryException::FactoryException(char const*)
 osgeo::proj::io::FactoryException::FactoryException(osgeo::proj::io::FactoryException const&)
@@ -467,7 +487,7 @@ osgeo::proj::metadata::VerticalExtent::~VerticalExtent()
 osgeo::proj::operation::ConcatenatedOperation::~ConcatenatedOperation()
 osgeo::proj::operation::ConcatenatedOperation::createComputeMetadata(std::vector<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::operation::CoordinateOperation> >, std::allocator<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::operation::CoordinateOperation> > > > const&, bool)
 osgeo::proj::operation::ConcatenatedOperation::create(osgeo::proj::util::PropertyMap const&, std::vector<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::operation::CoordinateOperation> >, std::allocator<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::operation::CoordinateOperation> > > > const&, std::vector<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::metadata::PositionalAccuracy> >, std::allocator<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::metadata::PositionalAccuracy> > > > const&)
-osgeo::proj::operation::ConcatenatedOperation::gridsNeeded(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&) const
+osgeo::proj::operation::ConcatenatedOperation::gridsNeeded(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&, bool) const
 osgeo::proj::operation::ConcatenatedOperation::inverse() const
 osgeo::proj::operation::ConcatenatedOperation::operations() const
 osgeo::proj::operation::Conversion::~Conversion()
@@ -577,7 +597,7 @@ osgeo::proj::operation::CoordinateOperationFactory::createOperation(dropbox::oxy
 osgeo::proj::operation::CoordinateOperationFactory::createOperations(dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::crs::CRS> > const&, dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::crs::CRS> > const&, dropbox::oxygen::nn<std::unique_ptr<osgeo::proj::operation::CoordinateOperationContext, std::default_delete<osgeo::proj::operation::CoordinateOperationContext> > > const&) const
 osgeo::proj::operation::CoordinateOperation::hasBallparkTransformation() const
 osgeo::proj::operation::CoordinateOperation::interpolationCRS() const
-osgeo::proj::operation::CoordinateOperation::isPROJInstantiable(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&) const
+osgeo::proj::operation::CoordinateOperation::isPROJInstantiable(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&, bool) const
 osgeo::proj::operation::CoordinateOperation::normalizeForVisualization() const
 osgeo::proj::operation::CoordinateOperation::operationVersion() const
 osgeo::proj::operation::CoordinateOperation::shallowClone() const
@@ -623,7 +643,7 @@ osgeo::proj::operation::ParameterValue::value() const
 osgeo::proj::operation::ParameterValue::valueFile() const
 osgeo::proj::operation::PointMotionOperation::~PointMotionOperation()
 osgeo::proj::operation::SingleOperation::createPROJBased(osgeo::proj::util::PropertyMap const&, std::string const&, std::shared_ptr<osgeo::proj::crs::CRS> const&, std::shared_ptr<osgeo::proj::crs::CRS> const&, std::vector<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::metadata::PositionalAccuracy> >, std::allocator<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::metadata::PositionalAccuracy> > > > const&)
-osgeo::proj::operation::SingleOperation::gridsNeeded(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&) const
+osgeo::proj::operation::SingleOperation::gridsNeeded(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&, bool) const
 osgeo::proj::operation::SingleOperation::method() const
 osgeo::proj::operation::SingleOperation::parameterValue(int) const
 osgeo::proj::operation::SingleOperation::parameterValueMeasure(int) const
@@ -698,6 +718,15 @@ osgeo::proj::util::PropertyMap::set(std::string const&, std::string const&)
 osgeo::proj::util::PropertyMap::set(std::string const&, std::vector<std::string, std::allocator<std::string> > const&)
 osgeo::proj::util::UnsupportedOperationException::~UnsupportedOperationException()
 osgeo::proj::util::UnsupportedOperationException::UnsupportedOperationException(osgeo::proj::util::UnsupportedOperationException const&)
+osgeo::proj::VerticalShiftGrid::gridAt(double, double) const
+osgeo::proj::VerticalShiftGridSet::gridAt(double, double) const
+osgeo::proj::VerticalShiftGridSet::open(projCtx_t*, std::string const&)
+osgeo::proj::VerticalShiftGridSet::reassign_context(projCtx_t*)
+osgeo::proj::VerticalShiftGridSet::reopen(projCtx_t*)
+osgeo::proj::VerticalShiftGridSet::~VerticalShiftGridSet()
+osgeo::proj::VerticalShiftGridSet::VerticalShiftGridSet()
+osgeo::proj::VerticalShiftGrid::~VerticalShiftGrid()
+osgeo::proj::VerticalShiftGrid::VerticalShiftGrid(std::string const&, int, int, osgeo::proj::ExtentAndRes const&)
 pj_acquire_lock
 pj_add_type_crs_if_needed(std::string const&)
 pj_apply_gridshift
@@ -709,6 +738,9 @@ pj_chomp(char*)
 pj_cleanup_lock
 pj_clear_initcache
 pj_compare_datums
+pj_context_get_grid_cache_filename(projCtx_t*)
+pj_context_get_user_writable_directory(projCtx_t*, bool)
+pj_context_is_network_enabled(projCtx_t*)
 pj_ctx_alloc
 pj_ctx_fclose
 pj_ctx_fgets
@@ -796,9 +828,14 @@ proj_context_get_use_proj4_init_rules
 proj_context_guess_wkt_dialect
 proj_context_set_autoclose_database
 proj_context_set_database_path
+proj_context_set_enable_network
+proj_context_set_fileapi
 proj_context_set_file_finder
+proj_context_set_network_callbacks
 proj_context_set(PJconsts*, projCtx_t*)
 proj_context_set_search_paths
+proj_context_set_sqlite3_vfs_name
+proj_context_set_url_endpoint
 proj_context_use_proj4_init_rules
 proj_convert_conversion_to_other_method
 proj_coord
@@ -928,6 +965,7 @@ proj_cs_get_axis_info
 proj_cs_get_type
 proj_destroy
 proj_dmstor
+proj_download_file
 proj_ellipsoid_get_parameters
 proj_errno
 proj_errno_reset
@@ -953,6 +991,11 @@ proj_get_scope
 proj_get_source_crs
 proj_get_target_crs
 proj_get_type
+proj_grid_cache_clear
+proj_grid_cache_set_enable
+proj_grid_cache_set_filename
+proj_grid_cache_set_max_size
+proj_grid_cache_set_ttl
 proj_grid_get_info_from_database
 proj_grid_info
 proj_identify
@@ -962,6 +1005,7 @@ proj_int_list_destroy
 proj_is_crs
 proj_is_deprecated
 proj_is_derived_crs
+proj_is_download_needed
 proj_is_equivalent_to
 proj_is_equivalent_to_with_ctx
 proj_list_angular_units
diff --git a/scripts/reformat_cpp.sh b/scripts/reformat_cpp.sh
index a8002a6c..20c32b29 100755
--- a/scripts/reformat_cpp.sh
+++ b/scripts/reformat_cpp.sh
@@ -15,7 +15,11 @@ esac
 
 TOPDIR="$SCRIPT_DIR/.."
 
-for i in "$TOPDIR"/include/proj/*.hpp "$TOPDIR"/include/proj/internal/*.hpp "$TOPDIR"/src/iso19111/*.cpp "$TOPDIR"/test/unit/*.cpp "$TOPDIR"/src/apps/projinfo.cpp "$TOPDIR"/src/tracing.cpp; do
+for i in "$TOPDIR"/include/proj/*.hpp "$TOPDIR"/include/proj/internal/*.hpp \
+         "$TOPDIR"/src/iso19111/*.cpp "$TOPDIR"/test/unit/*.cpp "$TOPDIR"/src/apps/projinfo.cpp \
+         "$TOPDIR"/src/tracing.cpp "$TOPDIR"/src/grids.hpp "$TOPDIR"/src/grids.cpp \
+         "$TOPDIR"/src/filemanager.hpp "$TOPDIR"/src/filemanager.cpp \
+         "$TOPDIR"/src/sqlite3_utils.hpp "$TOPDIR"/src/sqlite3_utils.cpp ; do
     if ! echo "$i" | grep -q "lru_cache.hpp"; then
         "$SCRIPT_DIR"/reformat.sh "$i";
     fi