Merge RFC4 (#1865)

This commit is the result of the squashing of rfc4_dev branch in a single
commit. It implements mostly RFC 4 related work.

* Grid handling:
  - remove obsolete and presumably unfinished implementation of grid catalog functionality
  - all grid functionality is in grids.cpp/.hpp
  - vertical and horizontal grid shift: rework to no longer load whole grid into memory
  - remove hgrids and vgrids member from PJ structure, and store them in hgridshift/vgridshift/deformation structures
  - build systems: add optional libtiff dependency. Must be explicitly disabled if not desired
  - add support for horizontal and vertical grids in GeoTIFF, if libtiff is available
  - add GenericShiftGridSet and GenericShiftGrid classes, relying on TIFF grids, that can be used for generic purpose grid-based adjustment
  - add a +proj=xyzgridshift method to perform geocentric translation by grid. Used for French NTF to RGF93 transformation using gr3df97a.tif grid
  - deformation: add support for +grids= for GeoTIFF grids
  - horizontal grid shift: fix failures on points slightly outside a subgrid (fixes #209)

* File management:
  - add a filemanager.cpp/.hpp to deal with file related work
  - test for legacy proj_api.h fileapi
  - proj.h: add proj_context_set_fileapi() and proj_context_set_sqlite3_vfs_name() (fixes #866)
  - add capability to read resource files from the user writable directory

* Network access:
  - build systems: add optional curl dependency
  - add a curl-based default implementation for network related functionality
  - proj.h: add C API to control network functionality, and optionaly provide network callbacks
  - add data/proj.ini with default settings
  - add a SQLite3 local cache of downloaded chunks
  - add proj_is_download_needed() and proj_download_file()

* Use Win32 Unicode APIs and expect all strings to be UTF-8 (fixes #1765)
  For backward compatibility, if PROJ_LIB content is found to be not UTF-8 or
  pointing to a non existing directory, then an attempt at interpretating it
  in the ANSI page encoding is done.
  proj_context_set_search_paths() now assumes strings to be in UTF-8, and
  functions returning paths will also return values in UTF-8.

3 files changed, 29 insertions, 12 deletions

diff --git a/include/proj/coordinateoperation.hpp b/include/proj/coordinateoperation.hpp
index 81d82474..c7459416 100644
--- a/include/proj/coordinateoperation.hpp
+++ b/include/proj/coordinateoperation.hpp
@@ -144,10 +144,12 @@ class PROJ_GCC_DLL CoordinateOperation : public common::ObjectUsage,
 
     /** \brief Return grids needed by an operation. */
     PROJ_DLL virtual std::set<GridDescription>
-    gridsNeeded(const io::DatabaseContextPtr &databaseContext) const = 0;
+    gridsNeeded(const io::DatabaseContextPtr &databaseContext,
+                bool considerKnownGridsAsAvailable) const = 0;
 
     PROJ_DLL bool
-    isPROJInstantiable(const io::DatabaseContextPtr &databaseContext) const;
+    isPROJInstantiable(const io::DatabaseContextPtr &databaseContext,
+                       bool considerKnownGridsAsAvailable) const;
 
     PROJ_DLL bool hasBallparkTransformation() const;
 
@@ -601,7 +603,8 @@ class PROJ_GCC_DLL SingleOperation : virtual public CoordinateOperation {
             std::vector<metadata::PositionalAccuracyNNPtr>());
 
     PROJ_DLL std::set<GridDescription>
-    gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override;
+    gridsNeeded(const io::DatabaseContextPtr &databaseContext,
+                bool considerKnownGridsAsAvailable) const override;
 
     PROJ_DLL std::list<std::string> validateParameters() const;
 
@@ -1680,7 +1683,8 @@ class PROJ_GCC_DLL ConcatenatedOperation final : public CoordinateOperation {
         bool checkExtent); // throw InvalidOperation
 
     PROJ_DLL std::set<GridDescription>
-    gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override;
+    gridsNeeded(const io::DatabaseContextPtr &databaseContext,
+                bool considerKnownGridsAsAvailable) const override;
 
     PROJ_PRIVATE :
 
@@ -1810,6 +1814,12 @@ class PROJ_GCC_DLL CoordinateOperationContext {
         /** Ignore grid availability at all. Results will be presented as if
          * all grids were available. */
         IGNORE_GRID_AVAILABILITY,
+
+        /** Results will be presented as if grids known to PROJ (that is
+        * registered in the grid_alternatives table of its database) were
+        * available. Used typically when networking is enabled.
+        */
+        KNOWN_AVAILABLE,
     };
 
     PROJ_DLL void setGridAvailabilityUse(GridAvailabilityUse use);
diff --git a/include/proj/internal/coordinateoperation_internal.hpp b/include/proj/internal/coordinateoperation_internal.hpp
index 361261c1..a75427c2 100644
--- a/include/proj/internal/coordinateoperation_internal.hpp
+++ b/include/proj/internal/coordinateoperation_internal.hpp
@@ -180,8 +180,10 @@ class InverseConversion : public Conversion, public InverseCoordinateOperation {
     // 'osgeo::proj::operation::SingleOperation::osgeo::proj::operation::SingleOperation::gridsNeeded'
     // via dominance
     std::set<GridDescription>
-    gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override {
-        return SingleOperation::gridsNeeded(databaseContext);
+    gridsNeeded(const io::DatabaseContextPtr &databaseContext,
+                bool considerKnownGridsAsAvailable) const override {
+        return SingleOperation::gridsNeeded(databaseContext,
+                                            considerKnownGridsAsAvailable);
     }
 #endif
 
@@ -232,8 +234,10 @@ class InverseTransformation : public Transformation,
     // 'osgeo::proj::operation::SingleOperation::osgeo::proj::operation::SingleOperation::gridsNeeded'
     // via dominance
     std::set<GridDescription>
-    gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override {
-        return SingleOperation::gridsNeeded(databaseContext);
+    gridsNeeded(const io::DatabaseContextPtr &databaseContext,
+                bool considerKnownGridsAsAvailable) const override {
+        return SingleOperation::gridsNeeded(databaseContext,
+                                            considerKnownGridsAsAvailable);
     }
 #endif
 
@@ -274,7 +278,8 @@ class PROJBasedOperation : public SingleOperation {
            bool hasRoughTransformation);
 
     std::set<GridDescription>
-    gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override;
+    gridsNeeded(const io::DatabaseContextPtr &databaseContext,
+                bool considerKnownGridsAsAvailable) const override;
 
   protected:
     PROJBasedOperation(const PROJBasedOperation &) = default;
diff --git a/include/proj/io.hpp b/include/proj/io.hpp
index 37941901..e2f48dd4 100644
--- a/include/proj/io.hpp
+++ b/include/proj/io.hpp
@@ -832,6 +832,7 @@ class PROJ_GCC_DLL DatabaseContext {
                                               bool &inverse) const;
 
     PROJ_DLL bool lookForGridInfo(const std::string &projFilename,
+                                  bool considerKnownGridsAsAvailable,
                                   std::string &fullFilename,
                                   std::string &packageName, std::string &url,
                                   bool &directDownload, bool &openLicense,
@@ -1055,7 +1056,8 @@ class PROJ_GCC_DLL AuthorityFactory {
         const std::string &sourceCRSAuthName, const std::string &sourceCRSCode,
         const std::string &targetCRSAuthName, const std::string &targetCRSCode,
         bool usePROJAlternativeGridNames, bool discardIfMissingGrid,
-        bool discardSuperseded, bool tryReverseOrder = false,
+        bool considerKnownGridsAsAvailable, bool discardSuperseded,
+        bool tryReverseOrder = false,
         bool reportOnlyIntersectingTransformations = false,
         const metadata::ExtentPtr &intersectingExtent1 = nullptr,
         const metadata::ExtentPtr &intersectingExtent2 = nullptr) const;
@@ -1065,7 +1067,7 @@ class PROJ_GCC_DLL AuthorityFactory {
         const std::string &sourceCRSAuthName, const std::string &sourceCRSCode,
         const std::string &targetCRSAuthName, const std::string &targetCRSCode,
         bool usePROJAlternativeGridNames, bool discardIfMissingGrid,
-        bool discardSuperseded,
+        bool considerKnownGridsAsAvailable, bool discardSuperseded,
         const std::vector<std::pair<std::string, std::string>>
             &intermediateCRSAuthCodes,
         ObjectType allowedIntermediateObjectType = ObjectType::CRS,
@@ -1131,7 +1133,7 @@ class PROJ_GCC_DLL AuthorityFactory {
         const std::string &sourceCRSCode, const crs::CRSNNPtr &targetCRS,
         const std::string &targetCRSAuthName, const std::string &targetCRSCode,
         bool usePROJAlternativeGridNames, bool discardIfMissingGrid,
-        bool discardSuperseded,
+        bool considerKnownGridsAsAvailable, bool discardSuperseded,
         const std::vector<std::string> &allowedAuthorities,
         const metadata::ExtentPtr &intersectingExtent1,
         const metadata::ExtentPtr &intersectingExtent2) const;