Merge RFC4 (#1865)

This commit is the result of the squashing of rfc4_dev branch in a single
commit. It implements mostly RFC 4 related work.

* Grid handling:
  - remove obsolete and presumably unfinished implementation of grid catalog functionality
  - all grid functionality is in grids.cpp/.hpp
  - vertical and horizontal grid shift: rework to no longer load whole grid into memory
  - remove hgrids and vgrids member from PJ structure, and store them in hgridshift/vgridshift/deformation structures
  - build systems: add optional libtiff dependency. Must be explicitly disabled if not desired
  - add support for horizontal and vertical grids in GeoTIFF, if libtiff is available
  - add GenericShiftGridSet and GenericShiftGrid classes, relying on TIFF grids, that can be used for generic purpose grid-based adjustment
  - add a +proj=xyzgridshift method to perform geocentric translation by grid. Used for French NTF to RGF93 transformation using gr3df97a.tif grid
  - deformation: add support for +grids= for GeoTIFF grids
  - horizontal grid shift: fix failures on points slightly outside a subgrid (fixes #209)

* File management:
  - add a filemanager.cpp/.hpp to deal with file related work
  - test for legacy proj_api.h fileapi
  - proj.h: add proj_context_set_fileapi() and proj_context_set_sqlite3_vfs_name() (fixes #866)
  - add capability to read resource files from the user writable directory

* Network access:
  - build systems: add optional curl dependency
  - add a curl-based default implementation for network related functionality
  - proj.h: add C API to control network functionality, and optionaly provide network callbacks
  - add data/proj.ini with default settings
  - add a SQLite3 local cache of downloaded chunks
  - add proj_is_download_needed() and proj_download_file()

* Use Win32 Unicode APIs and expect all strings to be UTF-8 (fixes #1765)
  For backward compatibility, if PROJ_LIB content is found to be not UTF-8 or
  pointing to a non existing directory, then an attempt at interpretating it
  in the ANSI page encoding is done.
  proj_context_set_search_paths() now assumes strings to be in UTF-8, and
  functions returning paths will also return values in UTF-8.

1 files changed, 92 insertions, 39 deletions

diff --git a/src/iso19111/coordinateoperation.cpp b/src/iso19111/coordinateoperation.cpp
index aaa0c413..a1650046 100644
--- a/src/iso19111/coordinateoperation.cpp
+++ b/src/iso19111/coordinateoperation.cpp
@@ -788,13 +788,15 @@ void CoordinateOperation::setAccuracies(
  * available.
  */
 bool CoordinateOperation::isPROJInstantiable(
-    const io::DatabaseContextPtr &databaseContext) const {
+    const io::DatabaseContextPtr &databaseContext,
+    bool considerKnownGridsAsAvailable) const {
     try {
         exportToPROJString(io::PROJStringFormatter::create().get());
     } catch (const std::exception &) {
         return false;
     }
-    for (const auto &gridDesc : gridsNeeded(databaseContext)) {
+    for (const auto &gridDesc :
+         gridsNeeded(databaseContext, considerKnownGridsAsAvailable)) {
         if (!gridDesc.available) {
             return false;
         }
@@ -2020,8 +2022,9 @@ bool SingleOperation::_isEquivalentTo(const util::IComparable *other,
 
 // ---------------------------------------------------------------------------
 
-std::set<GridDescription> SingleOperation::gridsNeeded(
-    const io::DatabaseContextPtr &databaseContext) const {
+std::set<GridDescription>
+SingleOperation::gridsNeeded(const io::DatabaseContextPtr &databaseContext,
+                             bool considerKnownGridsAsAvailable) const {
     std::set<GridDescription> res;
     for (const auto &genOpParamvalue : parameterValues()) {
         auto opParamvalue = dynamic_cast<const OperationParameterValue *>(
@@ -2033,9 +2036,9 @@ std::set<GridDescription> SingleOperation::gridsNeeded(
                 desc.shortName = value->valueFile();
                 if (databaseContext) {
                     databaseContext->lookForGridInfo(
-                        desc.shortName, desc.fullName, desc.packageName,
-                        desc.url, desc.directDownload, desc.openLicense,
-                        desc.available);
+                        desc.shortName, considerKnownGridsAsAvailable,
+                        desc.fullName, desc.packageName, desc.url,
+                        desc.directDownload, desc.openLicense, desc.available);
                 }
                 res.insert(desc);
             }
@@ -10233,10 +10236,12 @@ bool ConcatenatedOperation::_isEquivalentTo(
 // ---------------------------------------------------------------------------
 
 std::set<GridDescription> ConcatenatedOperation::gridsNeeded(
-    const io::DatabaseContextPtr &databaseContext) const {
+    const io::DatabaseContextPtr &databaseContext,
+    bool considerKnownGridsAsAvailable) const {
     std::set<GridDescription> res;
     for (const auto &operation : operations()) {
-        const auto l_gridsNeeded = operation->gridsNeeded(databaseContext);
+        const auto l_gridsNeeded = operation->gridsNeeded(
+            databaseContext, considerKnownGridsAsAvailable);
         for (const auto &gridDesc : l_gridsNeeded) {
             res.insert(gridDesc);
         }
@@ -11156,7 +11161,10 @@ struct FilterResults {
             bool gridsKnown = true;
             if (context->getAuthorityFactory()) {
                 const auto gridsNeeded = op->gridsNeeded(
-                    context->getAuthorityFactory()->databaseContext());
+                    context->getAuthorityFactory()->databaseContext(),
+                    gridAvailabilityUse ==
+                        CoordinateOperationContext::GridAvailabilityUse::
+                            KNOWN_AVAILABLE);
                 for (const auto &gridDesc : gridsNeeded) {
                     hasGrids = true;
                     if (gridAvailabilityUse ==
@@ -11278,6 +11286,7 @@ struct FilterResults {
         CoordinateOperationPtr lastOp;
 
         bool first = true;
+        const auto gridAvailabilityUse = context->getGridAvailabilityUse();
         for (const auto &op : res) {
             const auto curAccuracy = getAccuracy(op);
             bool dummy = false;
@@ -11290,7 +11299,10 @@ struct FilterResults {
 
             if (context->getAuthorityFactory()) {
                 const auto gridsNeeded = op->gridsNeeded(
-                    context->getAuthorityFactory()->databaseContext());
+                    context->getAuthorityFactory()->databaseContext(),
+                    gridAvailabilityUse ==
+                        CoordinateOperationContext::GridAvailabilityUse::
+                            KNOWN_AVAILABLE);
                 for (const auto &gridDesc : gridsNeeded) {
                     curHasGrids = true;
                     curSetOfGrids.insert(gridDesc.shortName);
@@ -11603,6 +11615,7 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirect(
     buildCRSIds(sourceCRS, context, sourceIds);
     buildCRSIds(targetCRS, context, targetIds);
 
+    const auto gridAvailabilityUse = context.context->getGridAvailabilityUse();
     for (const auto &idSrc : sourceIds) {
         const auto &srcAuthName = idSrc.first;
         const auto &srcCode = idSrc.second;
@@ -11622,9 +11635,16 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirect(
                     tmpAuthFactory->createFromCoordinateReferenceSystemCodes(
                         srcAuthName, srcCode, targetAuthName, targetCode,
                         context.context->getUsePROJAlternativeGridNames(),
-                        context.context->getGridAvailabilityUse() ==
+                        gridAvailabilityUse ==
+                                CoordinateOperationContext::
+                                    GridAvailabilityUse::
+                                        DISCARD_OPERATION_IF_MISSING_GRID ||
+                            gridAvailabilityUse ==
+                                CoordinateOperationContext::
+                                    GridAvailabilityUse::KNOWN_AVAILABLE,
+                        gridAvailabilityUse ==
                             CoordinateOperationContext::GridAvailabilityUse::
-                                DISCARD_OPERATION_IF_MISSING_GRID,
+                                KNOWN_AVAILABLE,
                         context.context->getDiscardSuperseded(), true, false,
                         context.extent1, context.extent2);
                 res.insert(res.end(), resTmp.begin(), resTmp.end());
@@ -11667,6 +11687,7 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirectTo(
     std::list<std::pair<std::string, std::string>> ids;
     buildCRSIds(targetCRS, context, ids);
 
+    const auto gridAvailabilityUse = context.context->getGridAvailabilityUse();
     for (const auto &id : ids) {
         const auto &targetAuthName = id.first;
         const auto &targetCode = id.second;
@@ -11680,9 +11701,15 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirectTo(
             auto res = tmpAuthFactory->createFromCoordinateReferenceSystemCodes(
                 std::string(), std::string(), targetAuthName, targetCode,
                 context.context->getUsePROJAlternativeGridNames(),
-                context.context->getGridAvailabilityUse() ==
-                    CoordinateOperationContext::GridAvailabilityUse::
-                        DISCARD_OPERATION_IF_MISSING_GRID,
+
+                gridAvailabilityUse ==
+                        CoordinateOperationContext::GridAvailabilityUse::
+                            DISCARD_OPERATION_IF_MISSING_GRID ||
+                    gridAvailabilityUse ==
+                        CoordinateOperationContext::GridAvailabilityUse::
+                            KNOWN_AVAILABLE,
+                gridAvailabilityUse == CoordinateOperationContext::
+                                           GridAvailabilityUse::KNOWN_AVAILABLE,
                 context.context->getDiscardSuperseded(), true, true,
                 context.extent1, context.extent2);
             if (!res.empty()) {
@@ -11730,6 +11757,7 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate(
     buildCRSIds(sourceCRS, context, sourceIds);
     buildCRSIds(targetCRS, context, targetIds);
 
+    const auto gridAvailabilityUse = context.context->getGridAvailabilityUse();
     for (const auto &idSrc : sourceIds) {
         const auto &srcAuthName = idSrc.first;
         const auto &srcCode = idSrc.second;
@@ -11749,21 +11777,28 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate(
 
             std::vector<CoordinateOperationNNPtr> res;
             if (useCreateBetweenGeodeticCRSWithDatumBasedIntermediates) {
-                res = tmpAuthFactory
-                          ->createBetweenGeodeticCRSWithDatumBasedIntermediates(
-                              sourceCRS, srcAuthName, srcCode, targetCRS,
-                              targetAuthName, targetCode,
-                              context.context->getUsePROJAlternativeGridNames(),
-                              context.context->getGridAvailabilityUse() ==
-                                  CoordinateOperationContext::
-                                      GridAvailabilityUse::
-                                          DISCARD_OPERATION_IF_MISSING_GRID,
-                              context.context->getDiscardSuperseded(),
-                              authFactory->getAuthority() != "any" &&
-                                      authorities.size() > 1
-                                  ? authorities
-                                  : std::vector<std::string>(),
-                              context.extent1, context.extent2);
+                res =
+                    tmpAuthFactory
+                        ->createBetweenGeodeticCRSWithDatumBasedIntermediates(
+                            sourceCRS, srcAuthName, srcCode, targetCRS,
+                            targetAuthName, targetCode,
+                            context.context->getUsePROJAlternativeGridNames(),
+                            gridAvailabilityUse ==
+                                    CoordinateOperationContext::
+                                        GridAvailabilityUse::
+                                            DISCARD_OPERATION_IF_MISSING_GRID ||
+                                gridAvailabilityUse ==
+                                    CoordinateOperationContext::
+                                        GridAvailabilityUse::KNOWN_AVAILABLE,
+                            gridAvailabilityUse ==
+                                CoordinateOperationContext::
+                                    GridAvailabilityUse::KNOWN_AVAILABLE,
+                            context.context->getDiscardSuperseded(),
+                            authFactory->getAuthority() != "any" &&
+                                    authorities.size() > 1
+                                ? authorities
+                                : std::vector<std::string>(),
+                            context.extent1, context.extent2);
             } else {
                 io::AuthorityFactory::ObjectType intermediateObjectType =
                     io::AuthorityFactory::ObjectType::CRS;
@@ -11781,9 +11816,15 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate(
                 res = tmpAuthFactory->createFromCRSCodesWithIntermediates(
                     srcAuthName, srcCode, targetAuthName, targetCode,
                     context.context->getUsePROJAlternativeGridNames(),
-                    context.context->getGridAvailabilityUse() ==
+                    gridAvailabilityUse ==
+                            CoordinateOperationContext::GridAvailabilityUse::
+                                DISCARD_OPERATION_IF_MISSING_GRID ||
+                        gridAvailabilityUse ==
+                            CoordinateOperationContext::GridAvailabilityUse::
+                                KNOWN_AVAILABLE,
+                    gridAvailabilityUse ==
                         CoordinateOperationContext::GridAvailabilityUse::
-                            DISCARD_OPERATION_IF_MISSING_GRID,
+                            KNOWN_AVAILABLE,
                     context.context->getDiscardSuperseded(),
                     context.context->getIntermediateCRS(),
                     intermediateObjectType,
@@ -14211,15 +14252,21 @@ void CoordinateOperationFactory::Private::createOperationsCompoundToGeog(
                 componentsSrc[1],
                 targetCRS->promoteTo3D(std::string(), dbContext), context);
             bool foundRegisteredTransformWithAllGridsAvailable = false;
+            const auto gridAvailabilityUse =
+                context.context->getGridAvailabilityUse();
             const bool ignoreMissingGrids =
-                context.context->getGridAvailabilityUse() ==
+                gridAvailabilityUse ==
                 CoordinateOperationContext::GridAvailabilityUse::
                     IGNORE_GRID_AVAILABILITY;
             for (const auto &op : verticalTransforms) {
                 if (hasIdentifiers(op) && dbContext) {
                     bool missingGrid = false;
                     if (!ignoreMissingGrids) {
-                        const auto gridsNeeded = op->gridsNeeded(dbContext);
+                        const auto gridsNeeded = op->gridsNeeded(
+                            dbContext,
+                            gridAvailabilityUse ==
+                                CoordinateOperationContext::
+                                    GridAvailabilityUse::KNOWN_AVAILABLE);
                         for (const auto &gridDesc : gridsNeeded) {
                             if (!gridDesc.available) {
                                 missingGrid = true;
@@ -14248,7 +14295,11 @@ void CoordinateOperationFactory::Private::createOperationsCompoundToGeog(
                     if (hasIdentifiers(op) && dbContext) {
                         bool missingGrid = false;
                         if (!ignoreMissingGrids) {
-                            const auto gridsNeeded = op->gridsNeeded(dbContext);
+                            const auto gridsNeeded = op->gridsNeeded(
+                                dbContext,
+                                gridAvailabilityUse ==
+                                    CoordinateOperationContext::
+                                        GridAvailabilityUse::KNOWN_AVAILABLE);
                             for (const auto &gridDesc : gridsNeeded) {
                                 if (!gridDesc.available) {
                                     missingGrid = true;
@@ -15079,8 +15130,9 @@ CoordinateOperationNNPtr PROJBasedOperation::_shallowClone() const {
 
 // ---------------------------------------------------------------------------
 
-std::set<GridDescription> PROJBasedOperation::gridsNeeded(
-    const io::DatabaseContextPtr &databaseContext) const {
+std::set<GridDescription>
+PROJBasedOperation::gridsNeeded(const io::DatabaseContextPtr &databaseContext,
+                                bool considerKnownGridsAsAvailable) const {
     std::set<GridDescription> res;
 
     try {
@@ -15093,7 +15145,8 @@ std::set<GridDescription> PROJBasedOperation::gridsNeeded(
             desc.shortName = shortName;
             if (databaseContext) {
                 databaseContext->lookForGridInfo(
-                    desc.shortName, desc.fullName, desc.packageName, desc.url,
+                    desc.shortName, considerKnownGridsAsAvailable,
+                    desc.fullName, desc.packageName, desc.url,
                     desc.directDownload, desc.openLicense, desc.available);
             }
             res.insert(desc);