Merge RFC4 (#1865)

This commit is the result of the squashing of rfc4_dev branch in a single
commit. It implements mostly RFC 4 related work.

* Grid handling:
  - remove obsolete and presumably unfinished implementation of grid catalog functionality
  - all grid functionality is in grids.cpp/.hpp
  - vertical and horizontal grid shift: rework to no longer load whole grid into memory
  - remove hgrids and vgrids member from PJ structure, and store them in hgridshift/vgridshift/deformation structures
  - build systems: add optional libtiff dependency. Must be explicitly disabled if not desired
  - add support for horizontal and vertical grids in GeoTIFF, if libtiff is available
  - add GenericShiftGridSet and GenericShiftGrid classes, relying on TIFF grids, that can be used for generic purpose grid-based adjustment
  - add a +proj=xyzgridshift method to perform geocentric translation by grid. Used for French NTF to RGF93 transformation using gr3df97a.tif grid
  - deformation: add support for +grids= for GeoTIFF grids
  - horizontal grid shift: fix failures on points slightly outside a subgrid (fixes #209)

* File management:
  - add a filemanager.cpp/.hpp to deal with file related work
  - test for legacy proj_api.h fileapi
  - proj.h: add proj_context_set_fileapi() and proj_context_set_sqlite3_vfs_name() (fixes #866)
  - add capability to read resource files from the user writable directory

* Network access:
  - build systems: add optional curl dependency
  - add a curl-based default implementation for network related functionality
  - proj.h: add C API to control network functionality, and optionaly provide network callbacks
  - add data/proj.ini with default settings
  - add a SQLite3 local cache of downloaded chunks
  - add proj_is_download_needed() and proj_download_file()

* Use Win32 Unicode APIs and expect all strings to be UTF-8 (fixes #1765)
  For backward compatibility, if PROJ_LIB content is found to be not UTF-8 or
  pointing to a non existing directory, then an attempt at interpretating it
  in the ANSI page encoding is done.
  proj_context_set_search_paths() now assumes strings to be in UTF-8, and
  functions returning paths will also return values in UTF-8.

3 files changed, 178 insertions, 186 deletions

diff --git a/src/iso19111/c_api.cpp b/src/iso19111/c_api.cpp
index 9f73f8e9..4c98af3d 100644
--- a/src/iso19111/c_api.cpp
+++ b/src/iso19111/c_api.cpp
@@ -177,7 +177,11 @@ static PJ *pj_obj_create(PJ_CONTEXT *ctx, const IdentifiedObjectNNPtr &objIn) {
             auto formatter = PROJStringFormatter::create(
                 PROJStringFormatter::Convention::PROJ_5, dbContext);
             auto projString = coordop->exportToPROJString(formatter.get());
+            if (pj_context_is_network_enabled(ctx)) {
+                ctx->defer_grid_opening = true;
+            }
             auto pj = pj_create_internal(ctx, projString.c_str());
+            ctx->defer_grid_opening = false;
             if (pj) {
                 pj->iso_obj = objIn;
                 if (ctx->cpp_context) {
@@ -766,7 +770,7 @@ int PROJ_DLL proj_grid_get_info_from_database(
         bool open_license;
         bool available;
         if (!db_context->lookForGridInfo(
-                grid_name, ctx->cpp_context->lastGridFullName_,
+                grid_name, false, ctx->cpp_context->lastGridFullName_,
                 ctx->cpp_context->lastGridPackageName_,
                 ctx->cpp_context->lastGridUrl_, direct_download, open_license,
                 available)) {
@@ -6615,7 +6619,10 @@ int proj_coordoperation_is_instantiable(PJ_CONTEXT *ctx,
     }
     auto dbContext = getDBcontextNoException(ctx, __FUNCTION__);
     try {
-        auto ret = op->isPROJInstantiable(dbContext) ? 1 : 0;
+        auto ret = op->isPROJInstantiable(dbContext,
+                                          pj_context_is_network_enabled(ctx))
+                       ? 1
+                       : 0;
         if (ctx->cpp_context) {
             ctx->cpp_context->autoCloseDbIfNeeded();
         }
@@ -6927,7 +6934,8 @@ int proj_coordoperation_get_grid_used_count(PJ_CONTEXT *ctx,
     try {
         if (!coordoperation->gridsNeededAsked) {
             coordoperation->gridsNeededAsked = true;
-            const auto gridsNeeded = co->gridsNeeded(dbContext);
+            const auto gridsNeeded =
+                co->gridsNeeded(dbContext, pj_context_is_network_enabled(ctx));
             for (const auto &gridDesc : gridsNeeded) {
                 coordoperation->gridsNeeded.emplace_back(gridDesc);
             }
@@ -7264,6 +7272,12 @@ void PROJ_DLL proj_operation_factory_context_set_grid_availability_use(
                 CoordinateOperationContext::GridAvailabilityUse::
                     IGNORE_GRID_AVAILABILITY);
             break;
+
+        case PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE:
+            factory_ctx->operationContext->setGridAvailabilityUse(
+                CoordinateOperationContext::GridAvailabilityUse::
+                    KNOWN_AVAILABLE);
+            break;
         }
     } catch (const std::exception &e) {
         proj_log_error(ctx, __FUNCTION__, e.what());
diff --git a/src/iso19111/coordinateoperation.cpp b/src/iso19111/coordinateoperation.cpp
index aaa0c413..a1650046 100644
--- a/src/iso19111/coordinateoperation.cpp
+++ b/src/iso19111/coordinateoperation.cpp
@@ -788,13 +788,15 @@ void CoordinateOperation::setAccuracies(
  * available.
  */
 bool CoordinateOperation::isPROJInstantiable(
-    const io::DatabaseContextPtr &databaseContext) const {
+    const io::DatabaseContextPtr &databaseContext,
+    bool considerKnownGridsAsAvailable) const {
     try {
         exportToPROJString(io::PROJStringFormatter::create().get());
     } catch (const std::exception &) {
         return false;
     }
-    for (const auto &gridDesc : gridsNeeded(databaseContext)) {
+    for (const auto &gridDesc :
+         gridsNeeded(databaseContext, considerKnownGridsAsAvailable)) {
         if (!gridDesc.available) {
             return false;
         }
@@ -2020,8 +2022,9 @@ bool SingleOperation::_isEquivalentTo(const util::IComparable *other,
 
 // ---------------------------------------------------------------------------
 
-std::set<GridDescription> SingleOperation::gridsNeeded(
-    const io::DatabaseContextPtr &databaseContext) const {
+std::set<GridDescription>
+SingleOperation::gridsNeeded(const io::DatabaseContextPtr &databaseContext,
+                             bool considerKnownGridsAsAvailable) const {
     std::set<GridDescription> res;
     for (const auto &genOpParamvalue : parameterValues()) {
         auto opParamvalue = dynamic_cast<const OperationParameterValue *>(
@@ -2033,9 +2036,9 @@ std::set<GridDescription> SingleOperation::gridsNeeded(
                 desc.shortName = value->valueFile();
                 if (databaseContext) {
                     databaseContext->lookForGridInfo(
-                        desc.shortName, desc.fullName, desc.packageName,
-                        desc.url, desc.directDownload, desc.openLicense,
-                        desc.available);
+                        desc.shortName, considerKnownGridsAsAvailable,
+                        desc.fullName, desc.packageName, desc.url,
+                        desc.directDownload, desc.openLicense, desc.available);
                 }
                 res.insert(desc);
             }
@@ -10233,10 +10236,12 @@ bool ConcatenatedOperation::_isEquivalentTo(
 // ---------------------------------------------------------------------------
 
 std::set<GridDescription> ConcatenatedOperation::gridsNeeded(
-    const io::DatabaseContextPtr &databaseContext) const {
+    const io::DatabaseContextPtr &databaseContext,
+    bool considerKnownGridsAsAvailable) const {
     std::set<GridDescription> res;
     for (const auto &operation : operations()) {
-        const auto l_gridsNeeded = operation->gridsNeeded(databaseContext);
+        const auto l_gridsNeeded = operation->gridsNeeded(
+            databaseContext, considerKnownGridsAsAvailable);
         for (const auto &gridDesc : l_gridsNeeded) {
             res.insert(gridDesc);
         }
@@ -11156,7 +11161,10 @@ struct FilterResults {
             bool gridsKnown = true;
             if (context->getAuthorityFactory()) {
                 const auto gridsNeeded = op->gridsNeeded(
-                    context->getAuthorityFactory()->databaseContext());
+                    context->getAuthorityFactory()->databaseContext(),
+                    gridAvailabilityUse ==
+                        CoordinateOperationContext::GridAvailabilityUse::
+                            KNOWN_AVAILABLE);
                 for (const auto &gridDesc : gridsNeeded) {
                     hasGrids = true;
                     if (gridAvailabilityUse ==
@@ -11278,6 +11286,7 @@ struct FilterResults {
         CoordinateOperationPtr lastOp;
 
         bool first = true;
+        const auto gridAvailabilityUse = context->getGridAvailabilityUse();
         for (const auto &op : res) {
             const auto curAccuracy = getAccuracy(op);
             bool dummy = false;
@@ -11290,7 +11299,10 @@ struct FilterResults {
 
             if (context->getAuthorityFactory()) {
                 const auto gridsNeeded = op->gridsNeeded(
-                    context->getAuthorityFactory()->databaseContext());
+                    context->getAuthorityFactory()->databaseContext(),
+                    gridAvailabilityUse ==
+                        CoordinateOperationContext::GridAvailabilityUse::
+                            KNOWN_AVAILABLE);
                 for (const auto &gridDesc : gridsNeeded) {
                     curHasGrids = true;
                     curSetOfGrids.insert(gridDesc.shortName);
@@ -11603,6 +11615,7 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirect(
     buildCRSIds(sourceCRS, context, sourceIds);
     buildCRSIds(targetCRS, context, targetIds);
 
+    const auto gridAvailabilityUse = context.context->getGridAvailabilityUse();
     for (const auto &idSrc : sourceIds) {
         const auto &srcAuthName = idSrc.first;
         const auto &srcCode = idSrc.second;
@@ -11622,9 +11635,16 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirect(
                     tmpAuthFactory->createFromCoordinateReferenceSystemCodes(
                         srcAuthName, srcCode, targetAuthName, targetCode,
                         context.context->getUsePROJAlternativeGridNames(),
-                        context.context->getGridAvailabilityUse() ==
+                        gridAvailabilityUse ==
+                                CoordinateOperationContext::
+                                    GridAvailabilityUse::
+                                        DISCARD_OPERATION_IF_MISSING_GRID ||
+                            gridAvailabilityUse ==
+                                CoordinateOperationContext::
+                                    GridAvailabilityUse::KNOWN_AVAILABLE,
+                        gridAvailabilityUse ==
                             CoordinateOperationContext::GridAvailabilityUse::
-                                DISCARD_OPERATION_IF_MISSING_GRID,
+                                KNOWN_AVAILABLE,
                         context.context->getDiscardSuperseded(), true, false,
                         context.extent1, context.extent2);
                 res.insert(res.end(), resTmp.begin(), resTmp.end());
@@ -11667,6 +11687,7 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirectTo(
     std::list<std::pair<std::string, std::string>> ids;
     buildCRSIds(targetCRS, context, ids);
 
+    const auto gridAvailabilityUse = context.context->getGridAvailabilityUse();
     for (const auto &id : ids) {
         const auto &targetAuthName = id.first;
         const auto &targetCode = id.second;
@@ -11680,9 +11701,15 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirectTo(
             auto res = tmpAuthFactory->createFromCoordinateReferenceSystemCodes(
                 std::string(), std::string(), targetAuthName, targetCode,
                 context.context->getUsePROJAlternativeGridNames(),
-                context.context->getGridAvailabilityUse() ==
-                    CoordinateOperationContext::GridAvailabilityUse::
-                        DISCARD_OPERATION_IF_MISSING_GRID,
+
+                gridAvailabilityUse ==
+                        CoordinateOperationContext::GridAvailabilityUse::
+                            DISCARD_OPERATION_IF_MISSING_GRID ||
+                    gridAvailabilityUse ==
+                        CoordinateOperationContext::GridAvailabilityUse::
+                            KNOWN_AVAILABLE,
+                gridAvailabilityUse == CoordinateOperationContext::
+                                           GridAvailabilityUse::KNOWN_AVAILABLE,
                 context.context->getDiscardSuperseded(), true, true,
                 context.extent1, context.extent2);
             if (!res.empty()) {
@@ -11730,6 +11757,7 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate(
     buildCRSIds(sourceCRS, context, sourceIds);
     buildCRSIds(targetCRS, context, targetIds);
 
+    const auto gridAvailabilityUse = context.context->getGridAvailabilityUse();
     for (const auto &idSrc : sourceIds) {
         const auto &srcAuthName = idSrc.first;
         const auto &srcCode = idSrc.second;
@@ -11749,21 +11777,28 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate(
 
             std::vector<CoordinateOperationNNPtr> res;
             if (useCreateBetweenGeodeticCRSWithDatumBasedIntermediates) {
-                res = tmpAuthFactory
-                          ->createBetweenGeodeticCRSWithDatumBasedIntermediates(
-                              sourceCRS, srcAuthName, srcCode, targetCRS,
-                              targetAuthName, targetCode,
-                              context.context->getUsePROJAlternativeGridNames(),
-                              context.context->getGridAvailabilityUse() ==
-                                  CoordinateOperationContext::
-                                      GridAvailabilityUse::
-                                          DISCARD_OPERATION_IF_MISSING_GRID,
-                              context.context->getDiscardSuperseded(),
-                              authFactory->getAuthority() != "any" &&
-                                      authorities.size() > 1
-                                  ? authorities
-                                  : std::vector<std::string>(),
-                              context.extent1, context.extent2);
+                res =
+                    tmpAuthFactory
+                        ->createBetweenGeodeticCRSWithDatumBasedIntermediates(
+                            sourceCRS, srcAuthName, srcCode, targetCRS,
+                            targetAuthName, targetCode,
+                            context.context->getUsePROJAlternativeGridNames(),
+                            gridAvailabilityUse ==
+                                    CoordinateOperationContext::
+                                        GridAvailabilityUse::
+                                            DISCARD_OPERATION_IF_MISSING_GRID ||
+                                gridAvailabilityUse ==
+                                    CoordinateOperationContext::
+                                        GridAvailabilityUse::KNOWN_AVAILABLE,
+                            gridAvailabilityUse ==
+                                CoordinateOperationContext::
+                                    GridAvailabilityUse::KNOWN_AVAILABLE,
+                            context.context->getDiscardSuperseded(),
+                            authFactory->getAuthority() != "any" &&
+                                    authorities.size() > 1
+                                ? authorities
+                                : std::vector<std::string>(),
+                            context.extent1, context.extent2);
             } else {
                 io::AuthorityFactory::ObjectType intermediateObjectType =
                     io::AuthorityFactory::ObjectType::CRS;
@@ -11781,9 +11816,15 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate(
                 res = tmpAuthFactory->createFromCRSCodesWithIntermediates(
                     srcAuthName, srcCode, targetAuthName, targetCode,
                     context.context->getUsePROJAlternativeGridNames(),
-                    context.context->getGridAvailabilityUse() ==
+                    gridAvailabilityUse ==
+                            CoordinateOperationContext::GridAvailabilityUse::
+                                DISCARD_OPERATION_IF_MISSING_GRID ||
+                        gridAvailabilityUse ==
+                            CoordinateOperationContext::GridAvailabilityUse::
+                                KNOWN_AVAILABLE,
+                    gridAvailabilityUse ==
                         CoordinateOperationContext::GridAvailabilityUse::
-                            DISCARD_OPERATION_IF_MISSING_GRID,
+                            KNOWN_AVAILABLE,
                     context.context->getDiscardSuperseded(),
                     context.context->getIntermediateCRS(),
                     intermediateObjectType,
@@ -14211,15 +14252,21 @@ void CoordinateOperationFactory::Private::createOperationsCompoundToGeog(
                 componentsSrc[1],
                 targetCRS->promoteTo3D(std::string(), dbContext), context);
             bool foundRegisteredTransformWithAllGridsAvailable = false;
+            const auto gridAvailabilityUse =
+                context.context->getGridAvailabilityUse();
             const bool ignoreMissingGrids =
-                context.context->getGridAvailabilityUse() ==
+                gridAvailabilityUse ==
                 CoordinateOperationContext::GridAvailabilityUse::
                     IGNORE_GRID_AVAILABILITY;
             for (const auto &op : verticalTransforms) {
                 if (hasIdentifiers(op) && dbContext) {
                     bool missingGrid = false;
                     if (!ignoreMissingGrids) {
-                        const auto gridsNeeded = op->gridsNeeded(dbContext);
+                        const auto gridsNeeded = op->gridsNeeded(
+                            dbContext,
+                            gridAvailabilityUse ==
+                                CoordinateOperationContext::
+                                    GridAvailabilityUse::KNOWN_AVAILABLE);
                         for (const auto &gridDesc : gridsNeeded) {
                             if (!gridDesc.available) {
                                 missingGrid = true;
@@ -14248,7 +14295,11 @@ void CoordinateOperationFactory::Private::createOperationsCompoundToGeog(
                     if (hasIdentifiers(op) && dbContext) {
                         bool missingGrid = false;
                         if (!ignoreMissingGrids) {
-                            const auto gridsNeeded = op->gridsNeeded(dbContext);
+                            const auto gridsNeeded = op->gridsNeeded(
+                                dbContext,
+                                gridAvailabilityUse ==
+                                    CoordinateOperationContext::
+                                        GridAvailabilityUse::KNOWN_AVAILABLE);
                             for (const auto &gridDesc : gridsNeeded) {
                                 if (!gridDesc.available) {
                                     missingGrid = true;
@@ -15079,8 +15130,9 @@ CoordinateOperationNNPtr PROJBasedOperation::_shallowClone() const {
 
 // ---------------------------------------------------------------------------
 
-std::set<GridDescription> PROJBasedOperation::gridsNeeded(
-    const io::DatabaseContextPtr &databaseContext) const {
+std::set<GridDescription>
+PROJBasedOperation::gridsNeeded(const io::DatabaseContextPtr &databaseContext,
+                                bool considerKnownGridsAsAvailable) const {
     std::set<GridDescription> res;
 
     try {
@@ -15093,7 +15145,8 @@ std::set<GridDescription> PROJBasedOperation::gridsNeeded(
             desc.shortName = shortName;
             if (databaseContext) {
                 databaseContext->lookForGridInfo(
-                    desc.shortName, desc.fullName, desc.packageName, desc.url,
+                    desc.shortName, considerKnownGridsAsAvailable,
+                    desc.fullName, desc.packageName, desc.url,
                     desc.directDownload, desc.openLicense, desc.available);
             }
             res.insert(desc);
diff --git a/src/iso19111/factory.cpp b/src/iso19111/factory.cpp
index 0c7692ad..7d15172e 100644
--- a/src/iso19111/factory.cpp
+++ b/src/iso19111/factory.cpp
@@ -45,6 +45,8 @@
 #include "proj/internal/lru_cache.hpp"
 #include "proj/internal/tracing.hpp"
 
+#include "sqlite3_utils.hpp"
+
 #include <cmath>
 #include <cstdlib>
 #include <cstring>
@@ -278,9 +280,7 @@ struct DatabaseContext::Private {
     void registerFunctions();
 
 #ifdef ENABLE_CUSTOM_LOCKLESS_VFS
-    std::string thisNamePtr_{};
-    sqlite3_vfs *vfs_{};
-    bool createCustomVFS();
+    std::unique_ptr<SQLite3VFS> vfs_{};
 #endif
 
     Private(const Private &) = delete;
@@ -297,13 +297,6 @@ DatabaseContext::Private::~Private() {
     assert(recLevel_ == 0);
 
     closeDB();
-
-#ifdef ENABLE_CUSTOM_LOCKLESS_VFS
-    if (vfs_) {
-        sqlite3_vfs_unregister(vfs_);
-        delete vfs_;
-    }
-#endif
 }
 
 // ---------------------------------------------------------------------------
@@ -499,104 +492,12 @@ void DatabaseContext::Private::cache(const std::string &code,
 
 // ---------------------------------------------------------------------------
 
-#ifdef ENABLE_CUSTOM_LOCKLESS_VFS
-
-typedef int (*ClosePtr)(sqlite3_file *);
-
-static int VFSClose(sqlite3_file *file) {
-    sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr);
-    assert(defaultVFS);
-    ClosePtr defaultClosePtr;
-    std::memcpy(&defaultClosePtr,
-                reinterpret_cast<char *>(file) + defaultVFS->szOsFile,
-                sizeof(ClosePtr));
-    void *methods = const_cast<sqlite3_io_methods *>(file->pMethods);
-    int ret = defaultClosePtr(file);
-    std::free(methods);
-    return ret;
-}
-
-// No-lock implementation
-static int VSFLock(sqlite3_file *, int) { return SQLITE_OK; }
-
-static int VSFUnlock(sqlite3_file *, int) { return SQLITE_OK; }
-
-static int VFSOpen(sqlite3_vfs *vfs, const char *name, sqlite3_file *file,
-                   int flags, int *outFlags) {
-    sqlite3_vfs *defaultVFS = static_cast<sqlite3_vfs *>(vfs->pAppData);
-    int ret = defaultVFS->xOpen(defaultVFS, name, file, flags, outFlags);
-    if (ret == SQLITE_OK) {
-        ClosePtr defaultClosePtr = file->pMethods->xClose;
-        assert(defaultClosePtr);
-        sqlite3_io_methods *methods = static_cast<sqlite3_io_methods *>(
-            std::malloc(sizeof(sqlite3_io_methods)));
-        if (!methods) {
-            file->pMethods->xClose(file);
-            return SQLITE_NOMEM;
-        }
-        memcpy(methods, file->pMethods, sizeof(sqlite3_io_methods));
-        methods->xClose = VFSClose;
-        methods->xLock = VSFLock;
-        methods->xUnlock = VSFUnlock;
-        file->pMethods = methods;
-        // Save original xClose pointer at end of file structure
-        std::memcpy(reinterpret_cast<char *>(file) + defaultVFS->szOsFile,
-                    &defaultClosePtr, sizeof(ClosePtr));
-    }
-    return ret;
-}
-
-static int VFSAccess(sqlite3_vfs *vfs, const char *zName, int flags,
-                     int *pResOut) {
-    sqlite3_vfs *defaultVFS = static_cast<sqlite3_vfs *>(vfs->pAppData);
-    // Do not bother stat'ing for journal or wal files
-    if (std::strstr(zName, "-journal") || std::strstr(zName, "-wal")) {
-        *pResOut = false;
-        return SQLITE_OK;
-    }
-    return defaultVFS->xAccess(defaultVFS, zName, flags, pResOut);
-}
-
-// ---------------------------------------------------------------------------
-
-bool DatabaseContext::Private::createCustomVFS() {
-
-    sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr);
-    assert(defaultVFS);
-
-    std::ostringstream buffer;
-    buffer << this;
-    thisNamePtr_ = buffer.str();
-
-    vfs_ = new sqlite3_vfs();
-    vfs_->iVersion = 1;
-    vfs_->szOsFile = defaultVFS->szOsFile + sizeof(ClosePtr);
-    vfs_->mxPathname = defaultVFS->mxPathname;
-    vfs_->zName = thisNamePtr_.c_str();
-    vfs_->pAppData = defaultVFS;
-    vfs_->xOpen = VFSOpen;
-    vfs_->xDelete = defaultVFS->xDelete;
-    vfs_->xAccess = VFSAccess;
-    vfs_->xFullPathname = defaultVFS->xFullPathname;
-    vfs_->xDlOpen = defaultVFS->xDlOpen;
-    vfs_->xDlError = defaultVFS->xDlError;
-    vfs_->xDlSym = defaultVFS->xDlSym;
-    vfs_->xDlClose = defaultVFS->xDlClose;
-    vfs_->xRandomness = defaultVFS->xRandomness;
-    vfs_->xSleep = defaultVFS->xSleep;
-    vfs_->xCurrentTime = defaultVFS->xCurrentTime;
-    vfs_->xGetLastError = defaultVFS->xGetLastError;
-    vfs_->xCurrentTimeInt64 = defaultVFS->xCurrentTimeInt64;
-    return sqlite3_vfs_register(vfs_, false) == SQLITE_OK;
-}
-
-#endif // ENABLE_CUSTOM_LOCKLESS_VFS
-
-// ---------------------------------------------------------------------------
-
 void DatabaseContext::Private::open(const std::string &databasePath,
                                     PJ_CONTEXT *ctx) {
-    setPjCtxt(ctx ? ctx : pj_get_default_ctx());
+    if (!ctx) {
+        ctx = pj_get_default_ctx();
+    }
+    setPjCtxt(ctx);
     std::string path(databasePath);
     if (path.empty()) {
         path.resize(2048);
@@ -608,18 +509,23 @@ void DatabaseContext::Private::open(const std::string &databasePath,
         }
     }
 
-    if (
+    std::string vfsName;
 #ifdef ENABLE_CUSTOM_LOCKLESS_VFS
-        !createCustomVFS() ||
+    if (ctx->custom_sqlite3_vfs_name.empty()) {
+        vfs_ = SQLite3VFS::create(false, true, true);
+        if (vfs_ == nullptr) {
+            throw FactoryException("Open of " + path + " failed");
+        }
+        vfsName = vfs_->name();
+    } else
 #endif
-        sqlite3_open_v2(path.c_str(), &sqlite_handle_,
+    {
+        vfsName = ctx->custom_sqlite3_vfs_name;
+    }
+    if (sqlite3_open_v2(path.c_str(), &sqlite_handle_,
                         SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX,
-#ifdef ENABLE_CUSTOM_LOCKLESS_VFS
-                        thisNamePtr_.c_str()
-#else
-                        nullptr
-#endif
-                            ) != SQLITE_OK ||
+                        vfsName.empty() ? nullptr : vfsName.c_str()) !=
+            SQLITE_OK ||
         !sqlite_handle_) {
         throw FactoryException("Open of " + path + " failed");
     }
@@ -1021,14 +927,14 @@ bool DatabaseContext::lookForGridAlternative(const std::string &officialName,
 
 // ---------------------------------------------------------------------------
 
-bool DatabaseContext::lookForGridInfo(const std::string &projFilename,
-                                      std::string &fullFilename,
-                                      std::string &packageName,
-                                      std::string &url, bool &directDownload,
-                                      bool &openLicense,
-                                      bool &gridAvailable) const {
+bool DatabaseContext::lookForGridInfo(
+    const std::string &projFilename, bool considerKnownGridsAsAvailable,
+    std::string &fullFilename, std::string &packageName, std::string &url,
+    bool &directDownload, bool &openLicense, bool &gridAvailable) const {
     Private::GridInfoCache info;
-    if (d->getGridInfoFromCache(projFilename, info)) {
+    const std::string key(projFilename +
+                          (considerKnownGridsAsAvailable ? "true" : "false"));
+    if (d->getGridInfoFromCache(key, info)) {
         fullFilename = info.fullFilename;
         packageName = info.packageName;
         url = info.url;
@@ -1044,16 +950,20 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename,
     openLicense = false;
     directDownload = false;
 
-    fullFilename.resize(2048);
-    if (d->pjCtxt() == nullptr) {
-        d->setPjCtxt(pj_get_default_ctx());
+    if (considerKnownGridsAsAvailable) {
+        fullFilename = projFilename;
+    } else {
+        fullFilename.resize(2048);
+        if (d->pjCtxt() == nullptr) {
+            d->setPjCtxt(pj_get_default_ctx());
+        }
+        int errno_before = proj_context_errno(d->pjCtxt());
+        gridAvailable =
+            pj_find_file(d->pjCtxt(), projFilename.c_str(), &fullFilename[0],
+                         fullFilename.size() - 1) != 0;
+        proj_context_errno_set(d->pjCtxt(), errno_before);
+        fullFilename.resize(strlen(fullFilename.c_str()));
     }
-    int errno_before = proj_context_errno(d->pjCtxt());
-    gridAvailable =
-        pj_find_file(d->pjCtxt(), projFilename.c_str(), &fullFilename[0],
-                     fullFilename.size() - 1) != 0;
-    proj_context_errno_set(d->pjCtxt(), errno_before);
-    fullFilename.resize(strlen(fullFilename.c_str()));
 
     auto res =
         d->run("SELECT "
@@ -1077,6 +987,10 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename,
         openLicense = (row[3].empty() ? row[4] : row[3]) == "1";
         directDownload = (row[5].empty() ? row[6] : row[5]) == "1";
 
+        if (considerKnownGridsAsAvailable && !packageName.empty()) {
+            gridAvailable = true;
+        }
+
         info.fullFilename = fullFilename;
         info.packageName = packageName;
         info.url = url;
@@ -1085,7 +999,7 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename,
     }
     info.gridAvailable = gridAvailable;
     info.found = ret;
-    d->cache(projFilename, info);
+    d->cache(key, info);
     return ret;
 }
 
@@ -1324,8 +1238,8 @@ struct AuthorityFactory::Private {
         return AuthorityFactory::create(context_, auth_name);
     }
 
-    bool
-    rejectOpDueToMissingGrid(const operation::CoordinateOperationNNPtr &op);
+    bool rejectOpDueToMissingGrid(const operation::CoordinateOperationNNPtr &op,
+                                  bool considerKnownGridsAsAvailable);
 
     UnitOfMeasure createUnitOfMeasure(const std::string &auth_name,
                                       const std::string &code);
@@ -1452,8 +1366,10 @@ util::PropertyMap AuthorityFactory::Private::createProperties(
 // ---------------------------------------------------------------------------
 
 bool AuthorityFactory::Private::rejectOpDueToMissingGrid(
-    const operation::CoordinateOperationNNPtr &op) {
-    for (const auto &gridDesc : op->gridsNeeded(context())) {
+    const operation::CoordinateOperationNNPtr &op,
+    bool considerKnownGridsAsAvailable) {
+    for (const auto &gridDesc :
+         op->gridsNeeded(context(), considerKnownGridsAsAvailable)) {
         if (!gridDesc.available) {
             return true;
         }
@@ -3449,7 +3365,7 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes(
     const std::string &sourceCRSCode, const std::string &targetCRSCode) const {
     return createFromCoordinateReferenceSystemCodes(
         d->authority(), sourceCRSCode, d->authority(), targetCRSCode, false,
-        false, false);
+        false, false, false);
 }
 
 // ---------------------------------------------------------------------------
@@ -3478,6 +3394,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes(
  * should be substituted to the official grid names.
  * @param discardIfMissingGrid Whether coordinate operations that reference
  * missing grids should be removed from the result set.
+ * @param considerKnownGridsAsAvailable Whether known grids should be considered
+ * as available (typically when network is enabled).
  * @param discardSuperseded Whether cordinate operations that are superseded
  * (but not deprecated) should be removed from the result set.
  * @param tryReverseOrder whether to search in the reverse order too (and thus
@@ -3498,8 +3416,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes(
     const std::string &sourceCRSAuthName, const std::string &sourceCRSCode,
     const std::string &targetCRSAuthName, const std::string &targetCRSCode,
     bool usePROJAlternativeGridNames, bool discardIfMissingGrid,
-    bool discardSuperseded, bool tryReverseOrder,
-    bool reportOnlyIntersectingTransformations,
+    bool considerKnownGridsAsAvailable, bool discardSuperseded,
+    bool tryReverseOrder, bool reportOnlyIntersectingTransformations,
     const metadata::ExtentPtr &intersectingExtent1,
     const metadata::ExtentPtr &intersectingExtent2) const {
 
@@ -3510,6 +3428,7 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes(
     cacheKey += targetCRSCode;
     cacheKey += (usePROJAlternativeGridNames ? '1' : '0');
     cacheKey += (discardIfMissingGrid ? '1' : '0');
+    cacheKey += (considerKnownGridsAsAvailable ? '1' : '0');
     cacheKey += (discardSuperseded ? '1' : '0');
     cacheKey += (tryReverseOrder ? '1' : '0');
     cacheKey += (reportOnlyIntersectingTransformations ? '1' : '0');
@@ -3748,7 +3667,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes(
                     target_crs_code != targetCRSCode))) {
             op = op->inverse();
         }
-        if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) {
+        if (!discardIfMissingGrid ||
+            !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) {
             list.emplace_back(op);
         }
     }
@@ -3813,6 +3733,8 @@ static bool useIrrelevantPivot(const operation::CoordinateOperationNNPtr &op,
  * should be substituted to the official grid names.
  * @param discardIfMissingGrid Whether coordinate operations that reference
  * missing grids should be removed from the result set.
+ * @param considerKnownGridsAsAvailable Whether known grids should be considered
+ * as available (typically when network is enabled).
  * @param discardSuperseded Whether cordinate operations that are superseded
  * (but not deprecated) should be removed from the result set.
  * @param intermediateCRSAuthCodes List of (auth_name, code) of CRS that can be
@@ -3841,7 +3763,7 @@ AuthorityFactory::createFromCRSCodesWithIntermediates(
     const std::string &sourceCRSAuthName, const std::string &sourceCRSCode,
     const std::string &targetCRSAuthName, const std::string &targetCRSCode,
     bool usePROJAlternativeGridNames, bool discardIfMissingGrid,
-    bool discardSuperseded,
+    bool considerKnownGridsAsAvailable, bool discardSuperseded,
     const std::vector<std::pair<std::string, std::string>>
         &intermediateCRSAuthCodes,
     ObjectType allowedIntermediateObjectType,
@@ -4289,7 +4211,8 @@ AuthorityFactory::createFromCRSCodesWithIntermediates(
 
     std::vector<operation::CoordinateOperationNNPtr> list;
     for (const auto &op : listTmp) {
-        if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) {
+        if (!discardIfMissingGrid ||
+            !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) {
             list.emplace_back(op);
         }
     }
@@ -4307,7 +4230,8 @@ AuthorityFactory::createBetweenGeodeticCRSWithDatumBasedIntermediates(
     const std::string &sourceCRSCode, const crs::CRSNNPtr &targetCRS,
     const std::string &targetCRSAuthName, const std::string &targetCRSCode,
     bool usePROJAlternativeGridNames, bool discardIfMissingGrid,
-    bool discardSuperseded, const std::vector<std::string> &allowedAuthorities,
+    bool considerKnownGridsAsAvailable, bool discardSuperseded,
+    const std::vector<std::string> &allowedAuthorities,
     const metadata::ExtentPtr &intersectingExtent1,
     const metadata::ExtentPtr &intersectingExtent2) const {
 
@@ -4890,7 +4814,8 @@ AuthorityFactory::createBetweenGeodeticCRSWithDatumBasedIntermediates(
 
     std::vector<operation::CoordinateOperationNNPtr> list;
     for (const auto &op : listTmp) {
-        if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) {
+        if (!discardIfMissingGrid ||
+            !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) {
             list.emplace_back(op);
         }
     }