hgridshift/vgridshift: defer grid opening when grid has already been opened

Relates to #2115

With the fix of https://github.com/OSGeo/PROJ/pull/2128, transforming between
EPSG:4326+3855 and EPSG:4269+5703 leads to many operations with many grids,
and opening a file handle for each operation saturates the limit of 1024
file handles opened simunalteously. This fix defers grid opening when a
transformation has already been instanciated with the same grid.

4 files changed, 96 insertions, 13 deletions

diff --git a/src/malloc.cpp b/src/malloc.cpp
index 9ac28546..10ab2fad 100644
--- a/src/malloc.cpp
+++ b/src/malloc.cpp
@@ -264,4 +264,6 @@ void proj_cleanup() {
   pj_clear_initcache();
   pj_deallocate_grids();
   FileManager::clearMemoryCache();
+  pj_clear_hgridshift_knowngrids_cache();
+  pj_clear_vgridshift_knowngrids_cache();
 }
diff --git a/src/proj_internal.h b/src/proj_internal.h
index 4dc63e1d..a7c1ddc8 100644
--- a/src/proj_internal.h
+++ b/src/proj_internal.h
@@ -890,6 +890,9 @@ int pj_get_suggested_operation(PJ_CONTEXT *ctx,
 const PJ_UNITS *pj_list_linear_units();
 const PJ_UNITS *pj_list_angular_units();
 
+void pj_clear_hgridshift_knowngrids_cache();
+void pj_clear_vgridshift_knowngrids_cache();
+
 /* classic public API */
 #include "proj_api.h"
 
diff --git a/src/transformations/hgridshift.cpp b/src/transformations/hgridshift.cpp
index 122a7ab2..b28eaf48 100644
--- a/src/transformations/hgridshift.cpp
+++ b/src/transformations/hgridshift.cpp
@@ -12,6 +12,25 @@ PROJ_HEAD(hgridshift, "Horizontal grid shift");
 
 using namespace NS_PROJ;
 
+#ifdef __MINGW32__
+// mingw32-win32 doesn't implement std::mutex
+namespace {
+class MyMutex {
+  public:
+    // cppcheck-suppress functionStatic
+    void lock() { pj_acquire_lock(); }
+    // cppcheck-suppress functionStatic
+    void unlock() { pj_release_lock(); }
+};
+}
+#else
+#include <mutex>
+#define MyMutex std::mutex
+#endif
+
+static MyMutex gMutex{};
+static std::set<std::string> gKnownGrids{};
+
 namespace { // anonymous namespace
 struct hgridshiftData {
     double t_final = 0;
@@ -160,18 +179,36 @@ PJ *TRANSFORMATION(hgridshift,0) {
     if (pj_param(P->ctx, P->params, "tt_epoch").i)
         Q->t_epoch = pj_param (P->ctx, P->params, "dt_epoch").f;
 
-
     if( P->ctx->defer_grid_opening ) {
         Q->defer_grid_opening = true;
     }
     else {
-        Q->grids = pj_hgrid_init(P, "grids");
-        /* Was gridlist compiled properly? */
-        if ( proj_errno(P) ) {
-            proj_log_error(P, "hgridshift: could not find required grid(s).");
-            return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID);
+        const char *gridnames = pj_param(P->ctx, P->params, "sgrids").s;
+        gMutex.lock();
+        const bool isKnownGrid = gKnownGrids.find(gridnames) != gKnownGrids.end();
+        gMutex.unlock();
+        if( isKnownGrid ) {
+            Q->defer_grid_opening = true;
         }
-     }
+        else {
+            Q->grids = pj_hgrid_init(P, "grids");
+            /* Was gridlist compiled properly? */
+            if ( proj_errno(P) ) {
+                proj_log_error(P, "hgridshift: could not find required grid(s).");
+                return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID);
+            }
+
+            gMutex.lock();
+            gKnownGrids.insert(gridnames);
+            gMutex.unlock();
+        }
+    }
 
     return P;
 }
+
+void pj_clear_hgridshift_knowngrids_cache() {
+    gMutex.lock();
+    gKnownGrids.clear();
+    gMutex.unlock();
+}
diff --git a/src/transformations/vgridshift.cpp b/src/transformations/vgridshift.cpp
index 121b795a..6ee8a6b0 100644
--- a/src/transformations/vgridshift.cpp
+++ b/src/transformations/vgridshift.cpp
@@ -12,6 +12,27 @@ PROJ_HEAD(vgridshift, "Vertical grid shift");
 
 using namespace NS_PROJ;
 
+
+#ifdef __MINGW32__
+// mingw32-win32 doesn't implement std::mutex
+namespace {
+class MyMutex {
+  public:
+    // cppcheck-suppress functionStatic
+    void lock() { pj_acquire_lock(); }
+    // cppcheck-suppress functionStatic
+    void unlock() { pj_release_lock(); }
+};
+}
+#else
+#include <mutex>
+#define MyMutex std::mutex
+#endif
+
+static MyMutex gMutex{};
+static std::set<std::string> gKnownGrids{};
+
+
 namespace { // anonymous namespace
 struct vgridshiftData {
     double t_final = 0;
@@ -192,13 +213,27 @@ PJ *TRANSFORMATION(vgridshift,0) {
         Q->defer_grid_opening = true;
     }
     else {
-        /* Build gridlist. P->vgridlist_geoid can be empty if +grids only ask for optional grids. */
-        Q->grids = pj_vgrid_init(P, "grids");
+        const char *gridnames = pj_param(P->ctx, P->params, "sgrids").s;
+        gMutex.lock();
+        const bool isKnownGrid = gKnownGrids.find(gridnames) != gKnownGrids.end();
+        gMutex.unlock();
 
-        /* Was gridlist compiled properly? */
-        if ( proj_errno(P) ) {
-            proj_log_error(P, "vgridshift: could not find required grid(s).");
-            return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID);
+        if( isKnownGrid ) {
+            Q->defer_grid_opening = true;
+        }
+        else {
+            /* Build gridlist. P->vgridlist_geoid can be empty if +grids only ask for optional grids. */
+            Q->grids = pj_vgrid_init(P, "grids");
+
+            /* Was gridlist compiled properly? */
+            if ( proj_errno(P) ) {
+                proj_log_error(P, "vgridshift: could not find required grid(s).");
+                return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID);
+            }
+
+            gMutex.lock();
+            gKnownGrids.insert(gridnames);
+            gMutex.unlock();
         }
     }
 
@@ -214,3 +249,9 @@ PJ *TRANSFORMATION(vgridshift,0) {
 
     return P;
 }
+
+void pj_clear_vgridshift_knowngrids_cache() {
+    gMutex.lock();
+    gKnownGrids.clear();
+    gMutex.unlock();
+}