Remote grid: add mechanism to re-open a grid if it has changed while being opened

8 files changed, 352 insertions, 126 deletions

diff --git a/src/filemanager.cpp b/src/filemanager.cpp
index 2d925f07..1a94216d 100644
--- a/src/filemanager.cpp
+++ b/src/filemanager.cpp
@@ -135,6 +135,9 @@ class FileStdio : public File {
     unsigned long long tell() override;
     void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; }
 
+    // We may lie, but the real use case is only for network files
+    bool hasChanged() const override { return false; }
+
     static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename);
 };
 
@@ -198,6 +201,9 @@ class FileLegacyAdapter : public File {
     unsigned long long tell() override;
     void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; }
 
+    // We may lie, but the real use case is only for network files
+    bool hasChanged() const override { return false; }
+
     static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename);
 };
 
@@ -1337,19 +1343,24 @@ class NetworkFile : public File {
     unsigned long long m_pos = 0;
     size_t m_nBlocksToDownload = 1;
     unsigned long long m_lastDownloadedOffset;
-    unsigned long long m_filesize;
+    FileProperties m_props;
     proj_network_close_cbk_type m_closeCbk;
+    bool m_hasChanged = false;
 
     NetworkFile(const NetworkFile &) = delete;
     NetworkFile &operator=(const NetworkFile &) = delete;
 
+    static bool get_props_from_headers(PJ_CONTEXT *ctx,
+                                       PROJ_NETWORK_HANDLE *handle,
+                                       FileProperties &props);
+
   protected:
     NetworkFile(PJ_CONTEXT *ctx, const std::string &url,
                 PROJ_NETWORK_HANDLE *handle,
                 unsigned long long lastDownloadOffset,
-                unsigned long long filesize)
+                const FileProperties &props)
         : File(url), m_ctx(ctx), m_url(url), m_handle(handle),
-          m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize),
+          m_lastDownloadedOffset(lastDownloadOffset), m_props(props),
           m_closeCbk(ctx->networking.close) {}
 
   public:
@@ -1359,18 +1370,47 @@ class NetworkFile : public File {
     bool seek(unsigned long long offset, int whence) override;
     unsigned long long tell() override;
     void reassign_context(PJ_CONTEXT *ctx) override;
+    bool hasChanged() const override { return m_hasChanged; }
 
     static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename);
 };
 
 // ---------------------------------------------------------------------------
 
+bool NetworkFile::get_props_from_headers(PJ_CONTEXT *ctx,
+                                         PROJ_NETWORK_HANDLE *handle,
+                                         FileProperties &props) {
+    const char *contentRange = ctx->networking.get_header_value(
+        ctx, handle, "Content-Range", ctx->networking.user_data);
+    if (contentRange) {
+        const char *slash = strchr(contentRange, '/');
+        if (slash) {
+            props.size = std::stoull(slash + 1);
+
+            const char *lastModified = ctx->networking.get_header_value(
+                ctx, handle, "Last-Modified", ctx->networking.user_data);
+            if (lastModified)
+                props.lastModified = lastModified;
+
+            const char *etag = ctx->networking.get_header_value(
+                ctx, handle, "ETag", ctx->networking.user_data);
+            if (etag)
+                props.etag = etag;
+
+            return true;
+        }
+    }
+    return false;
+}
+
+// ---------------------------------------------------------------------------
+
 std::unique_ptr<File> NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) {
     FileProperties props;
     if (gNetworkChunkCache.get(ctx, filename, 0, props)) {
         return std::unique_ptr<File>(new NetworkFile(
             ctx, filename, nullptr,
-            std::numeric_limits<unsigned long long>::max(), props.size));
+            std::numeric_limits<unsigned long long>::max(), props));
     } else {
         std::vector<unsigned char> buffer(DOWNLOAD_CHUNK_SIZE);
         size_t size_read = 0;
@@ -1387,40 +1427,18 @@ std::unique_ptr<File> NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) {
                    errorBuffer.c_str());
         }
 
-        unsigned long long filesize = 0;
+        bool ok = false;
         if (handle) {
-            const char *contentRange = ctx->networking.get_header_value(
-                ctx, handle, "Content-Range", ctx->networking.user_data);
-            if (contentRange) {
-                const char *slash = strchr(contentRange, '/');
-                if (slash) {
-                    filesize = std::stoull(slash + 1);
-
-                    props.size = filesize;
-
-                    const char *lastModified = ctx->networking.get_header_value(
-                        ctx, handle, "Last-Modified",
-                        ctx->networking.user_data);
-                    if (lastModified)
-                        props.lastModified = lastModified;
-
-                    const char *etag = ctx->networking.get_header_value(
-                        ctx, handle, "ETag", ctx->networking.user_data);
-                    if (etag)
-                        props.etag = etag;
-
-                    gNetworkFileProperties.insert(ctx, filename, props);
-                }
-            }
-            if (filesize != 0) {
+            if (get_props_from_headers(ctx, handle, props)) {
+                ok = true;
+                gNetworkFileProperties.insert(ctx, filename, props);
                 gNetworkChunkCache.insert(ctx, filename, 0, std::move(buffer));
             }
         }
 
         return std::unique_ptr<File>(
-            handle != nullptr && filesize != 0
-                ? new NetworkFile(ctx, filename, handle, size_read, filesize)
-                : nullptr);
+            ok ? new NetworkFile(ctx, filename, handle, size_read, props)
+               : nullptr);
     }
 }
 
@@ -1505,6 +1523,20 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) {
                 }
                 return 0;
             }
+
+            if (!m_hasChanged) {
+                FileProperties props;
+                if (get_props_from_headers(m_ctx, m_handle, props)) {
+                    if (props.size != m_props.size ||
+                        props.lastModified != m_props.lastModified ||
+                        props.etag != m_props.etag) {
+                        gNetworkFileProperties.insert(m_ctx, m_url, props);
+                        gNetworkChunkCache.clearMemoryCache();
+                        m_hasChanged = true;
+                    }
+                }
+            }
+
             region.resize(nRead);
             m_lastDownloadedOffset = offsetToDownload + nRead;
 
@@ -1547,7 +1579,7 @@ bool NetworkFile::seek(unsigned long long offset, int whence) {
     } else {
         if (offset != 0)
             return false;
-        m_pos = m_filesize;
+        m_pos = m_props.size;
     }
     return true;
 }
@@ -1871,6 +1903,7 @@ static size_t pj_curl_read_range(PJ_CONTEXT *ctx,
     auto hCurlHandle = handle->m_handle;
 
     double oldDelay = MIN_RETRY_DELAY_MS;
+    std::string headers;
     std::string body;
 
     char szBuffer[128];
@@ -1880,6 +1913,12 @@ static size_t pj_curl_read_range(PJ_CONTEXT *ctx,
     while (true) {
         curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer);
 
+        headers.clear();
+        headers.reserve(16 * 1024);
+        curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &headers);
+        curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION,
+                         pj_curl_write_func);
+
         body.clear();
         body.reserve(size_to_read);
         curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body);
@@ -1930,6 +1969,8 @@ static size_t pj_curl_read_range(PJ_CONTEXT *ctx,
     if (!body.empty()) {
         memcpy(buffer, body.data(), std::min(size_to_read, body.size()));
     }
+    handle->m_headers = std::move(headers);
+
     return std::min(size_to_read, body.size());
 }
 
diff --git a/src/filemanager.hpp b/src/filemanager.hpp
index 993048a7..11fb6356 100644
--- a/src/filemanager.hpp
+++ b/src/filemanager.hpp
@@ -69,6 +69,7 @@ class File {
     virtual bool seek(unsigned long long offset, int whence = SEEK_SET) = 0;
     virtual unsigned long long tell() = 0;
     virtual void reassign_context(PJ_CONTEXT *ctx) = 0;
+    virtual bool hasChanged() const = 0;
 
     const std::string &name() const { return name_; }
 };
diff --git a/src/grids.cpp b/src/grids.cpp
index 66854188..0904a3c2 100644
--- a/src/grids.cpp
+++ b/src/grids.cpp
@@ -140,6 +140,7 @@ class NullVerticalShiftGrid : public VerticalShiftGrid {
     bool valueAt(int, int, float &out) const override;
     bool isNodata(float, double) const override { return false; }
     void reassign_context(PJ_CONTEXT *) override {}
+    bool hasChanged() const override { return false; }
 };
 
 // ---------------------------------------------------------------------------
@@ -177,6 +178,8 @@ class GTXVerticalShiftGrid : public VerticalShiftGrid {
         m_ctx = ctx;
         m_fp->reassign_context(ctx);
     }
+
+    bool hasChanged() const override { return m_fp->hasChanged(); }
 };
 
 // ---------------------------------------------------------------------------
@@ -373,6 +376,7 @@ class GTiffGrid : public Grid {
     PJ_CONTEXT *m_ctx;   // owned by the belonging GTiffDataset
     TIFF *m_hTIFF;       // owned by the belonging GTiffDataset
     BlockCache &m_cache; // owned by the belonging GTiffDataset
+    File *m_fp;          // owned by the belonging GTiffDataset
     uint32 m_ifdIdx;
     TIFFDataType m_dt;
     uint16 m_samplesPerPixel;
@@ -402,9 +406,9 @@ class GTiffGrid : public Grid {
                     uint32 offsetInBlock, uint16 sample) const;
 
   public:
-    GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, uint32 ifdIdx,
-              const std::string &nameIn, int widthIn, int heightIn,
-              const ExtentAndRes &extentIn, TIFFDataType dtIn,
+    GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, File *fp,
+              uint32 ifdIdx, const std::string &nameIn, int widthIn,
+              int heightIn, const ExtentAndRes &extentIn, TIFFDataType dtIn,
               uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn);
 
     ~GTiffGrid() override;
@@ -420,17 +424,19 @@ class GTiffGrid : public Grid {
     uint32 subfileType() const { return m_subfileType; }
 
     void reassign_context(PJ_CONTEXT *ctx) { m_ctx = ctx; }
+
+    bool hasChanged() const override { return m_fp->hasChanged(); }
 };
 
 // ---------------------------------------------------------------------------
 
-GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache,
+GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, File *fp,
                      uint32 ifdIdx, const std::string &nameIn, int widthIn,
                      int heightIn, const ExtentAndRes &extentIn,
                      TIFFDataType dtIn, uint16 samplesPerPixelIn,
                      uint16 planarConfig, bool bottomUpIn)
     : Grid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF),
-      m_cache(cache), m_ifdIdx(ifdIdx), m_dt(dtIn),
+      m_cache(cache), m_fp(fp), m_ifdIdx(ifdIdx), m_dt(dtIn),
       m_samplesPerPixel(samplesPerPixelIn), m_planarConfig(planarConfig),
       m_bottomUp(bottomUpIn), m_dirOffset(TIFFCurrentDirOffset(hTIFF)),
       m_tiled(TIFFIsTiled(hTIFF) != 0) {
@@ -1048,8 +1054,8 @@ std::unique_ptr<GTiffGrid> GTiffDataset::nextGrid() {
     }
 
     auto ret = std::unique_ptr<GTiffGrid>(new GTiffGrid(
-        m_ctx, m_hTIFF, m_cache, m_ifdIdx, m_filename, width, height, extent,
-        dt, samplesPerPixel, planarConfig, vRes < 0));
+        m_ctx, m_hTIFF, m_cache, m_fp.get(), m_ifdIdx, m_filename, width,
+        height, extent, dt, samplesPerPixel, planarConfig, vRes < 0));
     m_ifdIdx++;
     m_hasNextGrid = TIFFReadDirectory(m_hTIFF) != 0;
     m_nextDirOffset = TIFFCurrentDirOffset(m_hTIFF);
@@ -1058,10 +1064,12 @@ std::unique_ptr<GTiffGrid> GTiffDataset::nextGrid() {
 
 // ---------------------------------------------------------------------------
 
-class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset {
+class GTiffVGridShiftSet : public VerticalShiftGridSet {
+
+    std::unique_ptr<GTiffDataset> m_GTiffDataset;
 
     GTiffVGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp)
-        : GTiffDataset(ctx, std::move(fp)) {}
+        : m_GTiffDataset(new GTiffDataset(ctx, std::move(fp))) {}
 
   public:
     ~GTiffVGridShiftSet() override;
@@ -1072,7 +1080,26 @@ class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset {
 
     void reassign_context(PJ_CONTEXT *ctx) override {
         VerticalShiftGridSet::reassign_context(ctx);
-        GTiffDataset::reassign_context(ctx);
+        if (m_GTiffDataset) {
+            m_GTiffDataset->reassign_context(ctx);
+        }
+    }
+
+    bool reopen(PJ_CONTEXT *ctx) override {
+        pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it",
+               m_name.c_str());
+        m_grids.clear();
+        m_GTiffDataset.reset();
+        auto fp = FileManager::open_resource_file(ctx, m_name.c_str());
+        if (!fp) {
+            return false;
+        }
+        auto newGS = open(ctx, std::move(fp), m_name);
+        if (newGS) {
+            m_grids = std::move(newGS->m_grids);
+            m_GTiffDataset = std::move(newGS->m_GTiffDataset);
+        }
+        return !m_grids.empty();
     }
 };
 
@@ -1172,6 +1199,8 @@ class GTiffVGrid : public VerticalShiftGrid {
     void reassign_context(PJ_CONTEXT *ctx) override {
         m_grid->reassign_context(ctx);
     }
+
+    bool hasChanged() const override { return m_grid->hasChanged(); }
 };
 
 // ---------------------------------------------------------------------------
@@ -1221,14 +1250,14 @@ GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp,
         new GTiffVGridShiftSet(ctx, std::move(fp)));
     set->m_name = filename;
     set->m_format = "gtiff";
-    if (!set->openTIFF(filename)) {
+    if (!set->m_GTiffDataset->openTIFF(filename)) {
         return nullptr;
     }
     uint16 idxSample = 0;
 
     std::map<std::string, GTiffVGrid *> mapGrids;
     for (int ifd = 0;; ++ifd) {
-        auto grid = set->nextGrid();
+        auto grid = set->m_GTiffDataset->nextGrid();
         if (!grid) {
             if (ifd == 0) {
                 return nullptr;
@@ -1364,6 +1393,19 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) {
 
 // ---------------------------------------------------------------------------
 
+bool VerticalShiftGridSet::reopen(PJ_CONTEXT *ctx) {
+    pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it",
+           m_name.c_str());
+    auto newGS = open(ctx, m_name);
+    m_grids.clear();
+    if (newGS) {
+        m_grids = std::move(newGS->m_grids);
+    }
+    return !m_grids.empty();
+}
+
+// ---------------------------------------------------------------------------
+
 const VerticalShiftGrid *VerticalShiftGrid::gridAt(double lon,
                                                    double lat) const {
     for (const auto &child : m_children) {
@@ -1435,6 +1477,8 @@ class NullHorizontalShiftGrid : public HorizontalShiftGrid {
                  float &latShift) const override;
 
     void reassign_context(PJ_CONTEXT *) override {}
+
+    bool hasChanged() const override { return false; }
 };
 
 // ---------------------------------------------------------------------------
@@ -1482,6 +1526,8 @@ class NTv1Grid : public HorizontalShiftGrid {
         m_ctx = ctx;
         m_fp->reassign_context(ctx);
     }
+
+    bool hasChanged() const override { return m_fp->hasChanged(); }
 };
 
 // ---------------------------------------------------------------------------
@@ -1603,6 +1649,8 @@ class CTable2Grid : public HorizontalShiftGrid {
         m_ctx = ctx;
         m_fp->reassign_context(ctx);
     }
+
+    bool hasChanged() const override { return m_fp->hasChanged(); }
 };
 
 // ---------------------------------------------------------------------------
@@ -1737,6 +1785,8 @@ class NTv2Grid : public HorizontalShiftGrid {
         m_ctx = ctx;
         m_fp->reassign_context(ctx);
     }
+
+    bool hasChanged() const override { return m_fp->hasChanged(); }
 };
 
 // ---------------------------------------------------------------------------
@@ -1906,10 +1956,12 @@ std::unique_ptr<NTv2GridSet> NTv2GridSet::open(PJ_CONTEXT *ctx,
 
 // ---------------------------------------------------------------------------
 
-class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset {
+class GTiffHGridShiftSet : public HorizontalShiftGridSet {
+
+    std::unique_ptr<GTiffDataset> m_GTiffDataset;
 
     GTiffHGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp)
-        : GTiffDataset(ctx, std::move(fp)) {}
+        : m_GTiffDataset(new GTiffDataset(ctx, std::move(fp))) {}
 
   public:
     ~GTiffHGridShiftSet() override;
@@ -1920,7 +1972,26 @@ class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset {
 
     void reassign_context(PJ_CONTEXT *ctx) override {
         HorizontalShiftGridSet::reassign_context(ctx);
-        GTiffDataset::reassign_context(ctx);
+        if (m_GTiffDataset) {
+            m_GTiffDataset->reassign_context(ctx);
+        }
+    }
+
+    bool reopen(PJ_CONTEXT *ctx) override {
+        pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it",
+               m_name.c_str());
+        m_grids.clear();
+        m_GTiffDataset.reset();
+        auto fp = FileManager::open_resource_file(ctx, m_name.c_str());
+        if (!fp) {
+            return false;
+        }
+        auto newGS = open(ctx, std::move(fp), m_name);
+        if (newGS) {
+            m_grids = std::move(newGS->m_grids);
+            m_GTiffDataset = std::move(newGS->m_GTiffDataset);
+        }
+        return !m_grids.empty();
     }
 };
 
@@ -1954,6 +2025,8 @@ class GTiffHGrid : public HorizontalShiftGrid {
     void reassign_context(PJ_CONTEXT *ctx) override {
         m_grid->reassign_context(ctx);
     }
+
+    bool hasChanged() const override { return m_grid->hasChanged(); }
 };
 
 // ---------------------------------------------------------------------------
@@ -2024,7 +2097,7 @@ GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp,
         new GTiffHGridShiftSet(ctx, std::move(fp)));
     set->m_name = filename;
     set->m_format = "gtiff";
-    if (!set->openTIFF(filename)) {
+    if (!set->m_GTiffDataset->openTIFF(filename)) {
         return nullptr;
     }
 
@@ -2037,7 +2110,7 @@ GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp,
 
     std::map<std::string, GTiffHGrid *> mapGrids;
     for (int ifd = 0;; ++ifd) {
-        auto grid = set->nextGrid();
+        auto grid = set->m_GTiffDataset->nextGrid();
         if (!grid) {
             if (ifd == 0) {
                 return nullptr;
@@ -2269,6 +2342,19 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) {
 
 // ---------------------------------------------------------------------------
 
+bool HorizontalShiftGridSet::reopen(PJ_CONTEXT *ctx) {
+    pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it",
+           m_name.c_str());
+    auto newGS = open(ctx, m_name);
+    m_grids.clear();
+    if (newGS) {
+        m_grids = std::move(newGS->m_grids);
+    }
+    return !m_grids.empty();
+}
+
+// ---------------------------------------------------------------------------
+
 const HorizontalShiftGrid *HorizontalShiftGrid::gridAt(double lon,
                                                        double lat) const {
     for (const auto &child : m_children) {
@@ -2317,11 +2403,12 @@ void HorizontalShiftGridSet::reassign_context(PJ_CONTEXT *ctx) {
 #ifdef TIFF_ENABLED
 // ---------------------------------------------------------------------------
 
-class GTiffGenericGridShiftSet : public GenericShiftGridSet,
-                                 public GTiffDataset {
+class GTiffGenericGridShiftSet : public GenericShiftGridSet {
+
+    std::unique_ptr<GTiffDataset> m_GTiffDataset;
 
     GTiffGenericGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp)
-        : GTiffDataset(ctx, std::move(fp)) {}
+        : m_GTiffDataset(new GTiffDataset(ctx, std::move(fp))) {}
 
   public:
     ~GTiffGenericGridShiftSet() override;
@@ -2332,7 +2419,26 @@ class GTiffGenericGridShiftSet : public GenericShiftGridSet,
 
     void reassign_context(PJ_CONTEXT *ctx) override {
         GenericShiftGridSet::reassign_context(ctx);
-        GTiffDataset::reassign_context(ctx);
+        if (m_GTiffDataset) {
+            m_GTiffDataset->reassign_context(ctx);
+        }
+    }
+
+    bool reopen(PJ_CONTEXT *ctx) override {
+        pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it",
+               m_name.c_str());
+        m_grids.clear();
+        m_GTiffDataset.reset();
+        auto fp = FileManager::open_resource_file(ctx, m_name.c_str());
+        if (!fp) {
+            return false;
+        }
+        auto newGS = open(ctx, std::move(fp), m_name);
+        if (newGS) {
+            m_grids = std::move(newGS->m_grids);
+            m_GTiffDataset = std::move(newGS->m_GTiffDataset);
+        }
+        return !m_grids.empty();
     }
 };
 
@@ -2375,6 +2481,8 @@ class GTiffGenericGrid : public GenericShiftGrid {
     void reassign_context(PJ_CONTEXT *ctx) override {
         m_grid->reassign_context(ctx);
     }
+
+    bool hasChanged() const override { return m_grid->hasChanged(); }
 };
 
 // ---------------------------------------------------------------------------
@@ -2446,6 +2554,8 @@ class NullGenericShiftGrid : public GenericShiftGrid {
     }
 
     void reassign_context(PJ_CONTEXT *) override {}
+
+    bool hasChanged() const override { return false; }
 };
 
 // ---------------------------------------------------------------------------
@@ -2466,13 +2576,13 @@ GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp,
         new GTiffGenericGridShiftSet(ctx, std::move(fp)));
     set->m_name = filename;
     set->m_format = "gtiff";
-    if (!set->openTIFF(filename)) {
+    if (!set->m_GTiffDataset->openTIFF(filename)) {
         return nullptr;
     }
 
     std::map<std::string, GTiffGenericGrid *> mapGrids;
     for (int ifd = 0;; ++ifd) {
-        auto grid = set->nextGrid();
+        auto grid = set->m_GTiffDataset->nextGrid();
         if (!grid) {
             if (ifd == 0) {
                 return nullptr;
@@ -2574,6 +2684,19 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) {
 
 // ---------------------------------------------------------------------------
 
+bool GenericShiftGridSet::reopen(PJ_CONTEXT *ctx) {
+    pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it",
+           m_name.c_str());
+    auto newGS = open(ctx, m_name);
+    m_grids.clear();
+    if (newGS) {
+        m_grids = std::move(newGS->m_grids);
+    }
+    return !m_grids.empty();
+}
+
+// ---------------------------------------------------------------------------
+
 const GenericShiftGrid *GenericShiftGrid::gridAt(double lon, double lat) const {
     for (const auto &child : m_children) {
         const auto &extentChild = child->extentAndRes();
@@ -2646,12 +2769,15 @@ ListOfGenericGrids pj_generic_grid_init(PJ *P, const char *gridkey) {
 
 // ---------------------------------------------------------------------------
 
-static const HorizontalShiftGrid *findGrid(const ListOfHGrids &grids,
-                                           PJ_LP input) {
+static const HorizontalShiftGrid *
+findGrid(const ListOfHGrids &grids, const PJ_LP& input,
+         HorizontalShiftGridSet *&gridSetOut) {
     for (const auto &gridset : grids) {
         auto grid = gridset->gridAt(input.lam, input.phi);
-        if (grid)
+        if (grid) {
+            gridSetOut = gridset.get();
             return grid;
+        }
     }
     return nullptr;
 }
@@ -2789,11 +2915,14 @@ static PJ_LP pj_hgrid_interpolate(PJ_LP t, const HorizontalShiftGrid *grid,
 static PJ_LP pj_hgrid_apply_internal(projCtx ctx, PJ_LP in,
                                      PJ_DIRECTION direction,
                                      const HorizontalShiftGrid *grid,
-                                     const ListOfHGrids &grids) {
+                                     HorizontalShiftGridSet *gridset,
+                                     const ListOfHGrids &grids,
+                                     bool &shouldRetry) {
     PJ_LP t, tb, del, dif;
     int i = MAX_ITERATIONS;
     const double toltol = TOL * TOL;
 
+    shouldRetry = false;
     if (in.lam == HUGE_VAL)
         return in;
 
@@ -2806,6 +2935,10 @@ static PJ_LP pj_hgrid_apply_internal(projCtx ctx, PJ_LP in,
     tb.lam = adjlon(tb.lam - M_PI) + M_PI;
 
     t = pj_hgrid_interpolate(tb, grid, true);
+    if (grid->hasChanged()) {
+        shouldRetry = gridset->reopen(ctx);
+        return t;
+    }
     if (t.lam == HUGE_VAL)
         return t;
 
@@ -2820,6 +2953,10 @@ static PJ_LP pj_hgrid_apply_internal(projCtx ctx, PJ_LP in,
 
     do {
         del = pj_hgrid_interpolate(t, grid, true);
+        if (grid->hasChanged()) {
+            shouldRetry = gridset->reopen(ctx);
+            return t;
+        }
 
         /* We can possibly go outside of the initial guessed grid, so try */
         /* to fetch a new grid into which iterate... */
@@ -2827,7 +2964,7 @@ static PJ_LP pj_hgrid_apply_internal(projCtx ctx, PJ_LP in,
             PJ_LP lp;
             lp.lam = t.lam + extent->westLon;
             lp.phi = t.phi + extent->southLat;
-            auto newGrid = findGrid(grids, lp);
+            auto newGrid = findGrid(grids, lp, gridset);
             if (newGrid == nullptr || newGrid == grid || newGrid->isNullGrid())
                 break;
             pj_log(ctx, PJ_LOG_DEBUG_MINOR, "Switching from grid %s to grid %s",
@@ -2882,16 +3019,24 @@ PJ_LP pj_hgrid_apply(PJ_CONTEXT *ctx, const ListOfHGrids &grids, PJ_LP lp,
     out.lam = HUGE_VAL;
     out.phi = HUGE_VAL;
 
-    const auto grid = findGrid(grids, lp);
-    if (!grid) {
-        pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID);
-        return out;
-    }
-    if (grid->isNullGrid()) {
-        return lp;
-    }
+    while (true) {
+        HorizontalShiftGridSet *gridset = nullptr;
+        const auto grid = findGrid(grids, lp, gridset);
+        if (!grid) {
+            pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID);
+            return out;
+        }
+        if (grid->isNullGrid()) {
+            return lp;
+        }
 
-    out = pj_hgrid_apply_internal(ctx, lp, direction, grid, grids);
+        bool shouldRetry = false;
+        out = pj_hgrid_apply_internal(ctx, lp, direction, grid, gridset, grids,
+                                      shouldRetry);
+        if (!shouldRetry) {
+            break;
+        }
+    }
 
     if (out.lam == HUGE_VAL || out.phi == HUGE_VAL)
         pj_ctx_set_errno(ctx, PJD_ERR_GRID_AREA);
@@ -2907,7 +3052,8 @@ PJ_LP pj_hgrid_apply(PJ_CONTEXT *ctx, const ListOfHGrids &grids, PJ_LP lp,
 PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp) {
     PJ_LP out = proj_coord_error().lp;
 
-    const auto grid = findGrid(grids, lp);
+    HorizontalShiftGridSet *gridset = nullptr;
+    const auto grid = findGrid(grids, lp, gridset);
     if (!grid) {
         pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA);
         return out;
@@ -2921,6 +3067,13 @@ PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp) {
     lp.lam = adjlon(lp.lam - M_PI) + M_PI;
 
     out = pj_hgrid_interpolate(lp, grid, false);
+    if (grid->hasChanged()) {
+        if (gridset->reopen(P->ctx)) {
+            return pj_hgrid_value(P, grids, lp);
+        }
+        out.lam = HUGE_VAL;
+        out.phi = HUGE_VAL;
+    }
 
     if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) {
         pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA);
@@ -2931,8 +3084,8 @@ PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp) {
 
 // ---------------------------------------------------------------------------
 
-static double read_vgrid_value(const ListOfVGrids &grids, PJ_LP input,
-                               double vmultiplier) {
+static double read_vgrid_value(PJ_CONTEXT *ctx, const ListOfVGrids &grids,
+                               const PJ_LP& input, const double vmultiplier) {
 
     /* do not deal with NaN coordinates */
     /* cppcheck-suppress duplicateExpression */
@@ -2940,11 +3093,14 @@ static double read_vgrid_value(const ListOfVGrids &grids, PJ_LP input,
         return HUGE_VAL;
     }
 
+    VerticalShiftGridSet *curGridset = nullptr;
     const VerticalShiftGrid *grid = nullptr;
     for (const auto &gridset : grids) {
         grid = gridset->gridAt(input.lam, input.phi);
-        if (grid)
+        if (grid) {
+            curGridset = gridset.get();
             break;
+        }
     }
     if (!grid) {
         return HUGE_VAL;
@@ -2985,10 +3141,18 @@ static double read_vgrid_value(const ListOfVGrids &grids, PJ_LP input,
     float value_b = 0;
     float value_c = 0;
     float value_d = 0;
-    if (!grid->valueAt(grid_ix, grid_iy, value_a) ||
-        !grid->valueAt(grid_ix2, grid_iy, value_b) ||
-        !grid->valueAt(grid_ix, grid_iy2, value_c) ||
-        !grid->valueAt(grid_ix2, grid_iy2, value_d)) {
+    bool error = (!grid->valueAt(grid_ix, grid_iy, value_a) ||
+                  !grid->valueAt(grid_ix2, grid_iy, value_b) ||
+                  !grid->valueAt(grid_ix, grid_iy2, value_c) ||
+                  !grid->valueAt(grid_ix2, grid_iy2, value_d));
+    if (grid->hasChanged()) {
+        if (curGridset->reopen(ctx)) {
+            return read_vgrid_value(ctx, grids, input, vmultiplier);
+        }
+        error = true;
+    }
+
+    if (error) {
         return HUGE_VAL;
     }
 
@@ -3086,7 +3250,7 @@ double pj_vgrid_value(PJ *P, const ListOfVGrids &grids, PJ_LP lp,
 
     double value;
 
-    value = read_vgrid_value(grids, lp, vmultiplier);
+    value = read_vgrid_value(P->ctx, grids, lp, vmultiplier);
     proj_log_trace(P, "proj_vgrid_value: (%f, %f) = %f", lp.lam * RAD_TO_DEG,
                    lp.phi * RAD_TO_DEG, value);
 
diff --git a/src/grids.hpp b/src/grids.hpp
index fbe4e7f8..ef365f06 100644
--- a/src/grids.hpp
+++ b/src/grids.hpp
@@ -70,6 +70,7 @@ class Grid {
     const std::string &name() const { return m_name; }
 
     virtual bool isNullGrid() const { return false; }
+    virtual bool hasChanged() const = 0;
 };
 
 // ---------------------------------------------------------------------------
@@ -116,6 +117,7 @@ class VerticalShiftGridSet {
     const VerticalShiftGrid *gridAt(double lon, double lat) const;
 
     virtual void reassign_context(PJ_CONTEXT *ctx);
+    virtual bool reopen(PJ_CONTEXT *ctx);
 };
 
 // ---------------------------------------------------------------------------
@@ -162,6 +164,7 @@ class HorizontalShiftGridSet {
     const HorizontalShiftGrid *gridAt(double lon, double lat) const;
 
     virtual void reassign_context(PJ_CONTEXT *ctx);
+    virtual bool reopen(PJ_CONTEXT *ctx);
 };
 
 // ---------------------------------------------------------------------------
@@ -217,6 +220,7 @@ class GenericShiftGridSet {
     const GenericShiftGrid *gridAt(double lon, double lat) const;
 
     virtual void reassign_context(PJ_CONTEXT *ctx);
+    virtual bool reopen(PJ_CONTEXT *ctx);
 };
 
 // ---------------------------------------------------------------------------
diff --git a/src/proj.h b/src/proj.h
index 33057617..5c9e81bb 100644
--- a/src/proj.h
+++ b/src/proj.h
@@ -401,6 +401,9 @@ typedef const char* (*proj_network_get_header_value_cbk_type)(
  *
  * Read size_to_read bytes from handle, starting at offset, into
  * buffer.
+ * During this read, the implementation should make sure to store the HTTP
+ * headers from the server response to be able to respond to
+ * proj_network_get_header_value_cbk_type callback.
  *
  * error_string_max_size should be the maximum size that can be written into
  * the out_error_string buffer (including terminating nul character).
diff --git a/src/sqlite3.cpp b/src/sqlite3.cpp
index 0c89c0b9..90e73c2a 100644
--- a/src/sqlite3.cpp
+++ b/src/sqlite3.cpp
@@ -25,8 +25,17 @@
  * DEALINGS IN THE SOFTWARE.
  *****************************************************************************/
 
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Weffc++"
+#endif
+
 #include "sqlite3.hpp"
 
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+
 #include <cstdlib>
 #include <cstring>
 #include <sstream> // std::ostringstream
@@ -182,4 +191,4 @@ SQLiteStatement::SQLiteStatement(sqlite3_stmt *hStmtIn) : hStmt(hStmtIn) {}
 
 // ---------------------------------------------------------------------------
 
-NS_PROJ_END
\ No newline at end of file
+NS_PROJ_END
diff --git a/src/transformations/deformation.cpp b/src/transformations/deformation.cpp
index 993647fc..778c1a2e 100644
--- a/src/transformations/deformation.cpp
+++ b/src/transformations/deformation.cpp
@@ -81,13 +81,16 @@ struct deformationData {
 // ---------------------------------------------------------------------------
 
 static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids,
-                                        const PJ_LP& input)
+                                        const PJ_LP& input,
+                                        GenericShiftGridSet *&gridSetOut)
 {
     for( const auto& gridset: grids )
     {
         auto grid = gridset->gridAt(input.lam, input.phi);
-        if( grid )
+        if( grid ) {
+            gridSetOut = gridset.get();
             return grid;
+        }
     }
     return nullptr;
 }
@@ -101,7 +104,8 @@ static bool get_grid_values(PJ* P,
                             double& vy,
                             double& vz)
 {
-    auto grid = findGrid(Q->grids, lp);
+    GenericShiftGridSet* gridset = nullptr;
+    auto grid = findGrid(Q->grids, lp, gridset);
     if( !grid ) {
         return false;
     }
@@ -145,30 +149,28 @@ static bool get_grid_values(PJ* P,
     int iy2 = std::min(iy + 1, grid->height() - 1);
 
     float dx1, dy1, dz1;
-    if( !grid->valueAt(ix, iy, sampleE, dx1) ||
-        !grid->valueAt(ix, iy, sampleN, dy1) ||
-        !grid->valueAt(ix, iy, sampleU, dz1) ) {
-        return false;
-    }
-
     float dx2, dy2, dz2;
-    if( !grid->valueAt(ix2, iy, sampleE, dx2) ||
-        !grid->valueAt(ix2, iy, sampleN, dy2) ||
-        !grid->valueAt(ix2, iy, sampleU, dz2) ) {
-        return false;
-    }
-
     float dx3, dy3, dz3;
-    if( !grid->valueAt(ix, iy2, sampleE, dx3) ||
-        !grid->valueAt(ix, iy2, sampleN, dy3) ||
-        !grid->valueAt(ix, iy2, sampleU, dz3) ) {
-        return false;
-    }
-
     float dx4, dy4, dz4;
-    if( !grid->valueAt(ix2, iy2, sampleE, dx4) ||
+    bool error =( !grid->valueAt(ix, iy, sampleE, dx1) ||
+        !grid->valueAt(ix, iy, sampleN, dy1) ||
+        !grid->valueAt(ix, iy, sampleU, dz1) ||
+        !grid->valueAt(ix2, iy, sampleE, dx2) ||
+        !grid->valueAt(ix2, iy, sampleN, dy2) ||
+        !grid->valueAt(ix2, iy, sampleU, dz2) ||
+        !grid->valueAt(ix, iy2, sampleE, dx3) ||
+        !grid->valueAt(ix, iy2, sampleN, dy3) ||
+        !grid->valueAt(ix, iy2, sampleU, dz3) ||
+        !grid->valueAt(ix2, iy2, sampleE, dx4) ||
         !grid->valueAt(ix2, iy2, sampleN, dy4) ||
-        !grid->valueAt(ix2, iy2, sampleU, dz4) ) {
+        !grid->valueAt(ix2, iy2, sampleU, dz4) );
+    if( grid->hasChanged() ) {
+        if( gridset->reopen(P->ctx) ) {
+            return get_grid_values( P, Q, lp, vx, vy, vz);
+        }
+        error = true;
+    }
+    if( error ) {
         return false;
     }
 
diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp
index 3ec3863c..4a85fb4c 100644
--- a/src/transformations/xyzgridshift.cpp
+++ b/src/transformations/xyzgridshift.cpp
@@ -55,13 +55,16 @@ struct xyzgridshiftData {
 // ---------------------------------------------------------------------------
 
 static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids,
-                                        const PJ_LP& input)
+                                        const PJ_LP& input,
+                                        GenericShiftGridSet *&gridSetOut)
 {
     for( const auto& gridset: grids )
     {
         auto grid = gridset->gridAt(input.lam, input.phi);
-        if( grid )
+        if( grid ) {
+            gridSetOut = gridset.get();
             return grid;
+        }
     }
     return nullptr;
 }
@@ -83,7 +86,8 @@ static bool get_grid_values(PJ* P,
         }
     }
 
-    auto grid = findGrid(Q->grids, lp);
+    GenericShiftGridSet* gridset = nullptr;
+    auto grid = findGrid(Q->grids, lp, gridset);
     if( !grid ) {
         return false;
     }
@@ -127,30 +131,28 @@ static bool get_grid_values(PJ* P,
     int iy2 = std::min(iy + 1, grid->height() - 1);
 
     float dx1, dy1, dz1;
-    if( !grid->valueAt(ix, iy, sampleX, dx1) ||
-        !grid->valueAt(ix, iy, sampleY, dy1) ||
-        !grid->valueAt(ix, iy, sampleZ, dz1) ) {
-        return false;
-    }
-
     float dx2, dy2, dz2;
-    if( !grid->valueAt(ix2, iy, sampleX, dx2) ||
-        !grid->valueAt(ix2, iy, sampleY, dy2) ||
-        !grid->valueAt(ix2, iy, sampleZ, dz2) ) {
-        return false;
-    }
-
     float dx3, dy3, dz3;
-    if( !grid->valueAt(ix, iy2, sampleX, dx3) ||
-        !grid->valueAt(ix, iy2, sampleY, dy3) ||
-        !grid->valueAt(ix, iy2, sampleZ, dz3) ) {
-        return false;
-    }
-
     float dx4, dy4, dz4;
-    if( !grid->valueAt(ix2, iy2, sampleX, dx4) ||
+    bool error =( !grid->valueAt(ix, iy, sampleX, dx1) ||
+        !grid->valueAt(ix, iy, sampleY, dy1) ||
+        !grid->valueAt(ix, iy, sampleZ, dz1) ||
+        !grid->valueAt(ix2, iy, sampleX, dx2) ||
+        !grid->valueAt(ix2, iy, sampleY, dy2) ||
+        !grid->valueAt(ix2, iy, sampleZ, dz2) ||
+        !grid->valueAt(ix, iy2, sampleX, dx3) ||
+        !grid->valueAt(ix, iy2, sampleY, dy3) ||
+        !grid->valueAt(ix, iy2, sampleZ, dz3) ||
+        !grid->valueAt(ix2, iy2, sampleX, dx4) ||
         !grid->valueAt(ix2, iy2, sampleY, dy4) ||
-        !grid->valueAt(ix2, iy2, sampleZ, dz4) ) {
+        !grid->valueAt(ix2, iy2, sampleZ, dz4) );
+    if( grid->hasChanged() ) {
+        if( gridset->reopen(P->ctx) ) {
+            return get_grid_values( P, Q, lp, dx, dy, dz);
+        }
+        error = true;
+    }
+    if( error ) {
         return false;
     }