Add a SQLite3 local cache of downloaded chunks

11 files changed, 1638 insertions, 156 deletions

diff --git a/src/Makefile.am b/src/Makefile.am
index afe4bcb7..d29eb976 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -217,7 +217,9 @@ libproj_la_SOURCES = \
 	grids.hpp \
 	grids.cpp \
 	filemanager.hpp \
-	filemanager.cpp
+	filemanager.cpp \
+	sqlite3.hpp \
+	sqlite3.cpp
 
 
 # The sed hack is to please MSVC
diff --git a/src/filemanager.cpp b/src/filemanager.cpp
index 625dca03..c0b80ac8 100644
--- a/src/filemanager.cpp
+++ b/src/filemanager.cpp
@@ -31,14 +31,18 @@
 #define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS
 
 #include <algorithm>
+#include <codecvt>
 #include <functional>
 #include <limits>
+#include <locale>
+#include <string>
 
 #include "filemanager.hpp"
 #include "proj.h"
 #include "proj/internal/internal.hpp"
 #include "proj/internal/lru_cache.hpp"
 #include "proj_internal.h"
+#include "sqlite3.hpp"
 
 #ifdef __MINGW32__
 // mingw32-win32 doesn't implement std::mutex
@@ -61,9 +65,16 @@ class MyMutex {
 #include <sqlite3.h> // for sqlite3_snprintf
 #endif
 
-#if defined(__linux)
+#include <sys/stat.h>
+
+#ifdef _WIN32
+#include <shlobj.h>
+#else
+#include <sys/types.h>
 #include <unistd.h>
-#elif defined(_WIN32)
+#endif
+
+#if defined(_WIN32)
 #include <windows.h>
 #elif defined(__MACH__) && defined(__APPLE__)
 #include <mach-o/dyld.h>
@@ -72,6 +83,8 @@ class MyMutex {
 #include <sys/types.h>
 #endif
 
+#include <time.h>
+
 //! @cond Doxygen_Suppress
 
 #define STR_HELPER(x) #x
@@ -83,6 +96,16 @@ NS_PROJ_START
 
 // ---------------------------------------------------------------------------
 
+static void proj_sleep_ms(int ms) {
+#ifdef _WIN32
+    Sleep(ms);
+#else
+    usleep(ms * 1000);
+#endif
+}
+
+// ---------------------------------------------------------------------------
+
 File::File(const std::string &name) : name_(name) {}
 
 // ---------------------------------------------------------------------------
@@ -219,15 +242,29 @@ std::unique_ptr<File> FileLegacyAdapter::open(PJ_CONTEXT *ctx,
 constexpr size_t DOWNLOAD_CHUNK_SIZE = 16 * 1024;
 constexpr int MAX_CHUNKS = 64;
 
+struct FileProperties {
+    unsigned long long size = 0;
+    time_t lastChecked = 0;
+    std::string lastModified{};
+    std::string etag{};
+};
+
 class NetworkChunkCache {
   public:
-    void insert(const std::string &url, unsigned long long chunkIdx,
-                std::vector<unsigned char> &&data);
+    void insert(PJ_CONTEXT *ctx, const std::string &url,
+                unsigned long long chunkIdx, std::vector<unsigned char> &&data);
 
     std::shared_ptr<std::vector<unsigned char>>
-    get(const std::string &url, unsigned long long chunkIdx);
+    get(PJ_CONTEXT *ctx, const std::string &url, unsigned long long chunkIdx);
+
+    std::shared_ptr<std::vector<unsigned char>> get(PJ_CONTEXT *ctx,
+                                                    const std::string &url,
+                                                    unsigned long long chunkIdx,
+                                                    FileProperties &props);
 
-    void clear();
+    void clearMemoryCache();
+
+    static void clearDiskChunkCache(PJ_CONTEXT *ctx);
 
   private:
     struct Key {
@@ -260,37 +297,1033 @@ class NetworkChunkCache {
 
 // ---------------------------------------------------------------------------
 
-void NetworkChunkCache::insert(const std::string &url,
+static NetworkChunkCache gNetworkChunkCache{};
+
+// ---------------------------------------------------------------------------
+
+class NetworkFilePropertiesCache {
+  public:
+    void insert(PJ_CONTEXT *ctx, const std::string &url, FileProperties &props);
+
+    bool tryGet(PJ_CONTEXT *ctx, const std::string &url, FileProperties &props);
+
+    void clearMemoryCache();
+
+  private:
+    lru11::Cache<std::string, FileProperties, MyMutex> cache_{};
+};
+
+// ---------------------------------------------------------------------------
+
+static NetworkFilePropertiesCache gNetworkFileProperties{};
+
+// ---------------------------------------------------------------------------
+
+class DiskChunkCache {
+    PJ_CONTEXT *ctx_ = nullptr;
+    std::string path_{};
+    sqlite3 *hDB_ = nullptr;
+    std::string thisNamePtr_{};
+    std::unique_ptr<SQLite3VFS> vfs_{};
+
+    explicit DiskChunkCache(PJ_CONTEXT *ctx, const std::string &path);
+
+    bool createDBStructure();
+    bool checkConsistency();
+    bool get_links(sqlite3_int64 chunk_id, sqlite3_int64 &link_id,
+                   sqlite3_int64 &prev, sqlite3_int64 &next,
+                   sqlite3_int64 &head, sqlite3_int64 &tail);
+    bool update_links_of_prev_and_next_links(sqlite3_int64 prev,
+                                             sqlite3_int64 next);
+    bool update_linked_chunks(sqlite3_int64 link_id, sqlite3_int64 prev,
+                              sqlite3_int64 next);
+    bool update_linked_chunks_head_tail(sqlite3_int64 head, sqlite3_int64 tail);
+
+    DiskChunkCache(const DiskChunkCache &) = delete;
+    DiskChunkCache &operator=(const DiskChunkCache &) = delete;
+
+  public:
+    static std::unique_ptr<DiskChunkCache> open(PJ_CONTEXT *ctx);
+    ~DiskChunkCache();
+
+    sqlite3 *handle() { return hDB_; }
+    std::unique_ptr<SQLiteStatement> prepare(const char *sql);
+    bool move_to_head(sqlite3_int64 chunk_id);
+    bool move_to_tail(sqlite3_int64 chunk_id);
+    void closeAndUnlink();
+};
+
+// ---------------------------------------------------------------------------
+
+static bool pj_context_get_grid_cache_is_enabled(PJ_CONTEXT *ctx) {
+    pj_load_ini(ctx);
+    return ctx->gridChunkCache.enabled;
+}
+
+// ---------------------------------------------------------------------------
+
+static long long pj_context_get_grid_cache_max_size(PJ_CONTEXT *ctx) {
+    pj_load_ini(ctx);
+    return ctx->gridChunkCache.max_size;
+}
+
+// ---------------------------------------------------------------------------
+
+static int pj_context_get_grid_cache_ttl(PJ_CONTEXT *ctx) {
+    pj_load_ini(ctx);
+    return ctx->gridChunkCache.ttl;
+}
+
+// ---------------------------------------------------------------------------
+
+std::unique_ptr<DiskChunkCache> DiskChunkCache::open(PJ_CONTEXT *ctx) {
+    if (!pj_context_get_grid_cache_is_enabled(ctx)) {
+        return nullptr;
+    }
+    const auto cachePath = pj_context_get_grid_cache_filename(ctx);
+    if (cachePath.empty()) {
+        return nullptr;
+    }
+
+    auto diskCache =
+        std::unique_ptr<DiskChunkCache>(new DiskChunkCache(ctx, cachePath));
+    if (!diskCache->hDB_)
+        diskCache.reset();
+    return diskCache;
+}
+
+// ---------------------------------------------------------------------------
+
+DiskChunkCache::DiskChunkCache(PJ_CONTEXT *ctx, const std::string &path)
+    : ctx_(ctx), path_(path), vfs_(SQLite3VFS::create(true, false, false)) {
+    if (vfs_ == nullptr) {
+        return;
+    }
+    sqlite3_open_v2(path.c_str(), &hDB_,
+                    SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, vfs_->name());
+    if (!hDB_) {
+        return;
+    }
+    for (int i = 0;; i++) {
+        int ret =
+            sqlite3_exec(hDB_, "BEGIN EXCLUSIVE", nullptr, nullptr, nullptr);
+        if (ret == SQLITE_OK) {
+            break;
+        }
+        if (ret != SQLITE_BUSY) {
+            pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_));
+            sqlite3_close(hDB_);
+            hDB_ = nullptr;
+            return;
+        }
+        const char *max_iters = getenv("PROJ_LOCK_MAX_ITERS");
+        if (i >= (max_iters && max_iters[0] ? atoi(max_iters)
+                                            : 30)) { // A bit more than 1 second
+            pj_log(ctx_, PJ_LOG_ERROR, "Cannot take exclusive lock on %s",
+                   path.c_str());
+            sqlite3_close(hDB_);
+            hDB_ = nullptr;
+            return;
+        }
+        pj_log(ctx, PJ_LOG_TRACE, "Lock taken on cache. Waiting a bit...");
+        // Retry every 5 ms for 50 ms, then every 10 ms for 100 ms, then
+        // every 100 ms
+        proj_sleep_ms(i < 10 ? 5 : i < 20 ? 10 : 100);
+    }
+    char **pasResult = nullptr;
+    int nRows = 0;
+    int nCols = 0;
+    sqlite3_get_table(hDB_,
+                      "SELECT 1 FROM sqlite_master WHERE name = 'properties'",
+                      &pasResult, &nRows, &nCols, nullptr);
+    sqlite3_free_table(pasResult);
+    if (nRows == 0) {
+        if (!createDBStructure()) {
+            sqlite3_close(hDB_);
+            hDB_ = nullptr;
+            return;
+        }
+    }
+
+    if (getenv("PROJ_CHECK_CACHE_CONSISTENCY")) {
+        checkConsistency();
+    }
+}
+
+// ---------------------------------------------------------------------------
+
+static const char *cache_db_structure_sql =
+    "CREATE TABLE properties("
+    " url          TEXT PRIMARY KEY NOT NULL,"
+    " lastChecked  TIMESTAMP NOT NULL,"
+    " fileSize     INTEGER NOT NULL,"
+    " lastModified TEXT,"
+    " etag         TEXT"
+    ");"
+    "CREATE TABLE chunk_data("
+    " id        INTEGER PRIMARY KEY AUTOINCREMENT CHECK (id > 0),"
+    " data      BLOB NOT NULL"
+    ");"
+    "CREATE TABLE chunks("
+    " id        INTEGER PRIMARY KEY AUTOINCREMENT CHECK (id > 0),"
+    " url       TEXT NOT NULL,"
+    " offset    INTEGER NOT NULL,"
+    " data_id   INTEGER NOT NULL,"
+    " data_size INTEGER NOT NULL,"
+    " CONSTRAINT fk_chunks_url FOREIGN KEY (url) REFERENCES properties(url),"
+    " CONSTRAINT fk_chunks_data FOREIGN KEY (data_id) REFERENCES chunk_data(id)"
+    ");"
+    "CREATE INDEX idx_chunks ON chunks(url, offset);"
+    "CREATE TABLE linked_chunks("
+    " id        INTEGER PRIMARY KEY AUTOINCREMENT CHECK (id > 0),"
+    " chunk_id  INTEGER NOT NULL,"
+    " prev      INTEGER,"
+    " next      INTEGER,"
+    " CONSTRAINT fk_links_chunkid FOREIGN KEY (chunk_id) REFERENCES chunks(id),"
+    " CONSTRAINT fk_links_prev FOREIGN KEY (prev) REFERENCES linked_chunks(id),"
+    " CONSTRAINT fk_links_next FOREIGN KEY (next) REFERENCES linked_chunks(id)"
+    ");"
+    "CREATE INDEX idx_linked_chunks_chunk_id ON linked_chunks(chunk_id);"
+    "CREATE TABLE linked_chunks_head_tail("
+    "  head       INTEGER,"
+    "  tail       INTEGER,"
+    "  CONSTRAINT lht_head FOREIGN KEY (head) REFERENCES linked_chunks(id),"
+    "  CONSTRAINT lht_tail FOREIGN KEY (tail) REFERENCES linked_chunks(id)"
+    ");"
+    "INSERT INTO linked_chunks_head_tail VALUES (NULL, NULL);";
+
+bool DiskChunkCache::createDBStructure() {
+
+    pj_log(ctx_, PJ_LOG_TRACE, "Creating cache DB structure");
+    if (sqlite3_exec(hDB_, cache_db_structure_sql, nullptr, nullptr, nullptr) !=
+        SQLITE_OK) {
+        pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_));
+        return false;
+    }
+    return true;
+}
+
+// ---------------------------------------------------------------------------
+
+#define INVALIDATED_SQL_LITERAL "'invalidated'"
+
+bool DiskChunkCache::checkConsistency() {
+
+    auto stmt = prepare("SELECT * FROM chunk_data WHERE id NOT IN (SELECT "
+                        "data_id FROM chunks)");
+    if (!stmt) {
+        return false;
+    }
+    if (stmt->execute() != SQLITE_DONE) {
+        fprintf(stderr, "Rows in chunk_data not referenced by chunks.\n");
+        return false;
+    }
+
+    stmt = prepare("SELECT * FROM chunks WHERE id NOT IN (SELECT chunk_id FROM "
+                   "linked_chunks)");
+    if (!stmt) {
+        return false;
+    }
+    if (stmt->execute() != SQLITE_DONE) {
+        fprintf(stderr, "Rows in chunks not referenced by linked_chunks.\n");
+        return false;
+    }
+
+    stmt = prepare("SELECT * FROM chunks WHERE url <> " INVALIDATED_SQL_LITERAL
+                   " AND url "
+                   "NOT IN (SELECT url FROM properties)");
+    if (!stmt) {
+        return false;
+    }
+    if (stmt->execute() != SQLITE_DONE) {
+        fprintf(stderr, "url values in chunks not referenced by properties.\n");
+        return false;
+    }
+
+    stmt = prepare("SELECT head, tail FROM linked_chunks_head_tail");
+    if (!stmt) {
+        return false;
+    }
+    if (stmt->execute() != SQLITE_ROW) {
+        fprintf(stderr, "linked_chunks_head_tail empty.\n");
+        return false;
+    }
+    const auto head = stmt->getInt64();
+    const auto tail = stmt->getInt64();
+    if (stmt->execute() != SQLITE_DONE) {
+        fprintf(stderr, "linked_chunks_head_tail has more than one row.\n");
+        return false;
+    }
+
+    stmt = prepare("SELECT COUNT(*) FROM linked_chunks");
+    if (!stmt) {
+        return false;
+    }
+    if (stmt->execute() != SQLITE_ROW) {
+        fprintf(stderr, "linked_chunks_head_tail empty.\n");
+        return false;
+    }
+    const auto count_linked_chunks = stmt->getInt64();
+
+    if (head) {
+        auto id = head;
+        std::set<sqlite3_int64> visitedIds;
+        stmt = prepare("SELECT next FROM linked_chunks WHERE id = ?");
+        if (!stmt) {
+            return false;
+        }
+        while (true) {
+            visitedIds.insert(id);
+            stmt->reset();
+            stmt->bindInt64(id);
+            if (stmt->execute() != SQLITE_ROW) {
+                fprintf(stderr, "cannot find linked_chunks.id = %d.\n",
+                        static_cast<int>(id));
+                return false;
+            }
+            auto next = stmt->getInt64();
+            if (next == 0) {
+                if (id != tail) {
+                    fprintf(stderr,
+                            "last item when following next is not tail.\n");
+                    return false;
+                }
+                break;
+            }
+            if (visitedIds.find(next) != visitedIds.end()) {
+                fprintf(stderr, "found cycle on linked_chunks.next = %d.\n",
+                        static_cast<int>(next));
+                return false;
+            }
+            id = next;
+        }
+        if (visitedIds.size() != static_cast<size_t>(count_linked_chunks)) {
+            fprintf(stderr,
+                    "ghost items in linked_chunks when following next.\n");
+            return false;
+        }
+    } else if (count_linked_chunks) {
+        fprintf(stderr, "linked_chunks_head_tail.head = NULL but linked_chunks "
+                        "not empty.\n");
+        return false;
+    }
+
+    if (tail) {
+        auto id = tail;
+        std::set<sqlite3_int64> visitedIds;
+        stmt = prepare("SELECT prev FROM linked_chunks WHERE id = ?");
+        if (!stmt) {
+            return false;
+        }
+        while (true) {
+            visitedIds.insert(id);
+            stmt->reset();
+            stmt->bindInt64(id);
+            if (stmt->execute() != SQLITE_ROW) {
+                fprintf(stderr, "cannot find linked_chunks.id = %d.\n",
+                        static_cast<int>(id));
+                return false;
+            }
+            auto prev = stmt->getInt64();
+            if (prev == 0) {
+                if (id != head) {
+                    fprintf(stderr,
+                            "last item when following prev is not head.\n");
+                    return false;
+                }
+                break;
+            }
+            if (visitedIds.find(prev) != visitedIds.end()) {
+                fprintf(stderr, "found cycle on linked_chunks.prev = %d.\n",
+                        static_cast<int>(prev));
+                return false;
+            }
+            id = prev;
+        }
+        if (visitedIds.size() != static_cast<size_t>(count_linked_chunks)) {
+            fprintf(stderr,
+                    "ghost items in linked_chunks when following prev.\n");
+            return false;
+        }
+    } else if (count_linked_chunks) {
+        fprintf(stderr, "linked_chunks_head_tail.tail = NULL but linked_chunks "
+                        "not empty.\n");
+        return false;
+    }
+
+    fprintf(stderr, "check ok\n");
+    return true;
+}
+
+// ---------------------------------------------------------------------------
+
+DiskChunkCache::~DiskChunkCache() {
+    if (hDB_) {
+        sqlite3_exec(hDB_, "COMMIT", nullptr, nullptr, nullptr);
+        sqlite3_close(hDB_);
+    }
+}
+
+// ---------------------------------------------------------------------------
+
+void DiskChunkCache::closeAndUnlink() {
+    if (hDB_) {
+        sqlite3_exec(hDB_, "COMMIT", nullptr, nullptr, nullptr);
+        sqlite3_close(hDB_);
+    }
+    if (vfs_) {
+        vfs_->raw()->xDelete(vfs_->raw(), path_.c_str(), 0);
+    }
+}
+
+// ---------------------------------------------------------------------------
+
+std::unique_ptr<SQLiteStatement> DiskChunkCache::prepare(const char *sql) {
+    sqlite3_stmt *hStmt = nullptr;
+    sqlite3_prepare_v2(hDB_, sql, -1, &hStmt, nullptr);
+    if (!hStmt) {
+        pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_));
+        return nullptr;
+    }
+    return std::unique_ptr<SQLiteStatement>(new SQLiteStatement(hStmt));
+}
+
+// ---------------------------------------------------------------------------
+
+bool DiskChunkCache::get_links(sqlite3_int64 chunk_id, sqlite3_int64 &link_id,
+                               sqlite3_int64 &prev, sqlite3_int64 &next,
+                               sqlite3_int64 &head, sqlite3_int64 &tail) {
+    auto stmt =
+        prepare("SELECT id, prev, next FROM linked_chunks WHERE chunk_id = ?");
+    if (!stmt)
+        return false;
+    stmt->bindInt64(chunk_id);
+    {
+        const auto ret = stmt->execute();
+        if (ret != SQLITE_ROW) {
+            pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_));
+            return false;
+        }
+    }
+    link_id = stmt->getInt64();
+    prev = stmt->getInt64();
+    next = stmt->getInt64();
+
+    stmt = prepare("SELECT head, tail FROM linked_chunks_head_tail");
+    {
+        const auto ret = stmt->execute();
+        if (ret != SQLITE_ROW) {
+            pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_));
+            return false;
+        }
+    }
+    head = stmt->getInt64();
+    tail = stmt->getInt64();
+    return true;
+}
+
+// ---------------------------------------------------------------------------
+
+bool DiskChunkCache::update_links_of_prev_and_next_links(sqlite3_int64 prev,
+                                                         sqlite3_int64 next) {
+    if (prev) {
+        auto stmt = prepare("UPDATE linked_chunks SET next = ? WHERE id = ?");
+        if (!stmt)
+            return false;
+        if (next)
+            stmt->bindInt64(next);
+        else
+            stmt->bindNull();
+        stmt->bindInt64(prev);
+        const auto ret = stmt->execute();
+        if (ret != SQLITE_DONE) {
+            pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_));
+            return false;
+        }
+    }
+
+    if (next) {
+        auto stmt = prepare("UPDATE linked_chunks SET prev = ? WHERE id = ?");
+        if (!stmt)
+            return false;
+        if (prev)
+            stmt->bindInt64(prev);
+        else
+            stmt->bindNull();
+        stmt->bindInt64(next);
+        const auto ret = stmt->execute();
+        if (ret != SQLITE_DONE) {
+            pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_));
+            return false;
+        }
+    }
+    return true;
+}
+
+// ---------------------------------------------------------------------------
+
+bool DiskChunkCache::update_linked_chunks(sqlite3_int64 link_id,
+                                          sqlite3_int64 prev,
+                                          sqlite3_int64 next) {
+    auto stmt =
+        prepare("UPDATE linked_chunks SET prev = ?, next = ? WHERE id = ?");
+    if (!stmt)
+        return false;
+    if (prev)
+        stmt->bindInt64(prev);
+    else
+        stmt->bindNull();
+    if (next)
+        stmt->bindInt64(next);
+    else
+        stmt->bindNull();
+    stmt->bindInt64(link_id);
+    const auto ret = stmt->execute();
+    if (ret != SQLITE_DONE) {
+        pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_));
+        return false;
+    }
+    return true;
+}
+
+// ---------------------------------------------------------------------------
+
+bool DiskChunkCache::update_linked_chunks_head_tail(sqlite3_int64 head,
+                                                    sqlite3_int64 tail) {
+    auto stmt =
+        prepare("UPDATE linked_chunks_head_tail SET head = ?, tail = ?");
+    if (!stmt)
+        return false;
+    if (head)
+        stmt->bindInt64(head);
+    else
+        stmt->bindNull(); // shouldn't happen normally
+    if (tail)
+        stmt->bindInt64(tail);
+    else
+        stmt->bindNull(); // shouldn't happen normally
+    const auto ret = stmt->execute();
+    if (ret != SQLITE_DONE) {
+        pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_));
+        return false;
+    }
+    return true;
+}
+
+// ---------------------------------------------------------------------------
+
+bool DiskChunkCache::move_to_head(sqlite3_int64 chunk_id) {
+
+    sqlite3_int64 link_id = 0;
+    sqlite3_int64 prev = 0;
+    sqlite3_int64 next = 0;
+    sqlite3_int64 head = 0;
+    sqlite3_int64 tail = 0;
+    if (!get_links(chunk_id, link_id, prev, next, head, tail)) {
+        return false;
+    }
+
+    if (link_id == head) {
+        return true;
+    }
+
+    if (!update_links_of_prev_and_next_links(prev, next)) {
+        return false;
+    }
+
+    if (head) {
+        auto stmt = prepare("UPDATE linked_chunks SET prev = ? WHERE id = ?");
+        if (!stmt)
+            return false;
+        stmt->bindInt64(link_id);
+        stmt->bindInt64(head);
+        const auto ret = stmt->execute();
+        if (ret != SQLITE_DONE) {
+            pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_));
+            return false;
+        }
+    }
+
+    return update_linked_chunks(link_id, 0, head) &&
+           update_linked_chunks_head_tail(link_id,
+                                          (link_id == tail) ? prev : tail);
+}
+
+// ---------------------------------------------------------------------------
+
+bool DiskChunkCache::move_to_tail(sqlite3_int64 chunk_id) {
+    sqlite3_int64 link_id = 0;
+    sqlite3_int64 prev = 0;
+    sqlite3_int64 next = 0;
+    sqlite3_int64 head = 0;
+    sqlite3_int64 tail = 0;
+    if (!get_links(chunk_id, link_id, prev, next, head, tail)) {
+        return false;
+    }
+
+    if (link_id == tail) {
+        return true;
+    }
+
+    if (!update_links_of_prev_and_next_links(prev, next)) {
+        return false;
+    }
+
+    if (tail) {
+        auto stmt = prepare("UPDATE linked_chunks SET next = ? WHERE id = ?");
+        if (!stmt)
+            return false;
+        stmt->bindInt64(link_id);
+        stmt->bindInt64(tail);
+        const auto ret = stmt->execute();
+        if (ret != SQLITE_DONE) {
+            pj_log(ctx_, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB_));
+            return false;
+        }
+    }
+
+    return update_linked_chunks(link_id, tail, 0) &&
+           update_linked_chunks_head_tail((link_id == head) ? next : head,
+                                          link_id);
+}
+
+// ---------------------------------------------------------------------------
+
+void NetworkChunkCache::insert(PJ_CONTEXT *ctx, const std::string &url,
                                unsigned long long chunkIdx,
                                std::vector<unsigned char> &&data) {
-    cache_.insert(
-        Key(url, chunkIdx),
-        std::make_shared<std::vector<unsigned char>>(std::move(data)));
+    auto dataPtr(std::make_shared<std::vector<unsigned char>>(std::move(data)));
+    cache_.insert(Key(url, chunkIdx), dataPtr);
+
+    auto diskCache = DiskChunkCache::open(ctx);
+    if (!diskCache)
+        return;
+    auto hDB = diskCache->handle();
+
+    // Always insert DOWNLOAD_CHUNK_SIZE bytes to avoid fragmentation
+    std::vector<unsigned char> blob(*dataPtr);
+    assert(blob.size() <= DOWNLOAD_CHUNK_SIZE);
+    blob.resize(DOWNLOAD_CHUNK_SIZE);
+
+    // Check if there is an existing entry for that URL and offset
+    auto stmt = diskCache->prepare(
+        "SELECT id, data_id FROM chunks WHERE url = ? AND offset = ?");
+    if (!stmt)
+        return;
+    stmt->bindText(url.c_str());
+    stmt->bindInt64(chunkIdx * DOWNLOAD_CHUNK_SIZE);
+
+    const auto mainRet = stmt->execute();
+    if (mainRet == SQLITE_ROW) {
+        const auto chunk_id = stmt->getInt64();
+        const auto data_id = stmt->getInt64();
+        stmt =
+            diskCache->prepare("UPDATE chunk_data SET data = ? WHERE id = ?");
+        if (!stmt)
+            return;
+        stmt->bindBlob(blob.data(), blob.size());
+        stmt->bindInt64(data_id);
+        {
+            const auto ret = stmt->execute();
+            if (ret != SQLITE_DONE) {
+                pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+                return;
+            }
+        }
+
+        diskCache->move_to_head(chunk_id);
+
+        return;
+    } else if (mainRet != SQLITE_DONE) {
+        pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+        return;
+    }
+
+    // Lambda to recycle an existing entry that was either invalidated, or
+    // least recently used.
+    const auto reuseExistingEntry = [ctx, &blob, &diskCache, hDB, &url,
+                                     chunkIdx, &dataPtr](
+        std::unique_ptr<SQLiteStatement> &stmtIn) {
+        const auto chunk_id = stmtIn->getInt64();
+        const auto data_id = stmtIn->getInt64();
+        if (data_id <= 0) {
+            pj_log(ctx, PJ_LOG_ERROR, "data_id <= 0");
+            return;
+        }
+
+        auto l_stmt =
+            diskCache->prepare("UPDATE chunk_data SET data = ? WHERE id = ?");
+        if (!l_stmt)
+            return;
+        l_stmt->bindBlob(blob.data(), blob.size());
+        l_stmt->bindInt64(data_id);
+        {
+            const auto ret2 = l_stmt->execute();
+            if (ret2 != SQLITE_DONE) {
+                pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+                return;
+            }
+        }
+
+        l_stmt = diskCache->prepare("UPDATE chunks SET url = ?, "
+                                    "offset = ?, data_size = ?, data_id = ? "
+                                    "WHERE id = ?");
+        if (!l_stmt)
+            return;
+        l_stmt->bindText(url.c_str());
+        l_stmt->bindInt64(chunkIdx * DOWNLOAD_CHUNK_SIZE);
+        l_stmt->bindInt64(dataPtr->size());
+        l_stmt->bindInt64(data_id);
+        l_stmt->bindInt64(chunk_id);
+        {
+            const auto ret2 = l_stmt->execute();
+            if (ret2 != SQLITE_DONE) {
+                pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+                return;
+            }
+        }
+
+        diskCache->move_to_head(chunk_id);
+    };
+
+    // Find if there is an invalidated chunk we can reuse
+    stmt = diskCache->prepare(
+        "SELECT id, data_id FROM chunks "
+        "WHERE id = (SELECT tail FROM linked_chunks_head_tail) AND "
+        "url = " INVALIDATED_SQL_LITERAL);
+    if (!stmt)
+        return;
+    {
+        const auto ret = stmt->execute();
+        if (ret == SQLITE_ROW) {
+            reuseExistingEntry(stmt);
+            return;
+        } else if (ret != SQLITE_DONE) {
+            pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+            return;
+        }
+    }
+
+    // Check if we have not reached the max size of the cache
+    stmt = diskCache->prepare("SELECT COUNT(*) FROM chunks");
+    if (!stmt)
+        return;
+    {
+        const auto ret = stmt->execute();
+        if (ret != SQLITE_ROW) {
+            pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+            return;
+        }
+    }
+
+    const auto max_size = pj_context_get_grid_cache_max_size(ctx);
+    if (max_size > 0 &&
+        static_cast<long long>(stmt->getInt64() * DOWNLOAD_CHUNK_SIZE) >=
+            max_size) {
+        stmt = diskCache->prepare(
+            "SELECT id, data_id FROM chunks "
+            "WHERE id = (SELECT tail FROM linked_chunks_head_tail)");
+        if (!stmt)
+            return;
+
+        const auto ret = stmt->execute();
+        if (ret != SQLITE_ROW) {
+            pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+            return;
+        }
+        reuseExistingEntry(stmt);
+        return;
+    }
+
+    // Otherwise just append a new entry
+    stmt = diskCache->prepare("INSERT INTO chunk_data(data) VALUES (?)");
+    if (!stmt)
+        return;
+    stmt->bindBlob(blob.data(), blob.size());
+    {
+        const auto ret = stmt->execute();
+        if (ret != SQLITE_DONE) {
+            pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+            return;
+        }
+    }
+
+    const auto chunk_data_id = sqlite3_last_insert_rowid(hDB);
+
+    stmt = diskCache->prepare("INSERT INTO chunks(url, offset, data_id, "
+                              "data_size) VALUES (?,?,?,?)");
+    if (!stmt)
+        return;
+    stmt->bindText(url.c_str());
+    stmt->bindInt64(chunkIdx * DOWNLOAD_CHUNK_SIZE);
+    stmt->bindInt64(chunk_data_id);
+    stmt->bindInt64(dataPtr->size());
+    {
+        const auto ret = stmt->execute();
+        if (ret != SQLITE_DONE) {
+            pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+            return;
+        }
+    }
+
+    const auto chunk_id = sqlite3_last_insert_rowid(hDB);
+
+    stmt = diskCache->prepare(
+        "INSERT INTO linked_chunks(chunk_id, prev, next) VALUES (?,NULL,NULL)");
+    if (!stmt)
+        return;
+    stmt->bindInt64(chunk_id);
+    if (stmt->execute() != SQLITE_DONE) {
+        pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+        return;
+    }
+
+    stmt = diskCache->prepare("SELECT head FROM linked_chunks_head_tail");
+    if (!stmt)
+        return;
+    if (stmt->execute() != SQLITE_ROW) {
+        pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+        return;
+    }
+    if (stmt->getInt64() == 0) {
+        stmt = diskCache->prepare(
+            "UPDATE linked_chunks_head_tail SET head = ?, tail = ?");
+        if (!stmt)
+            return;
+        stmt->bindInt64(chunk_id);
+        stmt->bindInt64(chunk_id);
+        if (stmt->execute() != SQLITE_DONE) {
+            pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+            return;
+        }
+    }
+
+    diskCache->move_to_head(chunk_id);
 }
 
 // ---------------------------------------------------------------------------
 
 std::shared_ptr<std::vector<unsigned char>>
-NetworkChunkCache::get(const std::string &url, unsigned long long chunkIdx) {
+NetworkChunkCache::get(PJ_CONTEXT *ctx, const std::string &url,
+                       unsigned long long chunkIdx) {
     std::shared_ptr<std::vector<unsigned char>> ret;
-    cache_.tryGet(Key(url, chunkIdx), ret);
+    if (cache_.tryGet(Key(url, chunkIdx), ret)) {
+        return ret;
+    }
+
+    auto diskCache = DiskChunkCache::open(ctx);
+    if (!diskCache)
+        return ret;
+    auto hDB = diskCache->handle();
+
+    auto stmt = diskCache->prepare(
+        "SELECT chunks.id, chunks.data_size, chunk_data.data FROM chunks "
+        "JOIN chunk_data ON chunks.id = chunk_data.id "
+        "WHERE chunks.url = ? AND chunks.offset = ?");
+    if (!stmt)
+        return ret;
+
+    stmt->bindText(url.c_str());
+    stmt->bindInt64(chunkIdx * DOWNLOAD_CHUNK_SIZE);
+
+    const auto mainRet = stmt->execute();
+    if (mainRet == SQLITE_ROW) {
+        const auto chunk_id = stmt->getInt64();
+        const auto data_size = stmt->getInt64();
+        int blob_size = 0;
+        const void *blob = stmt->getBlob(blob_size);
+        if (blob_size < data_size) {
+            pj_log(ctx, PJ_LOG_ERROR,
+                   "blob_size=%d < data_size for chunk_id=%d", blob_size,
+                   static_cast<int>(chunk_id));
+            return ret;
+        }
+        if (data_size > static_cast<sqlite3_int64>(DOWNLOAD_CHUNK_SIZE)) {
+            pj_log(ctx, PJ_LOG_ERROR, "data_size > DOWNLOAD_CHUNK_SIZE");
+            return ret;
+        }
+        ret.reset(new std::vector<unsigned char>());
+        ret->assign(reinterpret_cast<const unsigned char *>(blob),
+                    reinterpret_cast<const unsigned char *>(blob) +
+                        static_cast<size_t>(data_size));
+        cache_.insert(Key(url, chunkIdx), ret);
+
+        if (!diskCache->move_to_head(chunk_id))
+            return ret;
+    } else if (mainRet != SQLITE_DONE) {
+        pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+    }
+
     return ret;
 }
 
 // ---------------------------------------------------------------------------
 
-void NetworkChunkCache::clear() { cache_.clear(); }
+std::shared_ptr<std::vector<unsigned char>>
+NetworkChunkCache::get(PJ_CONTEXT *ctx, const std::string &url,
+                       unsigned long long chunkIdx, FileProperties &props) {
+    if (!gNetworkFileProperties.tryGet(ctx, url, props)) {
+        return nullptr;
+    }
+
+    return get(ctx, url, chunkIdx);
+}
 
 // ---------------------------------------------------------------------------
 
-static NetworkChunkCache gNetworkChunkCache{};
+void NetworkChunkCache::clearMemoryCache() { cache_.clear(); }
 
-struct FileProperties {
-    unsigned long long size;
-};
+// ---------------------------------------------------------------------------
 
-static lru11::Cache<std::string, FileProperties, MyMutex>
-    gNetworkFileProperties{};
+void NetworkChunkCache::clearDiskChunkCache(PJ_CONTEXT *ctx) {
+    auto diskCache = DiskChunkCache::open(ctx);
+    if (!diskCache)
+        return;
+    diskCache->closeAndUnlink();
+}
+
+// ---------------------------------------------------------------------------
+
+void NetworkFilePropertiesCache::insert(PJ_CONTEXT *ctx, const std::string &url,
+                                        FileProperties &props) {
+    time(&props.lastChecked);
+    cache_.insert(url, props);
+
+    auto diskCache = DiskChunkCache::open(ctx);
+    if (!diskCache)
+        return;
+    auto hDB = diskCache->handle();
+    auto stmt = diskCache->prepare("SELECT fileSize, lastModified, etag "
+                                   "FROM properties WHERE url = ?");
+    if (!stmt)
+        return;
+    stmt->bindText(url.c_str());
+    if (stmt->execute() == SQLITE_ROW) {
+        FileProperties cachedProps;
+        cachedProps.size = stmt->getInt64();
+        const char *lastModified = stmt->getText();
+        cachedProps.lastModified = lastModified ? lastModified : std::string();
+        const char *etag = stmt->getText();
+        cachedProps.etag = etag ? etag : std::string();
+        if (props.size != cachedProps.size ||
+            props.lastModified != cachedProps.lastModified ||
+            props.etag != cachedProps.etag) {
+
+            // If cached properties don't match recent fresh ones, invalidate
+            // cached chunks
+            stmt = diskCache->prepare("SELECT id FROM chunks WHERE url = ?");
+            if (!stmt)
+                return;
+            stmt->bindText(url.c_str());
+            std::vector<sqlite3_int64> ids;
+            while (stmt->execute() == SQLITE_ROW) {
+                ids.emplace_back(stmt->getInt64());
+                stmt->resetResIndex();
+            }
+
+            for (const auto id : ids) {
+                diskCache->move_to_tail(id);
+            }
+
+            stmt = diskCache->prepare(
+                "UPDATE chunks SET url = " INVALIDATED_SQL_LITERAL ", "
+                "offset = -1, data_size = 0 WHERE url = ?");
+            if (!stmt)
+                return;
+            stmt->bindText(url.c_str());
+            if (stmt->execute() != SQLITE_DONE) {
+                pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+                return;
+            }
+        }
+
+        stmt = diskCache->prepare("UPDATE properties SET lastChecked = ?, "
+                                  "fileSize = ?, lastModified = ?, etag = ? "
+                                  "WHERE url = ?");
+        if (!stmt)
+            return;
+        stmt->bindInt64(props.lastChecked);
+        stmt->bindInt64(props.size);
+        if (props.lastModified.empty())
+            stmt->bindNull();
+        else
+            stmt->bindText(props.lastModified.c_str());
+        if (props.etag.empty())
+            stmt->bindNull();
+        else
+            stmt->bindText(props.etag.c_str());
+        stmt->bindText(url.c_str());
+        if (stmt->execute() != SQLITE_DONE) {
+            pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+            return;
+        }
+    } else {
+        stmt = diskCache->prepare("INSERT INTO properties (url, lastChecked, "
+                                  "fileSize, lastModified, etag) VALUES "
+                                  "(?,?,?,?,?)");
+        if (!stmt)
+            return;
+        stmt->bindText(url.c_str());
+        stmt->bindInt64(props.lastChecked);
+        stmt->bindInt64(props.size);
+        if (props.lastModified.empty())
+            stmt->bindNull();
+        else
+            stmt->bindText(props.lastModified.c_str());
+        if (props.etag.empty())
+            stmt->bindNull();
+        else
+            stmt->bindText(props.etag.c_str());
+        if (stmt->execute() != SQLITE_DONE) {
+            pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+            return;
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+
+bool NetworkFilePropertiesCache::tryGet(PJ_CONTEXT *ctx, const std::string &url,
+                                        FileProperties &props) {
+    if (cache_.tryGet(url, props)) {
+        return true;
+    }
+
+    auto diskCache = DiskChunkCache::open(ctx);
+    if (!diskCache)
+        return false;
+    auto stmt =
+        diskCache->prepare("SELECT lastChecked, fileSize, lastModified, etag "
+                           "FROM properties WHERE url = ?");
+    if (!stmt)
+        return false;
+    stmt->bindText(url.c_str());
+    if (stmt->execute() != SQLITE_ROW) {
+        return false;
+    }
+    props.lastChecked = stmt->getInt64();
+    props.size = stmt->getInt64();
+    const char *lastModified = stmt->getText();
+    props.lastModified = lastModified ? lastModified : std::string();
+    const char *etag = stmt->getText();
+    props.etag = etag ? etag : std::string();
+
+    const auto ttl = pj_context_get_grid_cache_ttl(ctx);
+    if (ttl > 0) {
+        time_t curTime;
+        time(&curTime);
+        if (curTime > props.lastChecked + ttl) {
+            props = FileProperties();
+            return false;
+        }
+    }
+    cache_.insert(url, props);
+    return true;
+}
+
+// ---------------------------------------------------------------------------
+
+void NetworkFilePropertiesCache::clearMemoryCache() { cache_.clear(); }
 
 // ---------------------------------------------------------------------------
 
@@ -330,15 +1363,11 @@ class NetworkFile : public File {
 // ---------------------------------------------------------------------------
 
 std::unique_ptr<File> NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) {
-    if (gNetworkChunkCache.get(filename, 0)) {
-        unsigned long long filesize = 0;
-        FileProperties props;
-        if (gNetworkFileProperties.tryGet(filename, props)) {
-            filesize = props.size;
-        }
+    FileProperties props;
+    if (gNetworkChunkCache.get(ctx, filename, 0, props)) {
         return std::unique_ptr<File>(new NetworkFile(
             ctx, filename, nullptr,
-            std::numeric_limits<unsigned long long>::max(), filesize));
+            std::numeric_limits<unsigned long long>::max(), props.size));
     } else {
         std::vector<unsigned char> buffer(DOWNLOAD_CHUNK_SIZE);
         size_t size_read = 0;
@@ -349,7 +1378,6 @@ std::unique_ptr<File> NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) {
             ctx, filename, 0, buffer.size(), &buffer[0], &size_read,
             errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data);
         buffer.resize(size_read);
-        gNetworkChunkCache.insert(filename, 0, std::move(buffer));
         if (!handle) {
             errorBuffer.resize(strlen(errorBuffer.data()));
             pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", filename,
@@ -364,16 +1392,32 @@ std::unique_ptr<File> NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) {
                 const char *slash = strchr(contentRange, '/');
                 if (slash) {
                     filesize = std::stoull(slash + 1);
-                    FileProperties props;
+
                     props.size = filesize;
-                    gNetworkFileProperties.insert(filename, props);
+
+                    const char *lastModified = ctx->networking.get_header_value(
+                        ctx, handle, "Last-Modified",
+                        ctx->networking.user_data);
+                    if (lastModified)
+                        props.lastModified = lastModified;
+
+                    const char *etag = ctx->networking.get_header_value(
+                        ctx, handle, "ETag", ctx->networking.user_data);
+                    if (etag)
+                        props.etag = etag;
+
+                    gNetworkFileProperties.insert(ctx, filename, props);
                 }
             }
+            if (filesize != 0) {
+                gNetworkChunkCache.insert(ctx, filename, 0, std::move(buffer));
+            }
         }
 
         return std::unique_ptr<File>(
-            handle ? new NetworkFile(ctx, filename, handle, size_read, filesize)
-                   : nullptr);
+            handle != nullptr && filesize != 0
+                ? new NetworkFile(ctx, filename, handle, size_read, filesize)
+                : nullptr);
     }
 }
 
@@ -389,7 +1433,7 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) {
         const auto chunkIdxToDownload = iterOffset / DOWNLOAD_CHUNK_SIZE;
         const auto offsetToDownload = chunkIdxToDownload * DOWNLOAD_CHUNK_SIZE;
         std::vector<unsigned char> region;
-        auto pChunk = gNetworkChunkCache.get(m_url, chunkIdxToDownload);
+        auto pChunk = gNetworkChunkCache.get(m_ctx, m_url, chunkIdxToDownload);
         if (pChunk != nullptr) {
             region = *pChunk;
         } else {
@@ -420,8 +1464,8 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) {
             // Note: this might get evicted if concurrent reads are done, but
             // this should not cause bugs. Just missed optimization.
             for (size_t i = 1; i < m_nBlocksToDownload; i++) {
-                if (gNetworkChunkCache.get(m_url, chunkIdxToDownload + i) !=
-                    nullptr) {
+                if (gNetworkChunkCache.get(m_ctx, m_url,
+                                           chunkIdxToDownload + i) != nullptr) {
                     m_nBlocksToDownload = i;
                     break;
                 }
@@ -468,7 +1512,7 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) {
                     region.data() + i * DOWNLOAD_CHUNK_SIZE,
                     region.data() +
                         std::min((i + 1) * DOWNLOAD_CHUNK_SIZE, region.size()));
-                gNetworkChunkCache.insert(m_url, chunkIdxToDownload + i,
+                gNetworkChunkCache.insert(m_ctx, m_url, chunkIdxToDownload + i,
                                           std::move(chunk));
             }
         }
@@ -826,13 +1870,15 @@ static const char *pj_curl_get_header_value(PJ_CONTEXT *,
     auto pos = ci_find(handle->m_headers, header_name);
     if (pos == std::string::npos)
         return nullptr;
+    pos += strlen(header_name);
     const char *c_str = handle->m_headers.c_str();
     if (c_str[pos] == ':')
         pos++;
     while (c_str[pos] == ' ')
         pos++;
     auto posEnd = pos;
-    while (c_str[posEnd] != '\n' && c_str[posEnd] != '\0')
+    while (c_str[posEnd] != '\r' && c_str[posEnd] != '\n' &&
+           c_str[posEnd] != '\0')
         posEnd++;
     handle->m_lastval = handle->m_headers.substr(pos, posEnd - pos);
     return handle->m_lastval.c_str();
@@ -843,7 +1889,7 @@ static const char *pj_curl_get_header_value(PJ_CONTEXT *,
 // ---------------------------------------------------------------------------
 
 static PROJ_NETWORK_HANDLE *
-no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */,
+no_op_network_open(PJ_CONTEXT *, const char * /* url */,
                    unsigned long long, /* offset */
                    size_t,             /* size to read */
                    void *,             /* buffer to update with bytes read*/
@@ -880,9 +1926,9 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) {
 
 // ---------------------------------------------------------------------------
 
-void FileManager::clearCache() {
-    gNetworkChunkCache.clear();
-    gNetworkFileProperties.clear();
+void FileManager::clearMemoryCache() {
+    gNetworkChunkCache.clearMemoryCache();
+    gNetworkFileProperties.clearMemoryCache();
 }
 
 // ---------------------------------------------------------------------------
@@ -974,6 +2020,99 @@ void proj_context_set_url_endpoint(PJ_CONTEXT *ctx, const char *url) {
 
 // ---------------------------------------------------------------------------
 
+/** Enable or disable the local cache of grid chunks
+*
+* This overrides the setting in the PROJ configuration file.
+*
+* @param ctx PROJ context, or NULL
+* @param enabled TRUE if the cache is enabled.
+*/
+void proj_grid_cache_set_enable(PJ_CONTEXT *ctx, int enabled) {
+    if (ctx == nullptr) {
+        ctx = pj_get_default_ctx();
+    }
+    // Load ini file, now so as to override its settings
+    pj_load_ini(ctx);
+    ctx->gridChunkCache.enabled = enabled != FALSE;
+}
+
+// ---------------------------------------------------------------------------
+
+/** Override, for the considered context, the path and file of the local
+* cache of grid chunks.
+*
+* @param ctx PROJ context, or NULL
+* @param fullname Full name to the cache (encoded in UTF-8). If set to NULL,
+*                 caching will be disabled.
+*/
+void proj_grid_cache_set_filename(PJ_CONTEXT *ctx, const char *fullname) {
+    if (ctx == nullptr) {
+        ctx = pj_get_default_ctx();
+    }
+    // Load ini file, now so as to override its settings
+    pj_load_ini(ctx);
+    ctx->gridChunkCache.filename = fullname ? fullname : std::string();
+}
+
+// ---------------------------------------------------------------------------
+
+/** Override, for the considered context, the maximum size of the local
+* cache of grid chunks.
+*
+* @param ctx PROJ context, or NULL
+* @param max_size_MB Maximum size, in mega-bytes (1024*1024 bytes), or
+*                    negative value to set unlimited size.
+*/
+void proj_grid_cache_set_max_size(PJ_CONTEXT *ctx, int max_size_MB) {
+    if (ctx == nullptr) {
+        ctx = pj_get_default_ctx();
+    }
+    // Load ini file, now so as to override its settings
+    pj_load_ini(ctx);
+    ctx->gridChunkCache.max_size =
+        max_size_MB < 0 ? -1
+                        : static_cast<long long>(max_size_MB) * 1024 * 1024;
+    if (max_size_MB == 0) {
+        // For debug purposes only
+        const char *env_var = getenv("PROJ_GRID_CACHE_MAX_SIZE_BYTES");
+        if (env_var && env_var[0] != '\0') {
+            ctx->gridChunkCache.max_size = atoi(env_var);
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+
+/** Override, for the considered context, the time-to-live delay for
+* re-checking if the cached properties of files are still up-to-date.
+*
+* @param ctx PROJ context, or NULL
+* @param ttl_seconds Delay in seconds. Use negative value for no expiration.
+*/
+void proj_grid_cache_set_ttl(PJ_CONTEXT *ctx, int ttl_seconds) {
+    if (ctx == nullptr) {
+        ctx = pj_get_default_ctx();
+    }
+    // Load ini file, now so as to override its settings
+    pj_load_ini(ctx);
+    ctx->gridChunkCache.ttl = ttl_seconds;
+}
+
+// ---------------------------------------------------------------------------
+
+/** Clear the local cache of grid chunks.
+*
+* @param ctx PROJ context, or NULL
+*/
+void proj_grid_cache_clear(PJ_CONTEXT *ctx) {
+    if (ctx == nullptr) {
+        ctx = pj_get_default_ctx();
+    }
+    NS_PROJ::gNetworkChunkCache.clearDiskChunkCache(ctx);
+}
+
+// ---------------------------------------------------------------------------
+
 //! @cond Doxygen_Suppress
 
 bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) {
@@ -994,4 +2133,98 @@ bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) {
     return ctx->networking.enabled;
 }
 
+// ---------------------------------------------------------------------------
+
+#ifdef _WIN32
+
+static std::wstring UTF8ToWString(const std::string &str) {
+    using convert_typeX = std::codecvt_utf8<wchar_t>;
+    std::wstring_convert<convert_typeX, wchar_t> converterX;
+
+    return converterX.from_bytes(str);
+}
+
+// ---------------------------------------------------------------------------
+
+static std::string WStringToUTF8(const std::wstring &wstr) {
+    using convert_typeX = std::codecvt_utf8<wchar_t>;
+    std::wstring_convert<convert_typeX, wchar_t> converterX;
+
+    return converterX.to_bytes(wstr);
+}
+#endif
+
+// ---------------------------------------------------------------------------
+
+static void CreateDirectory(const std::string &path) {
+#ifdef _WIN32
+    struct __stat64 buf;
+    const auto wpath = UTF8ToWString(path);
+    if (_wstat64(wpath.c_str(), &buf) == 0)
+        return;
+    auto pos = path.find_last_of("/\\");
+    if (pos == 0 || pos == std::string::npos)
+        return;
+    CreateDirectory(path.substr(0, pos));
+    _wmkdir(wpath.c_str());
+#else
+    struct stat buf;
+    if (stat(path.c_str(), &buf) == 0)
+        return;
+    auto pos = path.find_last_of("/\\");
+    if (pos == 0 || pos == std::string::npos)
+        return;
+    CreateDirectory(path.substr(0, pos));
+    mkdir(path.c_str(), 0755);
+#endif
+}
+
+// ---------------------------------------------------------------------------
+
+std::string pj_context_get_grid_cache_filename(PJ_CONTEXT *ctx) {
+    pj_load_ini(ctx);
+    if (!ctx->gridChunkCache.filename.empty()) {
+        return ctx->gridChunkCache.filename;
+    }
+    std::string path;
+#ifdef _WIN32
+    std::wstring wPath;
+    wPath.resize(MAX_PATH);
+    if (SHGetFolderPathW(nullptr, CSIDL_LOCAL_APPDATA, nullptr, 0, &wPath[0]) ==
+        S_OK) {
+        wPath.resize(wcslen(wPath.data()));
+        path = WStringToUTF8(wPath);
+    } else {
+        const char *local_app_data = getenv("LOCALAPPDATA");
+        if (!local_app_data) {
+            local_app_data = getenv("TEMP");
+            if (!local_app_data) {
+                local_app_data = "c:/users";
+            }
+        }
+        path = local_app_data;
+    }
+#else
+    const char *xdg_data_home = getenv("XDG_DATA_HOME");
+    if (xdg_data_home != nullptr) {
+        path = xdg_data_home;
+    } else {
+        const char *home = getenv("HOME");
+        if (home) {
+#if defined(__MACH__) && defined(__APPLE__)
+            path = std::string(home) + "/Library/Logs";
+#else
+            path = std::string(home) + "/.local/share";
+#endif
+        } else {
+            path = "/tmp";
+        }
+    }
+#endif
+    path += "/proj";
+    CreateDirectory(path);
+    ctx->gridChunkCache.filename = path + "/cache.db";
+    return ctx->gridChunkCache.filename;
+}
+
 //! @endcond
diff --git a/src/filemanager.hpp b/src/filemanager.hpp
index 972634c2..993048a7 100644
--- a/src/filemanager.hpp
+++ b/src/filemanager.hpp
@@ -53,7 +53,7 @@ class FileManager {
 
     static void fillDefaultNetworkInterface(PJ_CONTEXT *ctx);
 
-    static void clearCache();
+    static void clearMemoryCache();
 };
 
 // ---------------------------------------------------------------------------
diff --git a/src/iso19111/factory.cpp b/src/iso19111/factory.cpp
index dae8680c..7fb248c6 100644
--- a/src/iso19111/factory.cpp
+++ b/src/iso19111/factory.cpp
@@ -45,6 +45,8 @@
 #include "proj/internal/lru_cache.hpp"
 #include "proj/internal/tracing.hpp"
 
+#include "sqlite3.hpp"
+
 #include <cmath>
 #include <cstdlib>
 #include <cstring>
@@ -275,9 +277,7 @@ struct DatabaseContext::Private {
     void registerFunctions();
 
 #ifdef ENABLE_CUSTOM_LOCKLESS_VFS
-    std::string thisNamePtr_{};
-    sqlite3_vfs *vfs_{};
-    bool createCustomVFS();
+    std::unique_ptr<SQLite3VFS> vfs_{};
 #endif
 
     Private(const Private &) = delete;
@@ -294,13 +294,6 @@ DatabaseContext::Private::~Private() {
     assert(recLevel_ == 0);
 
     closeDB();
-
-#ifdef ENABLE_CUSTOM_LOCKLESS_VFS
-    if (vfs_) {
-        sqlite3_vfs_unregister(vfs_);
-        delete vfs_;
-    }
-#endif
 }
 
 // ---------------------------------------------------------------------------
@@ -496,101 +489,6 @@ void DatabaseContext::Private::cache(const std::string &code,
 
 // ---------------------------------------------------------------------------
 
-#ifdef ENABLE_CUSTOM_LOCKLESS_VFS
-
-typedef int (*ClosePtr)(sqlite3_file *);
-
-static int VFSClose(sqlite3_file *file) {
-    sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr);
-    assert(defaultVFS);
-    ClosePtr defaultClosePtr;
-    std::memcpy(&defaultClosePtr,
-                reinterpret_cast<char *>(file) + defaultVFS->szOsFile,
-                sizeof(ClosePtr));
-    void *methods = const_cast<sqlite3_io_methods *>(file->pMethods);
-    int ret = defaultClosePtr(file);
-    std::free(methods);
-    return ret;
-}
-
-// No-lock implementation
-static int VSFLock(sqlite3_file *, int) { return SQLITE_OK; }
-
-static int VSFUnlock(sqlite3_file *, int) { return SQLITE_OK; }
-
-static int VFSOpen(sqlite3_vfs *vfs, const char *name, sqlite3_file *file,
-                   int flags, int *outFlags) {
-    sqlite3_vfs *defaultVFS = static_cast<sqlite3_vfs *>(vfs->pAppData);
-    int ret = defaultVFS->xOpen(defaultVFS, name, file, flags, outFlags);
-    if (ret == SQLITE_OK) {
-        ClosePtr defaultClosePtr = file->pMethods->xClose;
-        assert(defaultClosePtr);
-        sqlite3_io_methods *methods = static_cast<sqlite3_io_methods *>(
-            std::malloc(sizeof(sqlite3_io_methods)));
-        if (!methods) {
-            file->pMethods->xClose(file);
-            return SQLITE_NOMEM;
-        }
-        memcpy(methods, file->pMethods, sizeof(sqlite3_io_methods));
-        methods->xClose = VFSClose;
-        methods->xLock = VSFLock;
-        methods->xUnlock = VSFUnlock;
-        file->pMethods = methods;
-        // Save original xClose pointer at end of file structure
-        std::memcpy(reinterpret_cast<char *>(file) + defaultVFS->szOsFile,
-                    &defaultClosePtr, sizeof(ClosePtr));
-    }
-    return ret;
-}
-
-static int VFSAccess(sqlite3_vfs *vfs, const char *zName, int flags,
-                     int *pResOut) {
-    sqlite3_vfs *defaultVFS = static_cast<sqlite3_vfs *>(vfs->pAppData);
-    // Do not bother stat'ing for journal or wal files
-    if (std::strstr(zName, "-journal") || std::strstr(zName, "-wal")) {
-        *pResOut = false;
-        return SQLITE_OK;
-    }
-    return defaultVFS->xAccess(defaultVFS, zName, flags, pResOut);
-}
-
-// ---------------------------------------------------------------------------
-
-bool DatabaseContext::Private::createCustomVFS() {
-
-    sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr);
-    assert(defaultVFS);
-
-    std::ostringstream buffer;
-    buffer << this;
-    thisNamePtr_ = buffer.str();
-
-    vfs_ = new sqlite3_vfs();
-    vfs_->iVersion = 1;
-    vfs_->szOsFile = defaultVFS->szOsFile + sizeof(ClosePtr);
-    vfs_->mxPathname = defaultVFS->mxPathname;
-    vfs_->zName = thisNamePtr_.c_str();
-    vfs_->pAppData = defaultVFS;
-    vfs_->xOpen = VFSOpen;
-    vfs_->xDelete = defaultVFS->xDelete;
-    vfs_->xAccess = VFSAccess;
-    vfs_->xFullPathname = defaultVFS->xFullPathname;
-    vfs_->xDlOpen = defaultVFS->xDlOpen;
-    vfs_->xDlError = defaultVFS->xDlError;
-    vfs_->xDlSym = defaultVFS->xDlSym;
-    vfs_->xDlClose = defaultVFS->xDlClose;
-    vfs_->xRandomness = defaultVFS->xRandomness;
-    vfs_->xSleep = defaultVFS->xSleep;
-    vfs_->xCurrentTime = defaultVFS->xCurrentTime;
-    vfs_->xGetLastError = defaultVFS->xGetLastError;
-    vfs_->xCurrentTimeInt64 = defaultVFS->xCurrentTimeInt64;
-    return sqlite3_vfs_register(vfs_, false) == SQLITE_OK;
-}
-
-#endif // ENABLE_CUSTOM_LOCKLESS_VFS
-
-// ---------------------------------------------------------------------------
-
 void DatabaseContext::Private::open(const std::string &databasePath,
                                     PJ_CONTEXT *ctx) {
     setPjCtxt(ctx ? ctx : pj_get_default_ctx());
@@ -605,21 +503,23 @@ void DatabaseContext::Private::open(const std::string &databasePath,
         }
     }
 
-    if (
 #ifdef ENABLE_CUSTOM_LOCKLESS_VFS
-        !createCustomVFS() ||
-#endif
+    vfs_ = SQLite3VFS::create(false, true, true);
+    if (vfs_ == nullptr ||
         sqlite3_open_v2(path.c_str(), &sqlite_handle_,
                         SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX,
-#ifdef ENABLE_CUSTOM_LOCKLESS_VFS
-                        thisNamePtr_.c_str()
+                        vfs_->name()) != SQLITE_OK ||
+        !sqlite_handle_) {
+        throw FactoryException("Open of " + path + " failed");
+    }
 #else
-                        nullptr
-#endif
-                            ) != SQLITE_OK ||
+    if (sqlite3_open_v2(path.c_str(), &sqlite_handle_,
+                        SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX,
+                        nullptr) != SQLITE_OK ||
         !sqlite_handle_) {
         throw FactoryException("Open of " + path + " failed");
     }
+#endif
 
     databasePath_ = path;
     registerFunctions();
diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake
index 12dcb366..18c91021 100644
--- a/src/lib_proj.cmake
+++ b/src/lib_proj.cmake
@@ -284,6 +284,8 @@ set(SRC_LIBPROJ_CORE
   grids.cpp
   filemanager.hpp
   filemanager.cpp
+  sqlite3.hpp
+  sqlite3.cpp
   ${CMAKE_CURRENT_BINARY_DIR}/proj_config.h
 )
 
diff --git a/src/malloc.cpp b/src/malloc.cpp
index 1c539b6b..9ac28546 100644
--- a/src/malloc.cpp
+++ b/src/malloc.cpp
@@ -263,5 +263,5 @@ void proj_cleanup() {
 /*****************************************************************************/
   pj_clear_initcache();
   pj_deallocate_grids();
-  FileManager::clearCache();
+  FileManager::clearMemoryCache();
 }
diff --git a/src/open_lib.cpp b/src/open_lib.cpp
index cde5be7b..23ee79a0 100644
--- a/src/open_lib.cpp
+++ b/src/open_lib.cpp
@@ -520,6 +520,16 @@ void pj_load_ini(projCtx ctx)
                                                 ci_equal(value, "YES") ||
                                                 ci_equal(value, "TRUE");
                 }
+            } else if ( key == "cache_enabled" ) {
+                ctx->gridChunkCache.enabled = ci_equal(value, "ON") ||
+                                              ci_equal(value, "YES") ||
+                                              ci_equal(value, "TRUE");
+            } else if ( key == "cache_size_MB" ) {
+                const int val = atoi(value.c_str());
+                ctx->gridChunkCache.max_size = val > 0 ?
+                    static_cast<long long>(val) * 1024 * 1024 : -1;
+            } else if ( key == "cache_ttl_sec" ) {
+                ctx->gridChunkCache.ttl = atoi(value.c_str());
             }
         }
 
diff --git a/src/proj.h b/src/proj.h
index 96b9c3f8..33057617 100644
--- a/src/proj.h
+++ b/src/proj.h
@@ -430,6 +430,16 @@ int PROJ_DLL proj_context_set_enable_network(PJ_CONTEXT* ctx,
 
 void PROJ_DLL proj_context_set_url_endpoint(PJ_CONTEXT* ctx, const char* url);
 
+void PROJ_DLL proj_grid_cache_set_enable(PJ_CONTEXT* ctx, int enabled);
+
+void PROJ_DLL proj_grid_cache_set_filename(PJ_CONTEXT* ctx, const char* fullname);
+
+void PROJ_DLL proj_grid_cache_set_max_size(PJ_CONTEXT* ctx, int max_size_MB);
+
+void PROJ_DLL proj_grid_cache_set_ttl(PJ_CONTEXT* ctx, int ttl_seconds);
+
+void PROJ_DLL proj_grid_cache_clear(PJ_CONTEXT* ctx);
+
 /*! @cond Doxygen_Suppress */
 
 /* Manage the transformation definition object PJ */
diff --git a/src/proj_internal.h b/src/proj_internal.h
index 12ada034..63c53551 100644
--- a/src/proj_internal.h
+++ b/src/proj_internal.h
@@ -677,6 +677,14 @@ struct projNetworkCallbacksAndData
     void* user_data = nullptr;
 };
 
+struct projGridChunkCache
+{
+    bool enabled = true;
+    std::string filename{};
+    long long max_size = 100 * 1024 * 1024;
+    int ttl = 86400; // 1 day
+};
+
 /* proj thread context */
 struct projCtx_t {
     int     last_errno = 0;
@@ -701,6 +709,8 @@ struct projCtx_t {
     bool iniFileLoaded = false;
     std::string endpoint{};
 
+    projGridChunkCache gridChunkCache{};
+
     int projStringParserCreateFromPROJStringRecursionCounter = 0; // to avoid potential infinite recursion in PROJStringParser::createFromPROJString()
 
     projCtx_t() = default;
@@ -834,6 +844,9 @@ std::string pj_context_get_url_endpoint(PJ_CONTEXT* ctx);
 
 void pj_load_ini(PJ_CONTEXT* ctx);
 
+// For testing purposes
+std::string PROJ_DLL pj_context_get_grid_cache_filename(PJ_CONTEXT *ctx);
+
 /* classic public API */
 #include "proj_api.h"
 
diff --git a/src/sqlite3.cpp b/src/sqlite3.cpp
new file mode 100644
index 00000000..0c89c0b9
--- /dev/null
+++ b/src/sqlite3.cpp
@@ -0,0 +1,185 @@
+/******************************************************************************
+ * Project:  PROJ
+ * Purpose:  SQLite3 related utilities
+ * Author:   Even Rouault, <even.rouault at spatialys.com>
+ *
+ ******************************************************************************
+ * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *****************************************************************************/
+
+#include "sqlite3.hpp"
+
+#include <cstdlib>
+#include <cstring>
+#include <sstream> // std::ostringstream
+
+NS_PROJ_START
+
+// ---------------------------------------------------------------------------
+
+SQLite3VFS::SQLite3VFS(sqlite3_vfs *vfs) : vfs_(vfs) {}
+
+// ---------------------------------------------------------------------------
+
+SQLite3VFS::~SQLite3VFS() {
+    if (vfs_) {
+        sqlite3_vfs_unregister(vfs_);
+        delete vfs_;
+    }
+}
+
+// ---------------------------------------------------------------------------
+
+struct pj_sqlite3_vfs : public sqlite3_vfs {
+    std::string namePtr{};
+    bool fakeSync = false;
+    bool fakeLock = false;
+};
+
+// ---------------------------------------------------------------------------
+
+const char *SQLite3VFS::name() const {
+    return static_cast<pj_sqlite3_vfs *>(vfs_)->namePtr.c_str();
+}
+
+// ---------------------------------------------------------------------------
+
+typedef int (*ClosePtr)(sqlite3_file *);
+
+// ---------------------------------------------------------------------------
+
+static int VFSClose(sqlite3_file *file) {
+    sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr);
+    assert(defaultVFS);
+    ClosePtr defaultClosePtr;
+    std::memcpy(&defaultClosePtr,
+                reinterpret_cast<char *>(file) + defaultVFS->szOsFile,
+                sizeof(ClosePtr));
+    void *methods = const_cast<sqlite3_io_methods *>(file->pMethods);
+    int ret = defaultClosePtr(file);
+    std::free(methods);
+    return ret;
+}
+
+// ---------------------------------------------------------------------------
+
+static int VSFNoOpLockUnlockSync(sqlite3_file *, int) { return SQLITE_OK; }
+
+// ---------------------------------------------------------------------------
+
+static int VFSCustomOpen(sqlite3_vfs *vfs, const char *name, sqlite3_file *file,
+                         int flags, int *outFlags) {
+    auto realVFS = static_cast<pj_sqlite3_vfs *>(vfs);
+    sqlite3_vfs *defaultVFS = static_cast<sqlite3_vfs *>(vfs->pAppData);
+    int ret = defaultVFS->xOpen(defaultVFS, name, file, flags, outFlags);
+    if (ret == SQLITE_OK) {
+        ClosePtr defaultClosePtr = file->pMethods->xClose;
+        assert(defaultClosePtr);
+        sqlite3_io_methods *methods = static_cast<sqlite3_io_methods *>(
+            std::malloc(sizeof(sqlite3_io_methods)));
+        if (!methods) {
+            file->pMethods->xClose(file);
+            return SQLITE_NOMEM;
+        }
+        memcpy(methods, file->pMethods, sizeof(sqlite3_io_methods));
+        methods->xClose = VFSClose;
+        if (realVFS->fakeSync) {
+            // Disable xSync because it can be significantly slow and we don't
+            // need
+            // that level of data integrity garanty for the cache.
+            methods->xSync = VSFNoOpLockUnlockSync;
+        }
+        if (realVFS->fakeLock) {
+            methods->xLock = VSFNoOpLockUnlockSync;
+            methods->xUnlock = VSFNoOpLockUnlockSync;
+        }
+        file->pMethods = methods;
+        // Save original xClose pointer at end of file structure
+        std::memcpy(reinterpret_cast<char *>(file) + defaultVFS->szOsFile,
+                    &defaultClosePtr, sizeof(ClosePtr));
+    }
+    return ret;
+}
+
+// ---------------------------------------------------------------------------
+
+static int VFSCustomAccess(sqlite3_vfs *vfs, const char *zName, int flags,
+                           int *pResOut) {
+    sqlite3_vfs *defaultVFS = static_cast<sqlite3_vfs *>(vfs->pAppData);
+    // Do not bother stat'ing for journal or wal files
+    if (std::strstr(zName, "-journal") || std::strstr(zName, "-wal")) {
+        *pResOut = false;
+        return SQLITE_OK;
+    }
+    return defaultVFS->xAccess(defaultVFS, zName, flags, pResOut);
+}
+
+// ---------------------------------------------------------------------------
+
+std::unique_ptr<SQLite3VFS> SQLite3VFS::create(bool fakeSync, bool fakeLock,
+                                               bool skipStatJournalAndWAL) {
+    sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr);
+    assert(defaultVFS);
+
+    auto vfs = new pj_sqlite3_vfs();
+    vfs->fakeSync = fakeSync;
+    vfs->fakeLock = fakeLock;
+
+    auto vfsUnique = std::unique_ptr<SQLite3VFS>(new SQLite3VFS(vfs));
+
+    std::ostringstream buffer;
+    buffer << vfs;
+    vfs->namePtr = buffer.str();
+
+    vfs->iVersion = 1;
+    vfs->szOsFile = defaultVFS->szOsFile + sizeof(ClosePtr);
+    vfs->mxPathname = defaultVFS->mxPathname;
+    vfs->zName = vfs->namePtr.c_str();
+    vfs->pAppData = defaultVFS;
+    vfs->xOpen = VFSCustomOpen;
+    vfs->xDelete = defaultVFS->xDelete;
+    vfs->xAccess =
+        skipStatJournalAndWAL ? VFSCustomAccess : defaultVFS->xAccess;
+    vfs->xFullPathname = defaultVFS->xFullPathname;
+    vfs->xDlOpen = defaultVFS->xDlOpen;
+    vfs->xDlError = defaultVFS->xDlError;
+    vfs->xDlSym = defaultVFS->xDlSym;
+    vfs->xDlClose = defaultVFS->xDlClose;
+    vfs->xRandomness = defaultVFS->xRandomness;
+    vfs->xSleep = defaultVFS->xSleep;
+    vfs->xCurrentTime = defaultVFS->xCurrentTime;
+    vfs->xGetLastError = defaultVFS->xGetLastError;
+    vfs->xCurrentTimeInt64 = defaultVFS->xCurrentTimeInt64;
+    if (sqlite3_vfs_register(vfs, false) == SQLITE_OK) {
+        return vfsUnique;
+    }
+    delete vfsUnique->vfs_;
+    vfsUnique->vfs_ = nullptr;
+    return nullptr;
+}
+
+// ---------------------------------------------------------------------------
+
+SQLiteStatement::SQLiteStatement(sqlite3_stmt *hStmtIn) : hStmt(hStmtIn) {}
+
+// ---------------------------------------------------------------------------
+
+NS_PROJ_END
\ No newline at end of file
diff --git a/src/sqlite3.hpp b/src/sqlite3.hpp
new file mode 100644
index 00000000..ef141d1f
--- /dev/null
+++ b/src/sqlite3.hpp
@@ -0,0 +1,127 @@
+/******************************************************************************
+ * Project:  PROJ
+ * Purpose:  SQLite3 related utilities
+ * Author:   Even Rouault, <even.rouault at spatialys.com>
+ *
+ ******************************************************************************
+ * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *****************************************************************************/
+
+#ifndef SQLITE3_HPP_INCLUDED
+#define SQLITE3_HPP_INCLUDED
+
+#include <memory>
+
+#include <sqlite3.h>
+
+#include "proj.h"
+#include "proj/util.hpp"
+
+NS_PROJ_START
+
+//! @cond Doxygen_Suppress
+
+class SQLite3VFS {
+    sqlite3_vfs *vfs_ = nullptr;
+
+    explicit SQLite3VFS(sqlite3_vfs *vfs);
+
+    SQLite3VFS(const SQLite3VFS &) = delete;
+    SQLite3VFS &operator=(const SQLite3VFS &) = delete;
+
+  public:
+    ~SQLite3VFS();
+
+    static std::unique_ptr<SQLite3VFS> create(bool fakeSync, bool fakeLock,
+                                              bool skipStatJournalAndWAL);
+    const char *name() const;
+    sqlite3_vfs *raw() { return vfs_; }
+};
+
+// ---------------------------------------------------------------------------
+
+class SQLiteStatement {
+    sqlite3_stmt *hStmt = nullptr;
+    int iBindIdx = 1;
+    int iResIdx = 0;
+    SQLiteStatement(const SQLiteStatement &) = delete;
+    SQLiteStatement &operator=(const SQLiteStatement &) = delete;
+
+  public:
+    explicit SQLiteStatement(sqlite3_stmt *hStmtIn);
+    ~SQLiteStatement() { sqlite3_finalize(hStmt); }
+
+    int execute() { return sqlite3_step(hStmt); }
+
+    void bindNull() {
+        sqlite3_bind_null(hStmt, iBindIdx);
+        iBindIdx++;
+    }
+
+    void bindText(const char *txt) {
+        sqlite3_bind_text(hStmt, iBindIdx, txt, -1, nullptr);
+        iBindIdx++;
+    }
+
+    void bindInt64(sqlite3_int64 v) {
+        sqlite3_bind_int64(hStmt, iBindIdx, v);
+        iBindIdx++;
+    }
+
+    void bindBlob(const void *blob, size_t blob_size) {
+        sqlite3_bind_blob(hStmt, iBindIdx, blob, static_cast<int>(blob_size),
+                          nullptr);
+        iBindIdx++;
+    }
+
+    const char *getText() {
+        auto ret = sqlite3_column_text(hStmt, iResIdx);
+        iResIdx++;
+        return reinterpret_cast<const char *>(ret);
+    }
+
+    sqlite3_int64 getInt64() {
+        auto ret = sqlite3_column_int64(hStmt, iResIdx);
+        iResIdx++;
+        return ret;
+    }
+
+    const void *getBlob(int &size) {
+        size = sqlite3_column_bytes(hStmt, iResIdx);
+        auto ret = sqlite3_column_blob(hStmt, iResIdx);
+        iResIdx++;
+        return ret;
+    }
+
+    void reset() {
+        sqlite3_reset(hStmt);
+        iBindIdx = 1;
+        iResIdx = 0;
+    }
+
+    void resetResIndex() { iResIdx = 0; }
+};
+
+//! @endcond Doxygen_Suppress
+
+NS_PROJ_END
+
+#endif // SQLITE3_HPP_INCLUDED