Merge RFC4 (#1865)

This commit is the result of the squashing of rfc4_dev branch in a single
commit. It implements mostly RFC 4 related work.

* Grid handling:
  - remove obsolete and presumably unfinished implementation of grid catalog functionality
  - all grid functionality is in grids.cpp/.hpp
  - vertical and horizontal grid shift: rework to no longer load whole grid into memory
  - remove hgrids and vgrids member from PJ structure, and store them in hgridshift/vgridshift/deformation structures
  - build systems: add optional libtiff dependency. Must be explicitly disabled if not desired
  - add support for horizontal and vertical grids in GeoTIFF, if libtiff is available
  - add GenericShiftGridSet and GenericShiftGrid classes, relying on TIFF grids, that can be used for generic purpose grid-based adjustment
  - add a +proj=xyzgridshift method to perform geocentric translation by grid. Used for French NTF to RGF93 transformation using gr3df97a.tif grid
  - deformation: add support for +grids= for GeoTIFF grids
  - horizontal grid shift: fix failures on points slightly outside a subgrid (fixes #209)

* File management:
  - add a filemanager.cpp/.hpp to deal with file related work
  - test for legacy proj_api.h fileapi
  - proj.h: add proj_context_set_fileapi() and proj_context_set_sqlite3_vfs_name() (fixes #866)
  - add capability to read resource files from the user writable directory

* Network access:
  - build systems: add optional curl dependency
  - add a curl-based default implementation for network related functionality
  - proj.h: add C API to control network functionality, and optionaly provide network callbacks
  - add data/proj.ini with default settings
  - add a SQLite3 local cache of downloaded chunks
  - add proj_is_download_needed() and proj_download_file()

* Use Win32 Unicode APIs and expect all strings to be UTF-8 (fixes #1765)
  For backward compatibility, if PROJ_LIB content is found to be not UTF-8 or
  pointing to a non existing directory, then an attempt at interpretating it
  in the ANSI page encoding is done.
  proj_context_set_search_paths() now assumes strings to be in UTF-8, and
  functions returning paths will also return values in UTF-8.

1 files changed, 194 insertions, 0 deletions

diff --git a/src/sqlite3_utils.cpp b/src/sqlite3_utils.cpp
new file mode 100644
index 00000000..673eb89c
--- /dev/null
+++ b/src/sqlite3_utils.cpp
@@ -0,0 +1,194 @@
+/******************************************************************************
+ * Project:  PROJ
+ * Purpose:  SQLite3 related utilities
+ * Author:   Even Rouault, <even.rouault at spatialys.com>
+ *
+ ******************************************************************************
+ * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *****************************************************************************/
+
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Weffc++"
+#endif
+
+#include "sqlite3_utils.hpp"
+
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+
+#include <cstdlib>
+#include <cstring>
+#include <sstream> // std::ostringstream
+
+NS_PROJ_START
+
+// ---------------------------------------------------------------------------
+
+SQLite3VFS::SQLite3VFS(sqlite3_vfs *vfs) : vfs_(vfs) {}
+
+// ---------------------------------------------------------------------------
+
+SQLite3VFS::~SQLite3VFS() {
+    if (vfs_) {
+        sqlite3_vfs_unregister(vfs_);
+        delete vfs_;
+    }
+}
+
+// ---------------------------------------------------------------------------
+
+struct pj_sqlite3_vfs : public sqlite3_vfs {
+    std::string namePtr{};
+    bool fakeSync = false;
+    bool fakeLock = false;
+};
+
+// ---------------------------------------------------------------------------
+
+const char *SQLite3VFS::name() const {
+    return static_cast<pj_sqlite3_vfs *>(vfs_)->namePtr.c_str();
+}
+
+// ---------------------------------------------------------------------------
+
+typedef int (*ClosePtr)(sqlite3_file *);
+
+// ---------------------------------------------------------------------------
+
+static int VFSClose(sqlite3_file *file) {
+    sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr);
+    assert(defaultVFS);
+    ClosePtr defaultClosePtr;
+    std::memcpy(&defaultClosePtr,
+                reinterpret_cast<char *>(file) + defaultVFS->szOsFile,
+                sizeof(ClosePtr));
+    void *methods = const_cast<sqlite3_io_methods *>(file->pMethods);
+    int ret = defaultClosePtr(file);
+    std::free(methods);
+    return ret;
+}
+
+// ---------------------------------------------------------------------------
+
+static int VSFNoOpLockUnlockSync(sqlite3_file *, int) { return SQLITE_OK; }
+
+// ---------------------------------------------------------------------------
+
+static int VFSCustomOpen(sqlite3_vfs *vfs, const char *name, sqlite3_file *file,
+                         int flags, int *outFlags) {
+    auto realVFS = static_cast<pj_sqlite3_vfs *>(vfs);
+    sqlite3_vfs *defaultVFS = static_cast<sqlite3_vfs *>(vfs->pAppData);
+    int ret = defaultVFS->xOpen(defaultVFS, name, file, flags, outFlags);
+    if (ret == SQLITE_OK) {
+        ClosePtr defaultClosePtr = file->pMethods->xClose;
+        assert(defaultClosePtr);
+        sqlite3_io_methods *methods = static_cast<sqlite3_io_methods *>(
+            std::malloc(sizeof(sqlite3_io_methods)));
+        if (!methods) {
+            file->pMethods->xClose(file);
+            return SQLITE_NOMEM;
+        }
+        memcpy(methods, file->pMethods, sizeof(sqlite3_io_methods));
+        methods->xClose = VFSClose;
+        if (realVFS->fakeSync) {
+            // Disable xSync because it can be significantly slow and we don't
+            // need
+            // that level of data integrity garanty for the cache.
+            methods->xSync = VSFNoOpLockUnlockSync;
+        }
+        if (realVFS->fakeLock) {
+            methods->xLock = VSFNoOpLockUnlockSync;
+            methods->xUnlock = VSFNoOpLockUnlockSync;
+        }
+        file->pMethods = methods;
+        // Save original xClose pointer at end of file structure
+        std::memcpy(reinterpret_cast<char *>(file) + defaultVFS->szOsFile,
+                    &defaultClosePtr, sizeof(ClosePtr));
+    }
+    return ret;
+}
+
+// ---------------------------------------------------------------------------
+
+static int VFSCustomAccess(sqlite3_vfs *vfs, const char *zName, int flags,
+                           int *pResOut) {
+    sqlite3_vfs *defaultVFS = static_cast<sqlite3_vfs *>(vfs->pAppData);
+    // Do not bother stat'ing for journal or wal files
+    if (std::strstr(zName, "-journal") || std::strstr(zName, "-wal")) {
+        *pResOut = false;
+        return SQLITE_OK;
+    }
+    return defaultVFS->xAccess(defaultVFS, zName, flags, pResOut);
+}
+
+// ---------------------------------------------------------------------------
+
+std::unique_ptr<SQLite3VFS> SQLite3VFS::create(bool fakeSync, bool fakeLock,
+                                               bool skipStatJournalAndWAL) {
+    sqlite3_vfs *defaultVFS = sqlite3_vfs_find(nullptr);
+    assert(defaultVFS);
+
+    auto vfs = new pj_sqlite3_vfs();
+    vfs->fakeSync = fakeSync;
+    vfs->fakeLock = fakeLock;
+
+    auto vfsUnique = std::unique_ptr<SQLite3VFS>(new SQLite3VFS(vfs));
+
+    std::ostringstream buffer;
+    buffer << vfs;
+    vfs->namePtr = buffer.str();
+
+    vfs->iVersion = 1;
+    vfs->szOsFile = defaultVFS->szOsFile + sizeof(ClosePtr);
+    vfs->mxPathname = defaultVFS->mxPathname;
+    vfs->zName = vfs->namePtr.c_str();
+    vfs->pAppData = defaultVFS;
+    vfs->xOpen = VFSCustomOpen;
+    vfs->xDelete = defaultVFS->xDelete;
+    vfs->xAccess =
+        skipStatJournalAndWAL ? VFSCustomAccess : defaultVFS->xAccess;
+    vfs->xFullPathname = defaultVFS->xFullPathname;
+    vfs->xDlOpen = defaultVFS->xDlOpen;
+    vfs->xDlError = defaultVFS->xDlError;
+    vfs->xDlSym = defaultVFS->xDlSym;
+    vfs->xDlClose = defaultVFS->xDlClose;
+    vfs->xRandomness = defaultVFS->xRandomness;
+    vfs->xSleep = defaultVFS->xSleep;
+    vfs->xCurrentTime = defaultVFS->xCurrentTime;
+    vfs->xGetLastError = defaultVFS->xGetLastError;
+    vfs->xCurrentTimeInt64 = defaultVFS->xCurrentTimeInt64;
+    if (sqlite3_vfs_register(vfs, false) == SQLITE_OK) {
+        return vfsUnique;
+    }
+    delete vfsUnique->vfs_;
+    vfsUnique->vfs_ = nullptr;
+    return nullptr;
+}
+
+// ---------------------------------------------------------------------------
+
+SQLiteStatement::SQLiteStatement(sqlite3_stmt *hStmtIn) : hStmt(hStmtIn) {}
+
+// ---------------------------------------------------------------------------
+
+NS_PROJ_END