Add proj_is_download_needed() and proj_download_file()

1 files changed, 416 insertions, 4 deletions

diff --git a/src/filemanager.cpp b/src/filemanager.cpp
index 1c49a16b..9acea83e 100644
--- a/src/filemanager.cpp
+++ b/src/filemanager.cpp
@@ -468,6 +468,13 @@ static const char *cache_db_structure_sql =
     " lastModified TEXT,"
     " etag         TEXT"
     ");"
+    "CREATE TABLE downloaded_file_properties("
+    " url          TEXT PRIMARY KEY NOT NULL,"
+    " lastChecked  TIMESTAMP NOT NULL,"
+    " fileSize     INTEGER NOT NULL,"
+    " lastModified TEXT,"
+    " etag         TEXT"
+    ");"
     "CREATE TABLE chunk_data("
     " id        INTEGER PRIMARY KEY AUTOINCREMENT CHECK (id > 0),"
     " data      BLOB NOT NULL"
@@ -1350,10 +1357,6 @@ class NetworkFile : public File {
     NetworkFile(const NetworkFile &) = delete;
     NetworkFile &operator=(const NetworkFile &) = delete;
 
-    static bool get_props_from_headers(PJ_CONTEXT *ctx,
-                                       PROJ_NETWORK_HANDLE *handle,
-                                       FileProperties &props);
-
   protected:
     NetworkFile(PJ_CONTEXT *ctx, const std::string &url,
                 PROJ_NETWORK_HANDLE *handle,
@@ -1373,6 +1376,10 @@ class NetworkFile : public File {
     bool hasChanged() const override { return m_hasChanged; }
 
     static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename);
+
+    static bool get_props_from_headers(PJ_CONTEXT *ctx,
+                                       PROJ_NETWORK_HANDLE *handle,
+                                       FileProperties &props);
 };
 
 // ---------------------------------------------------------------------------
@@ -2297,6 +2304,15 @@ static void CreateDirectory(const std::string &path) {
 std::string pj_context_get_user_writable_directory(PJ_CONTEXT *ctx,
                                                    bool create) {
     if (ctx->user_writable_directory.empty()) {
+        // For testing purposes only
+        const char *env_var_PROJ_USER_WRITABLE_DIRECTORY =
+            getenv("PROJ_USER_WRITABLE_DIRECTORY");
+        if (env_var_PROJ_USER_WRITABLE_DIRECTORY &&
+            env_var_PROJ_USER_WRITABLE_DIRECTORY[0] != '\0') {
+            ctx->user_writable_directory = env_var_PROJ_USER_WRITABLE_DIRECTORY;
+        }
+    }
+    if (ctx->user_writable_directory.empty()) {
         std::string path;
 #ifdef _WIN32
         std::wstring wPath;
@@ -2353,4 +2369,400 @@ std::string pj_context_get_grid_cache_filename(PJ_CONTEXT *ctx) {
     return ctx->gridChunkCache.filename;
 }
 
+// ---------------------------------------------------------------------------
+
+#ifdef WIN32
+static const char dir_chars[] = "/\\";
+#else
+static const char dir_chars[] = "/";
+#endif
+
+static bool is_tilde_slash(const char *name) {
+    return *name == '~' && strchr(dir_chars, name[1]);
+}
+
+static bool is_rel_or_absolute_filename(const char *name) {
+    return strchr(dir_chars, *name) ||
+           (*name == '.' && strchr(dir_chars, name[1])) ||
+           (!strncmp(name, "..", 2) && strchr(dir_chars, name[2])) ||
+           (name[0] != '\0' && name[1] == ':' && strchr(dir_chars, name[2]));
+}
+
+static std::string build_url(PJ_CONTEXT *ctx, const char *name) {
+    if (!is_tilde_slash(name) && !is_rel_or_absolute_filename(name) &&
+        !starts_with(name, "http://") && !starts_with(name, "https://")) {
+        std::string remote_file(pj_context_get_url_endpoint(ctx));
+        if (!remote_file.empty()) {
+            if (remote_file.back() != '/') {
+                remote_file += '/';
+            }
+            remote_file += name;
+            auto pos = remote_file.rfind('.');
+            if (pos + 4 == remote_file.size()) {
+                remote_file = remote_file.substr(0, pos) + ".tif";
+            } else {
+                // For example for resource files like 'alaska'
+                remote_file += ".tif";
+            }
+        }
+        return remote_file;
+    }
+    return name;
+}
+
 //! @endcond
+
+// ---------------------------------------------------------------------------
+
+/** Return if a file must be downloaded or is already available in the
+ * PROJ user-writable directory.
+ *
+ * The file will be determinted to have to be downloaded if it does not exist
+ * yet in the user-writable directory, or if it is determined that a more recent
+ * version exists. To determine if a more recent version exists, PROJ will
+ * use the "downloaded_file_properties" table of its grid cache database.
+ * Consequently files manually placed in the user-writable
+ * directory without using this function would be considered as
+ * non-existing/obsolete and would be unconditionnaly downloaded again.
+ *
+ * This function can only be used if networking is enabled, and either
+ * the default curl network API or a custom one have been installed.
+ *
+ * @param ctx PROJ context, or NULL
+ * @param url_or_filename URL or filename (without directory component)
+ * @param ignore_ttl_setting If set to FALSE, PROJ will only check the
+ *                           recentness of an already downloaded file, if
+ *                           the delay between the last time it has been
+ *                           verified and the current time exceeds the TTL
+ *                           setting. This can save network accesses.
+ *                           If set to TRUE, PROJ will unconditionnally
+ *                           check from the server the recentness of the file.
+ * @return TRUE if the file must be downloaded with proj_download_file()
+ * @since 7.0
+ */
+
+int proj_is_download_needed(PJ_CONTEXT *ctx, const char *url_or_filename,
+                            int ignore_ttl_setting) {
+    if (ctx == nullptr) {
+        ctx = pj_get_default_ctx();
+    }
+    if (!pj_context_is_network_enabled(ctx)) {
+        pj_log(ctx, PJ_LOG_ERROR, "Networking capabilities are not enabled");
+        return false;
+    }
+
+    const auto url(build_url(ctx, url_or_filename));
+    const char *filename = strrchr(url.c_str(), '/');
+    if (filename == nullptr)
+        return false;
+    const auto localFilename(
+        pj_context_get_user_writable_directory(ctx, false) + filename);
+
+    auto f = NS_PROJ::FileManager::open(ctx, localFilename.c_str());
+    if (!f) {
+        return true;
+    }
+    f.reset();
+
+    auto diskCache = NS_PROJ::DiskChunkCache::open(ctx);
+    if (!diskCache)
+        return false;
+    auto stmt =
+        diskCache->prepare("SELECT lastChecked, fileSize, lastModified, etag "
+                           "FROM downloaded_file_properties WHERE url = ?");
+    if (!stmt)
+        return true;
+    stmt->bindText(url.c_str());
+    if (stmt->execute() != SQLITE_ROW) {
+        return true;
+    }
+
+    NS_PROJ::FileProperties cachedProps;
+    cachedProps.lastChecked = stmt->getInt64();
+    cachedProps.size = stmt->getInt64();
+    const char *lastModified = stmt->getText();
+    cachedProps.lastModified = lastModified ? lastModified : std::string();
+    const char *etag = stmt->getText();
+    cachedProps.etag = etag ? etag : std::string();
+
+    if (!ignore_ttl_setting) {
+        const auto ttl = NS_PROJ::pj_context_get_grid_cache_ttl(ctx);
+        if (ttl > 0) {
+            time_t curTime;
+            time(&curTime);
+            if (curTime > cachedProps.lastChecked + ttl) {
+
+                unsigned char dummy;
+                size_t size_read = 0;
+                std::string errorBuffer;
+                errorBuffer.resize(1024);
+                auto handle = ctx->networking.open(
+                    ctx, url.c_str(), 0, 1, &dummy, &size_read,
+                    errorBuffer.size(), &errorBuffer[0],
+                    ctx->networking.user_data);
+                if (!handle) {
+                    errorBuffer.resize(strlen(errorBuffer.data()));
+                    pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", url.c_str(),
+                           errorBuffer.c_str());
+                    return false;
+                }
+                NS_PROJ::FileProperties props;
+                if (!NS_PROJ::NetworkFile::get_props_from_headers(ctx, handle,
+                                                                  props)) {
+                    ctx->networking.close(ctx, handle,
+                                          ctx->networking.user_data);
+                    return false;
+                }
+                ctx->networking.close(ctx, handle, ctx->networking.user_data);
+
+                if (props.size != cachedProps.size ||
+                    props.lastModified != cachedProps.lastModified ||
+                    props.etag != cachedProps.etag) {
+                    return true;
+                }
+
+                stmt = diskCache->prepare(
+                    "UPDATE downloaded_file_properties SET lastChecked = ? "
+                    "WHERE url = ?");
+                if (!stmt)
+                    return false;
+                stmt->bindInt64(curTime);
+                stmt->bindText(url.c_str());
+                if (stmt->execute() != SQLITE_DONE) {
+                    auto hDB = diskCache->handle();
+                    pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+                    return false;
+                }
+            }
+        }
+    }
+
+    return false;
+}
+
+// ---------------------------------------------------------------------------
+
+/** Download a file in the PROJ user-writable directory.
+ *
+ * The file will only be downloaded if it does not exist yet in the
+ * user-writable directory, or if it is determined that a more recent
+ * version exists. To determine if a more recent version exists, PROJ will
+ * use the "downloaded_file_properties" table of its grid cache database.
+ * Consequently files manually placed in the user-writable
+ * directory without using this function would be considered as
+ * non-existing/obsolete and would be unconditionnaly downloaded again.
+ *
+ * This function can only be used if networking is enabled, and either
+ * the default curl network API or a custom one have been installed.
+ *
+ * @param ctx PROJ context, or NULL
+ * @param url_or_filename URL or filename (without directory component)
+ * @param ignore_ttl_setting If set to FALSE, PROJ will only check the
+ *                           recentness of an already downloaded file, if
+ *                           the delay between the last time it has been
+ *                           verified and the current time exceeds the TTL
+ *                           setting. This can save network accesses.
+ *                           If set to TRUE, PROJ will unconditionnally
+ *                           check from the server the recentness of the file.
+ * @param progress_cbk Progress callback, or NULL.
+ *                     The passed percentage is in the [0, 1] range.
+ *                     The progress callback must return TRUE
+ *                     if download must be continued.
+ * @param user_data User data to provide to the progress callback, or NULL
+ * @return TRUE if the download was successful (or not needed)
+ * @since 7.0
+ */
+
+int proj_download_file(PJ_CONTEXT *ctx, const char *url_or_filename,
+                       int ignore_ttl_setting,
+                       int (*progress_cbk)(PJ_CONTEXT *, double pct,
+                                           void *user_data),
+                       void *user_data) {
+    if (ctx == nullptr) {
+        ctx = pj_get_default_ctx();
+    }
+    if (!pj_context_is_network_enabled(ctx)) {
+        pj_log(ctx, PJ_LOG_ERROR, "Networking capabilities are not enabled");
+        return false;
+    }
+    if (!proj_is_download_needed(ctx, url_or_filename, ignore_ttl_setting)) {
+        return true;
+    }
+
+    const auto url(build_url(ctx, url_or_filename));
+    const char *filename = strrchr(url.c_str(), '/');
+    if (filename == nullptr)
+        return false;
+    const auto localFilename(pj_context_get_user_writable_directory(ctx, true) +
+                             filename);
+
+#ifdef _WIN32
+    const int nPID = GetCurrentProcessId();
+#else
+    const int nPID = getpid();
+#endif
+    char szUniqueSuffix[128];
+    snprintf(szUniqueSuffix, sizeof(szUniqueSuffix), "%d_%p", nPID, &url);
+    const auto localFilenameTmp(localFilename + szUniqueSuffix);
+    FILE *f = fopen(localFilenameTmp.c_str(), "wb");
+    if (!f) {
+        pj_log(ctx, PJ_LOG_ERROR, "Cannot create %s", localFilenameTmp.c_str());
+        return false;
+    }
+
+    constexpr size_t FULL_FILE_CHUNK_SIZE = 1024 * 1024;
+    std::vector<unsigned char> buffer(FULL_FILE_CHUNK_SIZE);
+    // For testing purposes only
+    const char *env_var_PROJ_FULL_FILE_CHUNK_SIZE =
+        getenv("PROJ_FULL_FILE_CHUNK_SIZE");
+    if (env_var_PROJ_FULL_FILE_CHUNK_SIZE &&
+        env_var_PROJ_FULL_FILE_CHUNK_SIZE[0] != '\0') {
+        buffer.resize(atoi(env_var_PROJ_FULL_FILE_CHUNK_SIZE));
+    }
+    size_t size_read = 0;
+    std::string errorBuffer;
+    errorBuffer.resize(1024);
+    auto handle = ctx->networking.open(
+        ctx, url.c_str(), 0, buffer.size(), &buffer[0], &size_read,
+        errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data);
+    if (!handle) {
+        errorBuffer.resize(strlen(errorBuffer.data()));
+        pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", url.c_str(),
+               errorBuffer.c_str());
+        fclose(f);
+        unlink(localFilenameTmp.c_str());
+        return false;
+    }
+
+    time_t curTime;
+    time(&curTime);
+    NS_PROJ::FileProperties props;
+    if (!NS_PROJ::NetworkFile::get_props_from_headers(ctx, handle, props)) {
+        ctx->networking.close(ctx, handle, ctx->networking.user_data);
+        fclose(f);
+        unlink(localFilenameTmp.c_str());
+        return false;
+    }
+
+    if (size_read <
+        std::min(static_cast<unsigned long long>(buffer.size()), props.size)) {
+        pj_log(ctx, PJ_LOG_ERROR, "Did not get as many bytes as expected");
+        ctx->networking.close(ctx, handle, ctx->networking.user_data);
+        fclose(f);
+        unlink(localFilenameTmp.c_str());
+        return false;
+    }
+    if (fwrite(buffer.data(), size_read, 1, f) != 1) {
+        pj_log(ctx, PJ_LOG_ERROR, "Write error");
+        ctx->networking.close(ctx, handle, ctx->networking.user_data);
+        fclose(f);
+        unlink(localFilenameTmp.c_str());
+        return false;
+    }
+
+    unsigned long long totalDownloaded = size_read;
+    while (totalDownloaded < props.size) {
+        if (totalDownloaded + buffer.size() > props.size) {
+            buffer.resize(static_cast<size_t>(props.size - totalDownloaded));
+        }
+        errorBuffer.resize(1024);
+        size_read = ctx->networking.read_range(
+            ctx, handle, totalDownloaded, buffer.size(), &buffer[0],
+            errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data);
+
+        if (size_read < buffer.size()) {
+            pj_log(ctx, PJ_LOG_ERROR, "Did not get as many bytes as expected");
+            ctx->networking.close(ctx, handle, ctx->networking.user_data);
+            fclose(f);
+            unlink(localFilenameTmp.c_str());
+            return false;
+        }
+        if (fwrite(buffer.data(), size_read, 1, f) != 1) {
+            pj_log(ctx, PJ_LOG_ERROR, "Write error");
+            ctx->networking.close(ctx, handle, ctx->networking.user_data);
+            fclose(f);
+            unlink(localFilenameTmp.c_str());
+            return false;
+        }
+
+        totalDownloaded += size_read;
+        if (progress_cbk &&
+            !progress_cbk(ctx, double(totalDownloaded) / props.size,
+                          user_data)) {
+            ctx->networking.close(ctx, handle, ctx->networking.user_data);
+            fclose(f);
+            unlink(localFilenameTmp.c_str());
+            return false;
+        }
+    }
+
+    ctx->networking.close(ctx, handle, ctx->networking.user_data);
+    fclose(f);
+
+    unlink(localFilename.c_str());
+    if (rename(localFilenameTmp.c_str(), localFilename.c_str()) != 0) {
+        pj_log(ctx, PJ_LOG_ERROR, "Cannot rename %s to %s",
+               localFilenameTmp.c_str(), localFilename.c_str());
+        return false;
+    }
+
+    auto diskCache = NS_PROJ::DiskChunkCache::open(ctx);
+    if (!diskCache)
+        return false;
+    auto stmt =
+        diskCache->prepare("SELECT lastChecked, fileSize, lastModified, etag "
+                           "FROM downloaded_file_properties WHERE url = ?");
+    if (!stmt)
+        return false;
+    stmt->bindText(url.c_str());
+
+    props.lastChecked = curTime;
+    auto hDB = diskCache->handle();
+
+    if (stmt->execute() == SQLITE_ROW) {
+        stmt = diskCache->prepare(
+            "UPDATE downloaded_file_properties SET lastChecked = ?, "
+            "fileSize = ?, lastModified = ?, etag = ? "
+            "WHERE url = ?");
+        if (!stmt)
+            return false;
+        stmt->bindInt64(props.lastChecked);
+        stmt->bindInt64(props.size);
+        if (props.lastModified.empty())
+            stmt->bindNull();
+        else
+            stmt->bindText(props.lastModified.c_str());
+        if (props.etag.empty())
+            stmt->bindNull();
+        else
+            stmt->bindText(props.etag.c_str());
+        stmt->bindText(url.c_str());
+        if (stmt->execute() != SQLITE_DONE) {
+            pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+            return false;
+        }
+    } else {
+        stmt = diskCache->prepare(
+            "INSERT INTO downloaded_file_properties (url, lastChecked, "
+            "fileSize, lastModified, etag) VALUES "
+            "(?,?,?,?,?)");
+        if (!stmt)
+            return false;
+        stmt->bindText(url.c_str());
+        stmt->bindInt64(props.lastChecked);
+        stmt->bindInt64(props.size);
+        if (props.lastModified.empty())
+            stmt->bindNull();
+        else
+            stmt->bindText(props.lastModified.c_str());
+        if (props.etag.empty())
+            stmt->bindNull();
+        else
+            stmt->bindText(props.etag.c_str());
+        if (stmt->execute() != SQLITE_DONE) {
+            pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB));
+            return false;
+        }
+    }
+    return true;
+}