projsync: make it filter out files not intended for the current version

* Add a PROJ_DATA.VERSION in proj.db to indicate the target PROJ-data
  package version
* Make projsync use that information and the version_added and
  version_removed properties added in https://github.com/OSGeo/PROJ-data/pull/67
  to filter out files that are not relevant
* Add --no-version-filtering and --verbose switches

4 files changed, 127 insertions, 2 deletions

diff --git a/data/sql/metadata.sql b/data/sql/metadata.sql
index 9cce8251..d7cbf026 100644
--- a/data/sql/metadata.sql
+++ b/data/sql/metadata.sql
@@ -15,3 +15,7 @@ INSERT INTO "metadata" VALUES('EPSG.DATE', '2021-05-16');
 -- The value of ${PROJ_VERSION} is substituted at build time by the actual
 -- value.
 INSERT INTO "metadata" VALUES('PROJ.VERSION', '${PROJ_VERSION}');
+
+-- Version of the PROJ-data package with which this database is the most
+-- compatible.
+INSERT INTO "metadata" VALUES('PROJ_DATA.VERSION', '1.7');
diff --git a/docs/source/apps/projsync.rst b/docs/source/apps/projsync.rst
index 247bd2e8..7722e52e 100644
--- a/docs/source/apps/projsync.rst
+++ b/docs/source/apps/projsync.rst
@@ -24,7 +24,8 @@ Synopsis
     |      [--source-id ID] [--area-of-use NAME]
     |      [--file NAME]
     |      [--all] [--exclude-world-coverage]
-    |      [--quiet] [--dry-run] [--list-files]
+    |      [--quiet | --verbose] [--dry-run] [--list-files]
+    |      [--no-version-filtering]
 
 Description
 ***********
@@ -108,6 +109,12 @@ The following control parameters can appear in any order:
 
     Quiet mode
 
+.. option:: --verbose
+
+    .. versionadded:: 8.1
+
+    Verbose mode (more than default)
+
 .. option:: --dry-run
 
     Simulate the behavior of the tool without downloading resource files.
@@ -116,6 +123,16 @@ The following control parameters can appear in any order:
 
     List file names, with the source_id and area_of_use properties.
 
+.. option:: --no-version-filtering
+
+    .. versionadded:: 8.1
+
+    By default, projsync only downloads files that are compatible of
+    the PROJ_DATA.VERSION metadata of :file:`proj.db`, taking into account the
+    ``version_added`` and ``version_removed`` properties of entries in :file:`files.geojson`.
+    When specifying this switch, all files referenced in :file:`files.geojson`
+    will be candidate (combined with other filters).
+
 
 At least one of  :option:`--list-files`,  :option:`--file`,  :option:`--source-id`,
 :option:`--area-of-use`,  :option:`--bbox` or  :option:`--all` must be specified.
diff --git a/src/apps/projsync.cpp b/src/apps/projsync.cpp
index ec4d00fd..b6b4469d 100644
--- a/src/apps/projsync.cpp
+++ b/src/apps/projsync.cpp
@@ -72,7 +72,9 @@ static void usage() {
     std::cerr << "          [--source-id ID] [--area-of-use NAME]" << std::endl;
     std::cerr << "          [--file NAME]" << std::endl;
     std::cerr << "          [--all] [--exclude-world-coverage]" << std::endl;
-    std::cerr << "          [--quiet] [--dry-run] [--list-files]" << std::endl;
+    std::cerr << "          [--quiet | --verbose] [--dry-run] [--list-files]"
+              << std::endl;
+    std::cerr << "          [--no-version-filtering]" << std::endl;
     std::exit(1);
 }
 
@@ -124,10 +126,12 @@ int main(int argc, char *argv[]) {
     double queried_north = 0.0;
     bool intersects = true;
     bool quiet = false;
+    bool verbose = false;
     bool includeWorldCoverage = true;
     bool queryAll = false;
     std::string queriedFilename;
     std::string files_geojson_local;
+    bool versionFiltering = true;
 
     for (int i = 1; i < argc; i++) {
         std::string arg(argv[i]);
@@ -208,8 +212,12 @@ int main(int argc, char *argv[]) {
             includeWorldCoverage = false;
         } else if (arg == "--all") {
             queryAll = true;
+        } else if (arg == "--no-version-filtering") {
+            versionFiltering = false;
         } else if (arg == "-q" || arg == "--quiet") {
             quiet = true;
+        } else if (arg == "--verbose") {
+            verbose = true;
         } else {
             usage();
         }
@@ -281,6 +289,22 @@ int main(int argc, char *argv[]) {
         std::cout << "filename,source_id,area_of_use,file_size" << std::endl;
     }
 
+    std::string proj_data_version_str;
+    int proj_data_version_major = 0;
+    int proj_data_version_minor = 0;
+    {
+        const char *proj_data_version =
+            proj_context_get_database_metadata(ctx, "PROJ_DATA.VERSION");
+        if (proj_data_version) {
+            proj_data_version_str = proj_data_version;
+            const auto tokens = split(proj_data_version, '.');
+            if (tokens.size() >= 2) {
+                proj_data_version_major = atoi(tokens[0].c_str());
+                proj_data_version_minor = atoi(tokens[1].c_str());
+            }
+        }
+    }
+
     try {
         const auto j = json::parse(text);
         bool foundMatchSourceIdCriterion = false;
@@ -315,6 +339,64 @@ int main(int argc, char *argv[]) {
                 continue;
             }
             const auto name(j_name.get<std::string>());
+
+            if (versionFiltering && proj_data_version_major > 0 &&
+                properties.contains("version_added")) {
+                const auto j_version_added = properties["version_added"];
+                if (j_version_added.is_string()) {
+                    const auto version_added(
+                        j_version_added.get<std::string>());
+                    const auto tokens = split(version_added, '.');
+                    if (tokens.size() >= 2) {
+                        int version_major = atoi(tokens[0].c_str());
+                        int version_minor = atoi(tokens[1].c_str());
+                        if (proj_data_version_major < version_major ||
+                            (proj_data_version_major == version_major &&
+                             proj_data_version_minor < version_minor)) {
+                            // File only useful for a later PROJ version
+                            if (verbose) {
+                                std::cout << "Skipping " << name
+                                          << " as it is only useful starting "
+                                             "with PROJ-data "
+                                          << version_added
+                                          << " and we are targetting "
+                                          << proj_data_version_str << std::endl;
+                            }
+                            continue;
+                        }
+                    }
+                }
+            }
+
+            if (versionFiltering &&
+                proj_data_version_major > 0 &&
+                properties.contains("version_removed")) {
+                const auto j_version_removed = properties["version_removed"];
+                if (j_version_removed.is_string()) {
+                    const auto version_removed(
+                        j_version_removed.get<std::string>());
+                    const auto tokens = split(version_removed, '.');
+                    if (tokens.size() >= 2) {
+                        int version_major = atoi(tokens[0].c_str());
+                        int version_minor = atoi(tokens[1].c_str());
+                        if (proj_data_version_major > version_major ||
+                            (proj_data_version_major == version_major &&
+                             proj_data_version_minor >= version_minor)) {
+                            // File only useful for a previous PROJ version
+                            if (verbose) {
+                                std::cout << "Skipping " << name
+                                          << " as it is no longer useful "
+                                             "starting with PROJ-data "
+                                          << version_removed
+                                          << " and we are targetting "
+                                          << proj_data_version_str << std::endl;
+                            }
+                            continue;
+                        }
+                    }
+                }
+            }
+
             files.insert(name);
 
             if (!properties.contains("source_id")) {
diff --git a/test/cli/test_projsync b/test/cli/test_projsync
index 0292ceaa..d2c9bec0 100755
--- a/test/cli/test_projsync
+++ b/test/cli/test_projsync
@@ -188,6 +188,28 @@ fi
 cat ${TMP_OUT} | grep "nz_linz_nzgeoid2009.tif" >/dev/null || (cat ${TMP_OUT}; exit 100)
 cat ${TMP_OUT} | grep "us_noaa_alaska.tif" >/dev/null || (cat ${TMP_OUT}; exit 100)
 
+cat > dummy.geojson <<EOF
+{
+"type": "FeatureCollection",
+"features": [
+{ "type": "Feature", "properties": { "url": "https://example.com/removed_in_1.7.tif", "name": "removed_in_1.7", "area_of_use": "Australia", "type": "VERTICAL_OFFSET_GEOGRAPHIC_TO_VERTICAL", "source_crs_code": "EPSG:7843", "source_crs_name": "GDA2020", "target_crs_code": "EPSG:9458", "target_crs_name": "AVWS height", "source": "Geoscience Australia", "source_country": "Australia", "source_id": "au_ga", "source_url": "http://www.ga.gov.au", "description": "GDA2020 (EPSG:7843) to AVWS height (EPSG:9458)", "file_size": 60878523, "sha256sum": "c06f24dae030587cc2556a316782ade83b6d91f1b8d03922ddd0ce58f3868baa", "version_removed": "1.7" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 93.0083333, -61.0083333 ], [ 174.0083333, -61.0083333 ], [ 174.0083333, -8.0083333 ], [ 93.0083333, -8.0083333 ], [ 93.0083333, -61.0083333 ] ] ] } },
+{ "type": "Feature", "properties": { "url": "https://example.com/added_in_99_99.tif", "name": "added_in_99_99.tif", "area_of_use": "Australia", "type": "VERTICAL_OFFSET_GEOGRAPHIC_TO_VERTICAL", "source_crs_code": "EPSG:7843", "source_crs_name": "GDA2020", "target_crs_code": "EPSG:9458", "target_crs_name": "AVWS height", "source": "Geoscience Australia", "source_country": "Australia", "source_id": "au_ga", "source_url": "http://www.ga.gov.au", "description": "GDA2020 (EPSG:7843) to AVWS height (EPSG:9458)", "file_size": 60878523, "sha256sum": "c06f24dae030587cc2556a316782ade83b6d91f1b8d03922ddd0ce58f3868baa", "version_added": "99.99" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 93.0083333, -61.0083333 ], [ 174.0083333, -61.0083333 ], [ 174.0083333, -8.0083333 ], [ 93.0083333, -8.0083333 ], [ 93.0083333, -61.0083333 ] ] ] } },
+{ "type": "Feature", "properties": { "url": "https://example.com/without_version.tif", "name": "without_version.tif", "area_of_use": "Australia", "type": "VERTICAL_OFFSET_GEOGRAPHIC_TO_VERTICAL", "source_crs_code": "EPSG:7843", "source_crs_name": "GDA2020", "target_crs_code": "EPSG:9458", "target_crs_name": "AVWS height", "source": "Geoscience Australia", "source_country": "Australia", "source_id": "au_ga", "source_url": "http://www.ga.gov.au", "description": "GDA2020 (EPSG:7843) to AVWS height (EPSG:9458)", "file_size": 60878523, "sha256sum": "c06f24dae030587cc2556a316782ade83b6d91f1b8d03922ddd0ce58f3868baa" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 93.0083333, -61.0083333 ], [ 174.0083333, -61.0083333 ], [ 174.0083333, -8.0083333 ], [ 93.0083333, -8.0083333 ], [ 93.0083333, -61.0083333 ] ] ] } }
+]
+}
+EOF
+
+echo "Testing $EXE --source-id au_ga --verbose --dry-run --local-geojson-file dummy.geojson"
+if ! $EXE --source-id au_ga --verbose --dry-run --local-geojson-file dummy.geojson > ${TMP_OUT}; then
+    echo "--source-id au_ga --verbose --dry-run --local-geojson-file dummy.geojson"
+    cat ${TMP_OUT}
+    exit 100
+fi
+rm dummy.geojson
+cat ${TMP_OUT} | grep "Skipping removed_in_1.7 as it is no longer useful starting with PROJ-data 1.7" >/dev/null || (cat ${TMP_OUT}; exit 100)
+cat ${TMP_OUT} | grep "Skipping added_in_99_99.tif as it is only useful starting with PROJ-data 99.99" >/dev/null || (cat ${TMP_OUT}; exit 100)
+cat ${TMP_OUT} | grep "Would download https://cdn.proj.org/without_version.tif" >/dev/null || (cat ${TMP_OUT}; exit 100)
+
 rm -rf ${PROJ_USER_WRITABLE_DIRECTORY}
 rm ${TMP_OUT}