From eafeb61ce59aeb34dabf38f55f70ba9a3b779c4b Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 6 Jan 2020 23:03:29 +0100 Subject: Grid refactoring: address review comments of https://github.com/OSGeo/PROJ/pull/1777 - Move content of legacy apply_gridshift.cpp and apply_vgridshift.cpp in grids.cpp - Rename nad_ functions to pj_hgrid_ - Rename internal proj_hgrid_/proj_vgrid_ functions to pj_ --- src/Makefile.am | 4 +- src/apply_gridshift.cpp | 397 ---------------------------- src/apply_vgridshift.cpp | 217 --------------- src/grids.cpp | 498 ++++++++++++++++++++++++++++++++++- src/grids.hpp | 18 +- src/lib_proj.cmake | 2 - src/transform.cpp | 8 +- src/transformations/deformation.cpp | 10 +- src/transformations/hgridshift.cpp | 10 +- src/transformations/vgridshift.cpp | 10 +- src/transformations/xyzgridshift.cpp | 4 +- 11 files changed, 529 insertions(+), 649 deletions(-) delete mode 100644 src/apply_gridshift.cpp delete mode 100644 src/apply_vgridshift.cpp diff --git a/src/Makefile.am b/src/Makefile.am index d29eb976..39667509 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -196,9 +196,9 @@ libproj_la_SOURCES = \ release.cpp gauss.cpp \ fileapi.cpp \ \ - apply_gridshift.cpp datums.cpp datum_set.cpp transform.cpp \ + datums.cpp datum_set.cpp transform.cpp \ geocent.cpp geocent.h utils.cpp \ - jniproj.cpp mutex.cpp initcache.cpp apply_vgridshift.cpp geodesic.c \ + jniproj.cpp mutex.cpp initcache.cpp geodesic.c \ strtod.cpp \ \ 4D_api.cpp pipeline.cpp \ diff --git a/src/apply_gridshift.cpp b/src/apply_gridshift.cpp deleted file mode 100644 index 4ef86fc0..00000000 --- a/src/apply_gridshift.cpp +++ /dev/null @@ -1,397 +0,0 @@ -/****************************************************************************** - * Project: PROJ.4 - * Purpose: Apply datum shifts based on grid shift files (normally NAD27 to - * NAD83 or the reverse). This module is responsible for keeping - * a list of loaded grids, and calling with each one that is - * allowed for a given datum (expressed as the nadgrids= parameter). - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ****************************************************************************** - * Copyright (c) 2000, Frank Warmerdam - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - *****************************************************************************/ - -#ifndef FROM_PROJ_CPP -#define FROM_PROJ_CPP -#endif - -#include -#include -#include - -#include - -#include "proj.h" -#include "proj_internal.h" -#include "proj/internal/internal.hpp" -#include "grids.hpp" - -NS_PROJ_START - -// --------------------------------------------------------------------------- - -static const HorizontalShiftGrid* findGrid(const ListOfHGrids& grids, - PJ_LP input) -{ - for( const auto& gridset: grids ) - { - auto grid = gridset->gridAt(input.lam, input.phi); - if( grid ) - return grid; - } - return nullptr; -} - -// --------------------------------------------------------------------------- - -static ListOfHGrids getListOfGridSets(PJ_CONTEXT* ctx, const char* grids) -{ - ListOfHGrids list; - auto listOfGrids = internal::split(std::string(grids), ','); - for( const auto& grid: listOfGrids ) - { - const char* gridname = grid.c_str(); - bool canFail = false; - if( gridname[0] == '@' ) - { - canFail = true; - gridname ++; - } - auto gridSet = HorizontalShiftGridSet::open(ctx, gridname); - if( !gridSet ) - { - if( !canFail ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return {}; - } - } - else - { - list.emplace_back(std::move(gridSet)); - } - } - return list; -} - - -/**********************************************/ -ListOfHGrids proj_hgrid_init(PJ* P, const char *gridkey) { -/********************************************** - - Initizalize and populate list of horizontal - grids. - - Takes a PJ-object and the plus-parameter - name that is used in the proj-string to - specify the grids to load, e.g. "+grids". - The + should be left out here. - - Returns the number of loaded grids. - -***********************************************/ - - std::string key("s"); - key += gridkey; - const char* grids = pj_param(P->ctx, P->params, key.c_str()).s; - if( grids == nullptr ) - return {}; - - return getListOfGridSets(P->ctx, grids); -} - -typedef struct { pj_int32 lam, phi; } ILP; - -static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid, bool compensateNTConvention) { - PJ_LP val, frct; - ILP indx; - int in; - - const auto& extent = grid->extentAndRes(); - t.lam /= extent.resLon; - indx.lam = isnan(t.lam) ? 0 : (pj_int32)lround(floor(t.lam)); - t.phi /= extent.resLat; - indx.phi = isnan(t.phi) ? 0 : (pj_int32)lround(floor(t.phi)); - - frct.lam = t.lam - indx.lam; - frct.phi = t.phi - indx.phi; - val.lam = val.phi = HUGE_VAL; - if (indx.lam < 0) { - if (indx.lam == -1 && frct.lam > 0.99999999999) { - ++indx.lam; - frct.lam = 0.; - } else - return val; - } else if ((in = indx.lam + 1) >= grid->width()) { - if (in == grid->width() && frct.lam < 1e-11) { - --indx.lam; - frct.lam = 1.; - } else - return val; - } - if (indx.phi < 0) { - if (indx.phi == -1 && frct.phi > 0.99999999999) { - ++indx.phi; - frct.phi = 0.; - } else - return val; - } else if ((in = indx.phi + 1) >= grid->height()) { - if (in == grid->height() && frct.phi < 1e-11) { - --indx.phi; - frct.phi = 1.; - } else - return val; - } - - float f00Lon = 0, f00Lat = 0; - float f10Lon = 0, f10Lat = 0; - float f01Lon = 0, f01Lat = 0; - float f11Lon = 0, f11Lat = 0; - if( !grid->valueAt(indx.lam, indx.phi, compensateNTConvention, f00Lon, f00Lat) || - !grid->valueAt(indx.lam + 1, indx.phi, compensateNTConvention, f10Lon, f10Lat) || - !grid->valueAt(indx.lam, indx.phi + 1, compensateNTConvention, f01Lon, f01Lat) || - !grid->valueAt(indx.lam + 1, indx.phi + 1, compensateNTConvention, f11Lon, f11Lat) ) - { - return val; - } - - double m10 = frct.lam; - double m11 = m10; - double m01 = 1. - frct.lam; - double m00 = m01; - m11 *= frct.phi; - m01 *= frct.phi; - frct.phi = 1. - frct.phi; - m00 *= frct.phi; - m10 *= frct.phi; - val.lam = m00 * f00Lon + m10 * f10Lon + - m01 * f01Lon + m11 * f11Lon; - val.phi = m00 * f00Lat + m10 * f10Lat + - m01 * f01Lat + m11 * f11Lat; - return val; -} - - -#define MAX_ITERATIONS 10 -#define TOL 1e-12 - -static -PJ_LP nad_cvt(projCtx ctx, PJ_LP in, int inverse, const HorizontalShiftGrid* grid, - const ListOfHGrids& grids) { - PJ_LP t, tb,del, dif; - int i = MAX_ITERATIONS; - const double toltol = TOL*TOL; - - if (in.lam == HUGE_VAL) - return in; - - /* normalize input to ll origin */ - tb = in; - const auto* extent = &(grid->extentAndRes()); - tb.lam -= extent->westLon; - tb.phi -= extent->southLat; - - tb.lam = adjlon (tb.lam - M_PI) + M_PI; - - t = nad_intr (tb, grid, true); - if (t.lam == HUGE_VAL) - return t; - - if (!inverse) { - in.lam += t.lam; - in.phi += t.phi; - return in; - } - - t.lam = tb.lam - t.lam; - t.phi = tb.phi - t.phi; - - do { - del = nad_intr(t, grid, true); - - /* We can possibly go outside of the initial guessed grid, so try */ - /* to fetch a new grid into which iterate... */ - if (del.lam == HUGE_VAL) - { - PJ_LP lp; - lp.lam = t.lam + extent->westLon; - lp.phi = t.phi + extent->southLat; - auto newGrid = findGrid(grids, lp); - if( newGrid == nullptr || newGrid == grid || newGrid->isNullGrid() ) - break; - pj_log(ctx, PJ_LOG_DEBUG_MINOR, "Switching from grid %s to grid %s", - grid->name().c_str(), - newGrid->name().c_str()); - grid = newGrid; - extent = &(grid->extentAndRes()); - t.lam = lp.lam - extent->westLon; - t.phi = lp.phi - extent->southLat; - tb = in; - tb.lam -= extent->westLon; - tb.phi -= extent->southLat; - tb.lam = adjlon (tb.lam - M_PI) + M_PI; - dif.lam = std::numeric_limits::max(); - dif.phi = std::numeric_limits::max(); - continue; - } - - dif.lam = t.lam + del.lam - tb.lam; - dif.phi = t.phi + del.phi - tb.phi; - t.lam -= dif.lam; - t.phi -= dif.phi; - - } while (--i && (dif.lam*dif.lam + dif.phi*dif.phi > toltol)); /* prob. slightly faster than hypot() */ - - if (i==0) { - /* If we had access to a context, this should go through pj_log, and we should set ctx->errno */ - if (getenv ("PROJ_DEBUG")) - fprintf( stderr, "Inverse grid shift iterator failed to converge.\n" ); - t.lam = t.phi = HUGE_VAL; - return t; - } - - /* and again: pj_log and ctx->errno */ - if (del.lam==HUGE_VAL && getenv ("PROJ_DEBUG")) - fprintf (stderr, "Inverse grid shift iteration failed, presumably at grid edge.\nUsing first approximation.\n"); - - in.lam = adjlon (t.lam + extent->westLon); - in.phi = t.phi + extent->southLat; - return in; -} - -/********************************************/ -/* proj_hgrid_value() */ -/* */ -/* Return coordinate offset in grid */ -/********************************************/ -PJ_LP proj_hgrid_value(PJ *P, const ListOfHGrids& grids, PJ_LP lp) { - PJ_LP out = proj_coord_error().lp; - - const auto grid = findGrid(grids, lp); - if( !grid ) { - pj_ctx_set_errno( P->ctx, PJD_ERR_GRID_AREA ); - return out; - } - - /* normalize input to ll origin */ - const auto& extent = grid->extentAndRes(); - lp.lam -= extent.westLon; - lp.phi -= extent.southLat; - - lp.lam = adjlon(lp.lam - M_PI) + M_PI; - - out = nad_intr(lp, grid, false); - - if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) { - pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); - } - - return out; -} - -static -PJ_LP proj_hgrid_apply_internal(PJ_CONTEXT* ctx, - PJ_LP lp, - PJ_DIRECTION direction, - const ListOfHGrids& grids) -{ - PJ_LP out; - - out.lam = HUGE_VAL; - out.phi = HUGE_VAL; - - const auto grid = findGrid(grids, lp); - if( !grid ) { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return out; - } - if( grid->isNullGrid() ) - { - return lp; - } - - int inverse = direction == PJ_FWD ? 0 : 1; - out = nad_cvt(ctx, lp, inverse, grid, grids); - - if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) - pj_ctx_set_errno(ctx, PJD_ERR_GRID_AREA); - - return out; -} - -PJ_LP proj_hgrid_apply(PJ *P, const ListOfHGrids& grids, PJ_LP lp, PJ_DIRECTION direction) { - return proj_hgrid_apply_internal(P->ctx, lp, direction, grids); -} - - -NS_PROJ_END - -/************************************************************************/ -/* pj_apply_gridshift() */ -/* */ -/* This is the externally callable interface - part of the */ -/* public API - though it is not used internally any more and I */ -/* doubt it is used by any other applications. But we preserve */ -/* it to honour our public api. */ -/************************************************************************/ - -int pj_apply_gridshift( projCtx ctx, const char *nadgrids, int inverse, - long point_count, int point_offset, - double *x, double *y, double * /*z */ ) - -{ - auto hgrids = NS_PROJ::getListOfGridSets(ctx, nadgrids); - if( hgrids.empty() ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return 1; - } - - - for( long i = 0; i < point_count; i++ ) - { - PJ_LP input; - - long io = i * point_offset; - input.phi = y[io]; - input.lam = x[io]; - - auto output = proj_hgrid_apply_internal(ctx, input, inverse ? PJ_INV : PJ_FWD, hgrids); - - if ( output.lam != HUGE_VAL ) - { - y[io] = output.phi; - x[io] = output.lam; - } - else - { - if( ctx->debug_level >= PJ_LOG_DEBUG_MAJOR ) - { - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "pj_apply_gridshift(): failed to find a grid shift table for\n" - " location (%.7fdW,%.7fdN)", - x[io] * RAD_TO_DEG, - y[io] * RAD_TO_DEG ); - } - } - } - - return 0; -} diff --git a/src/apply_vgridshift.cpp b/src/apply_vgridshift.cpp deleted file mode 100644 index b0136e5c..00000000 --- a/src/apply_vgridshift.cpp +++ /dev/null @@ -1,217 +0,0 @@ -/****************************************************************************** - * Project: PROJ.4 - * Purpose: Apply vertical datum shifts based on grid shift files, normally - * geoid grids mapping WGS84 to NAVD88 or something similar. - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ****************************************************************************** - * Copyright (c) 2010, Frank Warmerdam - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - *****************************************************************************/ - -#ifndef FROM_PROJ_CPP -#define FROM_PROJ_CPP -#endif - -#include -#include -#include - -#include -#include "proj_internal.h" -#include "proj/internal/internal.hpp" -#include "grids.hpp" - -NS_PROJ_START - -static double read_vgrid_value(const ListOfVGrids& grids, PJ_LP input, double vmultiplier) { - - /* do not deal with NaN coordinates */ - /* cppcheck-suppress duplicateExpression */ - if( isnan(input.phi) || isnan(input.lam) ) - { - return HUGE_VAL; - } - - const VerticalShiftGrid* grid = nullptr; - for( const auto& gridset: grids ) - { - grid = gridset->gridAt(input.lam, input.phi); - if( grid ) - break; - } - if( !grid ) - { - return HUGE_VAL; - } - - const auto& extent = grid->extentAndRes(); - - /* Interpolation a location within the grid */ - double grid_x = (input.lam - extent.westLon) / extent.resLon; - if( extent.fullWorldLongitude() ) { - // The first fmod goes to ]-lim, lim[ range - // So we add lim again to be in ]0, 2*lim[ and fmod again - grid_x = fmod( - fmod(grid_x + grid->width(), grid->width()) + grid->width(), - grid->width()); - } - double grid_y = (input.phi - extent.southLat) / extent.resLat; - int grid_ix = static_cast(lround(floor(grid_x))); - assert(grid_ix >= 0 && grid_ix < grid->width()); - int grid_iy = static_cast(lround(floor(grid_y))); - assert(grid_iy >= 0 && grid_iy < grid->height()); - grid_x -= grid_ix; - grid_y -= grid_iy; - - int grid_ix2 = grid_ix + 1; - if( grid_ix2 >= grid->width() ) { - if( extent.fullWorldLongitude() ) { - grid_ix2 = 0; - } else { - grid_ix2 = grid->width() - 1; - } - } - int grid_iy2 = grid_iy + 1; - if( grid_iy2 >= grid->height() ) - grid_iy2 = grid->height() - 1; - - float value_a = 0; - float value_b = 0; - float value_c = 0; - float value_d = 0; - if( !grid->valueAt(grid_ix, grid_iy, value_a) || - !grid->valueAt(grid_ix2, grid_iy, value_b) || - !grid->valueAt(grid_ix, grid_iy2, value_c) || - !grid->valueAt(grid_ix2, grid_iy2, value_d) ) - { - return HUGE_VAL; - } - - double total_weight = 0.0; - int n_weights = 0; - double value = 0.0f; - - if( !grid->isNodata(value_a, vmultiplier) ) - { - double weight = (1.0-grid_x) * (1.0-grid_y); - value += value_a * weight; - total_weight += weight; - n_weights ++; - } - if( !grid->isNodata(value_b, vmultiplier) ) - { - double weight = (grid_x) * (1.0-grid_y); - value += value_b * weight; - total_weight += weight; - n_weights ++; - } - if( !grid->isNodata(value_c, vmultiplier) ) - { - double weight = (1.0-grid_x) * (grid_y); - value += value_c * weight; - total_weight += weight; - n_weights ++; - } - if( !grid->isNodata(value_d, vmultiplier) ) - { - double weight = (grid_x) * (grid_y); - value += value_d * weight; - total_weight += weight; - n_weights ++; - } - if( n_weights == 0 ) - value = HUGE_VAL; - else if( n_weights != 4 ) - value /= total_weight; - - return value * vmultiplier; -} - -/**********************************************/ -ListOfVGrids proj_vgrid_init(PJ* P, const char *gridkey) { -/********************************************** - - Initizalize and populate gridlist. - - Takes a PJ-object and the plus-parameter - name that is used in the proj-string to - specify the grids to load, e.g. "+grids". - The + should be left out here. - - Returns the number of loaded grids. - -***********************************************/ - - std::string key("s"); - key += gridkey; - const char* gridnames = pj_param(P->ctx, P->params, key.c_str()).s; - if( gridnames == nullptr ) - return {}; - - auto listOfGridNames = internal::split(std::string(gridnames), ','); - ListOfVGrids grids; - for( const auto& gridnameStr: listOfGridNames ) - { - const char* gridname = gridnameStr.c_str(); - bool canFail = false; - if( gridname[0] == '@' ) - { - canFail = true; - gridname ++; - } - auto gridSet = VerticalShiftGridSet::open(P->ctx, gridname); - if( !gridSet ) - { - if( !canFail ) - { - pj_ctx_set_errno( P->ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return {}; - } - } - else - { - grids.emplace_back(std::move(gridSet)); - } - } - - return grids; -} - -/***********************************************/ -double proj_vgrid_value(PJ *P, const ListOfVGrids& grids, PJ_LP lp, double vmultiplier){ -/*********************************************** - - Read grid value at position lp in grids loaded - with proj_grid_init. - - Returns the grid value of the given coordinate. - -************************************************/ - - double value; - - value = read_vgrid_value(grids, lp, vmultiplier); - proj_log_trace(P, "proj_vgrid_value: (%f, %f) = %f", lp.lam*RAD_TO_DEG, lp.phi*RAD_TO_DEG, value); - - return value; -} - -NS_PROJ_END diff --git a/src/grids.cpp b/src/grids.cpp index 5a99106b..66854188 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -2614,7 +2614,7 @@ void GenericShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { // --------------------------------------------------------------------------- -ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *gridkey) { +ListOfGenericGrids pj_generic_grid_init(PJ *P, const char *gridkey) { std::string key("s"); key += gridkey; const char *gridnames = pj_param(P->ctx, P->params, key.c_str()).s; @@ -2644,4 +2644,500 @@ ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *gridkey) { return grids; } +// --------------------------------------------------------------------------- + +static const HorizontalShiftGrid *findGrid(const ListOfHGrids &grids, + PJ_LP input) { + for (const auto &gridset : grids) { + auto grid = gridset->gridAt(input.lam, input.phi); + if (grid) + return grid; + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +static ListOfHGrids getListOfGridSets(PJ_CONTEXT *ctx, const char *grids) { + ListOfHGrids list; + auto listOfGrids = internal::split(std::string(grids), ','); + for (const auto &grid : listOfGrids) { + const char *gridname = grid.c_str(); + bool canFail = false; + if (gridname[0] == '@') { + canFail = true; + gridname++; + } + auto gridSet = HorizontalShiftGridSet::open(ctx, gridname); + if (!gridSet) { + if (!canFail) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return {}; + } + } else { + list.emplace_back(std::move(gridSet)); + } + } + return list; +} + +/**********************************************/ +ListOfHGrids pj_hgrid_init(PJ *P, const char *gridkey) { + /********************************************** + + Initizalize and populate list of horizontal + grids. + + Takes a PJ-object and the plus-parameter + name that is used in the proj-string to + specify the grids to load, e.g. "+grids". + The + should be left out here. + + Returns the number of loaded grids. + + ***********************************************/ + + std::string key("s"); + key += gridkey; + const char *grids = pj_param(P->ctx, P->params, key.c_str()).s; + if (grids == nullptr) + return {}; + + return getListOfGridSets(P->ctx, grids); +} + +// --------------------------------------------------------------------------- + +typedef struct { pj_int32 lam, phi; } ILP; + +static PJ_LP pj_hgrid_interpolate(PJ_LP t, const HorizontalShiftGrid *grid, + bool compensateNTConvention) { + PJ_LP val, frct; + ILP indx; + int in; + + const auto &extent = grid->extentAndRes(); + t.lam /= extent.resLon; + indx.lam = std::isnan(t.lam) ? 0 : (pj_int32)lround(floor(t.lam)); + t.phi /= extent.resLat; + indx.phi = std::isnan(t.phi) ? 0 : (pj_int32)lround(floor(t.phi)); + + frct.lam = t.lam - indx.lam; + frct.phi = t.phi - indx.phi; + val.lam = val.phi = HUGE_VAL; + if (indx.lam < 0) { + if (indx.lam == -1 && frct.lam > 0.99999999999) { + ++indx.lam; + frct.lam = 0.; + } else + return val; + } else if ((in = indx.lam + 1) >= grid->width()) { + if (in == grid->width() && frct.lam < 1e-11) { + --indx.lam; + frct.lam = 1.; + } else + return val; + } + if (indx.phi < 0) { + if (indx.phi == -1 && frct.phi > 0.99999999999) { + ++indx.phi; + frct.phi = 0.; + } else + return val; + } else if ((in = indx.phi + 1) >= grid->height()) { + if (in == grid->height() && frct.phi < 1e-11) { + --indx.phi; + frct.phi = 1.; + } else + return val; + } + + float f00Lon = 0, f00Lat = 0; + float f10Lon = 0, f10Lat = 0; + float f01Lon = 0, f01Lat = 0; + float f11Lon = 0, f11Lat = 0; + if (!grid->valueAt(indx.lam, indx.phi, compensateNTConvention, f00Lon, + f00Lat) || + !grid->valueAt(indx.lam + 1, indx.phi, compensateNTConvention, f10Lon, + f10Lat) || + !grid->valueAt(indx.lam, indx.phi + 1, compensateNTConvention, f01Lon, + f01Lat) || + !grid->valueAt(indx.lam + 1, indx.phi + 1, compensateNTConvention, + f11Lon, f11Lat)) { + return val; + } + + double m10 = frct.lam; + double m11 = m10; + double m01 = 1. - frct.lam; + double m00 = m01; + m11 *= frct.phi; + m01 *= frct.phi; + frct.phi = 1. - frct.phi; + m00 *= frct.phi; + m10 *= frct.phi; + val.lam = m00 * f00Lon + m10 * f10Lon + m01 * f01Lon + m11 * f11Lon; + val.phi = m00 * f00Lat + m10 * f10Lat + m01 * f01Lat + m11 * f11Lat; + return val; +} + +// --------------------------------------------------------------------------- + +#define MAX_ITERATIONS 10 +#define TOL 1e-12 + +static PJ_LP pj_hgrid_apply_internal(projCtx ctx, PJ_LP in, + PJ_DIRECTION direction, + const HorizontalShiftGrid *grid, + const ListOfHGrids &grids) { + PJ_LP t, tb, del, dif; + int i = MAX_ITERATIONS; + const double toltol = TOL * TOL; + + if (in.lam == HUGE_VAL) + return in; + + /* normalize input to ll origin */ + tb = in; + const auto *extent = &(grid->extentAndRes()); + tb.lam -= extent->westLon; + tb.phi -= extent->southLat; + + tb.lam = adjlon(tb.lam - M_PI) + M_PI; + + t = pj_hgrid_interpolate(tb, grid, true); + if (t.lam == HUGE_VAL) + return t; + + if (direction == PJ_FWD) { + in.lam += t.lam; + in.phi += t.phi; + return in; + } + + t.lam = tb.lam - t.lam; + t.phi = tb.phi - t.phi; + + do { + del = pj_hgrid_interpolate(t, grid, true); + + /* We can possibly go outside of the initial guessed grid, so try */ + /* to fetch a new grid into which iterate... */ + if (del.lam == HUGE_VAL) { + PJ_LP lp; + lp.lam = t.lam + extent->westLon; + lp.phi = t.phi + extent->southLat; + auto newGrid = findGrid(grids, lp); + if (newGrid == nullptr || newGrid == grid || newGrid->isNullGrid()) + break; + pj_log(ctx, PJ_LOG_DEBUG_MINOR, "Switching from grid %s to grid %s", + grid->name().c_str(), newGrid->name().c_str()); + grid = newGrid; + extent = &(grid->extentAndRes()); + t.lam = lp.lam - extent->westLon; + t.phi = lp.phi - extent->southLat; + tb = in; + tb.lam -= extent->westLon; + tb.phi -= extent->southLat; + tb.lam = adjlon(tb.lam - M_PI) + M_PI; + dif.lam = std::numeric_limits::max(); + dif.phi = std::numeric_limits::max(); + continue; + } + + dif.lam = t.lam + del.lam - tb.lam; + dif.phi = t.phi + del.phi - tb.phi; + t.lam -= dif.lam; + t.phi -= dif.phi; + + } while (--i && (dif.lam * dif.lam + dif.phi * dif.phi > + toltol)); /* prob. slightly faster than hypot() */ + + if (i == 0) { + /* If we had access to a context, this should go through pj_log, and we + * should set ctx->errno */ + if (getenv("PROJ_DEBUG")) + fprintf(stderr, + "Inverse grid shift iterator failed to converge.\n"); + t.lam = t.phi = HUGE_VAL; + return t; + } + + /* and again: pj_log and ctx->errno */ + if (del.lam == HUGE_VAL && getenv("PROJ_DEBUG")) + fprintf(stderr, "Inverse grid shift iteration failed, presumably at " + "grid edge.\nUsing first approximation.\n"); + + in.lam = adjlon(t.lam + extent->westLon); + in.phi = t.phi + extent->southLat; + return in; +} + +// --------------------------------------------------------------------------- + +PJ_LP pj_hgrid_apply(PJ_CONTEXT *ctx, const ListOfHGrids &grids, PJ_LP lp, + PJ_DIRECTION direction) { + PJ_LP out; + + out.lam = HUGE_VAL; + out.phi = HUGE_VAL; + + const auto grid = findGrid(grids, lp); + if (!grid) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return out; + } + if (grid->isNullGrid()) { + return lp; + } + + out = pj_hgrid_apply_internal(ctx, lp, direction, grid, grids); + + if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) + pj_ctx_set_errno(ctx, PJD_ERR_GRID_AREA); + + return out; +} + +/********************************************/ +/* proj_hgrid_value() */ +/* */ +/* Return coordinate offset in grid */ +/********************************************/ +PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp) { + PJ_LP out = proj_coord_error().lp; + + const auto grid = findGrid(grids, lp); + if (!grid) { + pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); + return out; + } + + /* normalize input to ll origin */ + const auto &extent = grid->extentAndRes(); + lp.lam -= extent.westLon; + lp.phi -= extent.southLat; + + lp.lam = adjlon(lp.lam - M_PI) + M_PI; + + out = pj_hgrid_interpolate(lp, grid, false); + + if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) { + pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); + } + + return out; +} + +// --------------------------------------------------------------------------- + +static double read_vgrid_value(const ListOfVGrids &grids, PJ_LP input, + double vmultiplier) { + + /* do not deal with NaN coordinates */ + /* cppcheck-suppress duplicateExpression */ + if (std::isnan(input.phi) || std::isnan(input.lam)) { + return HUGE_VAL; + } + + const VerticalShiftGrid *grid = nullptr; + for (const auto &gridset : grids) { + grid = gridset->gridAt(input.lam, input.phi); + if (grid) + break; + } + if (!grid) { + return HUGE_VAL; + } + + const auto &extent = grid->extentAndRes(); + + /* Interpolation a location within the grid */ + double grid_x = (input.lam - extent.westLon) / extent.resLon; + if (extent.fullWorldLongitude()) { + // The first fmod goes to ]-lim, lim[ range + // So we add lim again to be in ]0, 2*lim[ and fmod again + grid_x = + fmod(fmod(grid_x + grid->width(), grid->width()) + grid->width(), + grid->width()); + } + double grid_y = (input.phi - extent.southLat) / extent.resLat; + int grid_ix = static_cast(lround(floor(grid_x))); + assert(grid_ix >= 0 && grid_ix < grid->width()); + int grid_iy = static_cast(lround(floor(grid_y))); + assert(grid_iy >= 0 && grid_iy < grid->height()); + grid_x -= grid_ix; + grid_y -= grid_iy; + + int grid_ix2 = grid_ix + 1; + if (grid_ix2 >= grid->width()) { + if (extent.fullWorldLongitude()) { + grid_ix2 = 0; + } else { + grid_ix2 = grid->width() - 1; + } + } + int grid_iy2 = grid_iy + 1; + if (grid_iy2 >= grid->height()) + grid_iy2 = grid->height() - 1; + + float value_a = 0; + float value_b = 0; + float value_c = 0; + float value_d = 0; + if (!grid->valueAt(grid_ix, grid_iy, value_a) || + !grid->valueAt(grid_ix2, grid_iy, value_b) || + !grid->valueAt(grid_ix, grid_iy2, value_c) || + !grid->valueAt(grid_ix2, grid_iy2, value_d)) { + return HUGE_VAL; + } + + double total_weight = 0.0; + int n_weights = 0; + double value = 0.0f; + + if (!grid->isNodata(value_a, vmultiplier)) { + double weight = (1.0 - grid_x) * (1.0 - grid_y); + value += value_a * weight; + total_weight += weight; + n_weights++; + } + if (!grid->isNodata(value_b, vmultiplier)) { + double weight = (grid_x) * (1.0 - grid_y); + value += value_b * weight; + total_weight += weight; + n_weights++; + } + if (!grid->isNodata(value_c, vmultiplier)) { + double weight = (1.0 - grid_x) * (grid_y); + value += value_c * weight; + total_weight += weight; + n_weights++; + } + if (!grid->isNodata(value_d, vmultiplier)) { + double weight = (grid_x) * (grid_y); + value += value_d * weight; + total_weight += weight; + n_weights++; + } + if (n_weights == 0) + value = HUGE_VAL; + else if (n_weights != 4) + value /= total_weight; + + return value * vmultiplier; +} + +/**********************************************/ +ListOfVGrids pj_vgrid_init(PJ *P, const char *gridkey) { + /********************************************** + + Initizalize and populate gridlist. + + Takes a PJ-object and the plus-parameter + name that is used in the proj-string to + specify the grids to load, e.g. "+grids". + The + should be left out here. + + Returns the number of loaded grids. + + ***********************************************/ + + std::string key("s"); + key += gridkey; + const char *gridnames = pj_param(P->ctx, P->params, key.c_str()).s; + if (gridnames == nullptr) + return {}; + + auto listOfGridNames = internal::split(std::string(gridnames), ','); + ListOfVGrids grids; + for (const auto &gridnameStr : listOfGridNames) { + const char *gridname = gridnameStr.c_str(); + bool canFail = false; + if (gridname[0] == '@') { + canFail = true; + gridname++; + } + auto gridSet = VerticalShiftGridSet::open(P->ctx, gridname); + if (!gridSet) { + if (!canFail) { + pj_ctx_set_errno(P->ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return {}; + } + } else { + grids.emplace_back(std::move(gridSet)); + } + } + + return grids; +} + +/***********************************************/ +double pj_vgrid_value(PJ *P, const ListOfVGrids &grids, PJ_LP lp, + double vmultiplier) { + /*********************************************** + + Read grid value at position lp in grids loaded + with proj_grid_init. + + Returns the grid value of the given coordinate. + + ************************************************/ + + double value; + + value = read_vgrid_value(grids, lp, vmultiplier); + proj_log_trace(P, "proj_vgrid_value: (%f, %f) = %f", lp.lam * RAD_TO_DEG, + lp.phi * RAD_TO_DEG, value); + + return value; +} + NS_PROJ_END + +/************************************************************************/ +/* pj_apply_gridshift() */ +/* */ +/* This is the externally callable interface - part of the */ +/* public API - though it is not used internally any more and I */ +/* doubt it is used by any other applications. But we preserve */ +/* it to honour our public api. */ +/************************************************************************/ + +int pj_apply_gridshift(projCtx ctx, const char *nadgrids, int inverse, + long point_count, int point_offset, double *x, double *y, + double * /*z */) + +{ + auto hgrids = NS_PROJ::getListOfGridSets(ctx, nadgrids); + if (hgrids.empty()) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return 1; + } + + for (long i = 0; i < point_count; i++) { + PJ_LP input; + + long io = i * point_offset; + input.phi = y[io]; + input.lam = x[io]; + + auto output = + pj_hgrid_apply(ctx, hgrids, input, inverse ? PJ_INV : PJ_FWD); + + if (output.lam != HUGE_VAL) { + y[io] = output.phi; + x[io] = output.lam; + } else { + if (ctx->debug_level >= PJ_LOG_DEBUG_MAJOR) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "pj_apply_gridshift(): failed to find a grid shift " + "table for\n" + " location (%.7fdW,%.7fdN)", + x[io] * RAD_TO_DEG, y[io] * RAD_TO_DEG); + } + } + } + + return 0; +} diff --git a/src/grids.hpp b/src/grids.hpp index aa852ef6..fbe4e7f8 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -225,15 +225,15 @@ typedef std::vector> ListOfHGrids; typedef std::vector> ListOfVGrids; typedef std::vector> ListOfGenericGrids; -ListOfVGrids proj_vgrid_init(PJ *P, const char *grids); -ListOfHGrids proj_hgrid_init(PJ *P, const char *grids); -ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *grids); - -double proj_vgrid_value(PJ *P, const ListOfVGrids &, PJ_LP lp, - double vmultiplier); -PJ_LP proj_hgrid_value(PJ *P, const ListOfHGrids &, PJ_LP lp); -PJ_LP proj_hgrid_apply(PJ *P, const ListOfHGrids &, PJ_LP lp, - PJ_DIRECTION direction); +ListOfVGrids pj_vgrid_init(PJ *P, const char *grids); +ListOfHGrids pj_hgrid_init(PJ *P, const char *grids); +ListOfGenericGrids pj_generic_grid_init(PJ *P, const char *grids); + +PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp); +double pj_vgrid_value(PJ *P, const ListOfVGrids &, PJ_LP lp, + double vmultiplier); +PJ_LP pj_hgrid_apply(PJ_CONTEXT *ctx, const ListOfHGrids &grids, PJ_LP lp, + PJ_DIRECTION direction); NS_PROJ_END diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index 18c91021..fdb59434 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -220,8 +220,6 @@ set(SRC_LIBPROJ_CORE 4D_api.cpp aasincos.cpp adjlon.cpp - apply_gridshift.cpp - apply_vgridshift.cpp auth.cpp ctx.cpp datum_set.cpp diff --git a/src/transform.cpp b/src/transform.cpp index 020d62ea..811e2a6a 100644 --- a/src/transform.cpp +++ b/src/transform.cpp @@ -451,7 +451,7 @@ static int pj_apply_vgridshift( PJ *defn, if( defn->vgrids_legacy == nullptr ) { defn->vgrids_legacy = new ListOfVGrids; - auto vgrids = proj_vgrid_init(defn, "geoidgrids"); + auto vgrids = pj_vgrid_init(defn, "geoidgrids"); if( vgrids.empty() ) return 0; *static_cast(defn->vgrids_legacy) = std::move(vgrids); @@ -470,7 +470,7 @@ static int pj_apply_vgridshift( PJ *defn, input.phi = y[io]; input.lam = x[io]; - value = proj_vgrid_value(defn, *static_cast(defn->vgrids_legacy), input, 1.0); + value = pj_vgrid_value(defn, *static_cast(defn->vgrids_legacy), input, 1.0); if( inverse ) z[io] -= value; @@ -896,7 +896,7 @@ int pj_apply_gridshift_2( PJ *defn, int inverse, if( defn->hgrids_legacy == nullptr ) { defn->hgrids_legacy = new ListOfHGrids; - auto hgrids = proj_hgrid_init(defn, "nadgrids"); + auto hgrids = pj_hgrid_init(defn, "nadgrids"); if( hgrids.empty() ) return 0; *static_cast(defn->hgrids_legacy) = std::move(hgrids); @@ -914,7 +914,7 @@ int pj_apply_gridshift_2( PJ *defn, int inverse, input.phi = y[io]; input.lam = x[io]; - auto output = proj_hgrid_apply(defn, *static_cast(defn->hgrids_legacy), input, inverse ? PJ_INV : PJ_FWD); + auto output = pj_hgrid_apply(defn->ctx, *static_cast(defn->hgrids_legacy), input, inverse ? PJ_INV : PJ_FWD); if ( output.lam != HUGE_VAL ) { diff --git a/src/transformations/deformation.cpp b/src/transformations/deformation.cpp index eb109826..993647fc 100644 --- a/src/transformations/deformation.cpp +++ b/src/transformations/deformation.cpp @@ -226,8 +226,8 @@ static PJ_XYZ get_grid_shift(PJ* P, const PJ_XYZ& cartesian) { } else { - shift.lp = proj_hgrid_value(P, Q->hgrids, geodetic.lp); - shift.enu.u = proj_vgrid_value(P, Q->vgrids, geodetic.lp, 1.0); + shift.lp = pj_hgrid_value(P, Q->hgrids, geodetic.lp); + shift.enu.u = pj_vgrid_value(P, Q->vgrids, geodetic.lp, 1.0); if (proj_errno(P) == PJD_ERR_GRID_AREA) proj_log_debug(P, "deformation: coordinate (%.3f, %.3f) outside deformation model", @@ -425,7 +425,7 @@ PJ *TRANSFORMATION(deformation,1) { if( has_grids ) { - Q->grids = proj_generic_grid_init(P, "grids"); + Q->grids = pj_generic_grid_init(P, "grids"); /* Was gridlist compiled properly? */ if ( proj_errno(P) ) { proj_log_error(P, "deformation: could not find required grid(s)."); @@ -434,13 +434,13 @@ PJ *TRANSFORMATION(deformation,1) { } else { - Q->hgrids = proj_hgrid_init(P, "xy_grids"); + Q->hgrids = pj_hgrid_init(P, "xy_grids"); if (proj_errno(P)) { proj_log_error(P, "deformation: could not find requested xy_grid(s)."); return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); } - Q->vgrids = proj_vgrid_init(P, "z_grids"); + Q->vgrids = pj_vgrid_init(P, "z_grids"); if (proj_errno(P)) { proj_log_error(P, "deformation: could not find requested z_grid(s)."); return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); diff --git a/src/transformations/hgridshift.cpp b/src/transformations/hgridshift.cpp index 24da4dde..122a7ab2 100644 --- a/src/transformations/hgridshift.cpp +++ b/src/transformations/hgridshift.cpp @@ -28,7 +28,7 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { if ( Q->defer_grid_opening ) { Q->defer_grid_opening = false; - Q->grids = proj_hgrid_init(P, "grids"); + Q->grids = pj_hgrid_init(P, "grids"); if ( proj_errno(P) ) { return proj_coord_error().xyz; } @@ -37,7 +37,7 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ - point.lp = proj_hgrid_apply(P, Q->grids, point.lp, PJ_FWD); + point.lp = pj_hgrid_apply(P->ctx, Q->grids, point.lp, PJ_FWD); } return point.xyz; @@ -51,7 +51,7 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { if ( Q->defer_grid_opening ) { Q->defer_grid_opening = false; - Q->grids = proj_hgrid_init(P, "grids"); + Q->grids = pj_hgrid_init(P, "grids"); if ( proj_errno(P) ) { return proj_coord_error().lpz; } @@ -60,7 +60,7 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ - point.lp = proj_hgrid_apply(P, Q->grids, point.lp, PJ_INV); + point.lp = pj_hgrid_apply(P->ctx, Q->grids, point.lp, PJ_INV); } return point.lpz; @@ -165,7 +165,7 @@ PJ *TRANSFORMATION(hgridshift,0) { Q->defer_grid_opening = true; } else { - Q->grids = proj_hgrid_init(P, "grids"); + Q->grids = pj_hgrid_init(P, "grids"); /* Was gridlist compiled properly? */ if ( proj_errno(P) ) { proj_log_error(P, "hgridshift: could not find required grid(s)."); diff --git a/src/transformations/vgridshift.cpp b/src/transformations/vgridshift.cpp index 3e7a015e..121b795a 100644 --- a/src/transformations/vgridshift.cpp +++ b/src/transformations/vgridshift.cpp @@ -56,7 +56,7 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { if ( Q->defer_grid_opening ) { Q->defer_grid_opening = false; - Q->grids = proj_vgrid_init(P, "grids"); + Q->grids = pj_vgrid_init(P, "grids"); deal_with_vertcon_gtx_hack(P); if ( proj_errno(P) ) { return proj_coord_error().xyz; @@ -66,7 +66,7 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ - point.xyz.z += proj_vgrid_value(P, Q->grids, point.lp, Q->forward_multiplier); + point.xyz.z += pj_vgrid_value(P, Q->grids, point.lp, Q->forward_multiplier); } return point.xyz; @@ -80,7 +80,7 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { if ( Q->defer_grid_opening ) { Q->defer_grid_opening = false; - Q->grids = proj_vgrid_init(P, "grids"); + Q->grids = pj_vgrid_init(P, "grids"); deal_with_vertcon_gtx_hack(P); if ( proj_errno(P) ) { return proj_coord_error().lpz; @@ -90,7 +90,7 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ - point.xyz.z -= proj_vgrid_value(P, Q->grids, point.lp, Q->forward_multiplier); + point.xyz.z -= pj_vgrid_value(P, Q->grids, point.lp, Q->forward_multiplier); } return point.lpz; @@ -193,7 +193,7 @@ PJ *TRANSFORMATION(vgridshift,0) { } else { /* Build gridlist. P->vgridlist_geoid can be empty if +grids only ask for optional grids. */ - Q->grids = proj_vgrid_init(P, "grids"); + Q->grids = pj_vgrid_init(P, "grids"); /* Was gridlist compiled properly? */ if ( proj_errno(P) ) { diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp index a76f3255..3ec3863c 100644 --- a/src/transformations/xyzgridshift.cpp +++ b/src/transformations/xyzgridshift.cpp @@ -77,7 +77,7 @@ static bool get_grid_values(PJ* P, { if ( Q->defer_grid_opening ) { Q->defer_grid_opening = false; - Q->grids = proj_generic_grid_init(P, "grids"); + Q->grids = pj_generic_grid_init(P, "grids"); if ( proj_errno(P) ) { return false; } @@ -341,7 +341,7 @@ PJ *TRANSFORMATION(xyzgridshift,0) { Q->defer_grid_opening = true; } else { - Q->grids = proj_generic_grid_init(P, "grids"); + Q->grids = pj_generic_grid_init(P, "grids"); /* Was gridlist compiled properly? */ if ( proj_errno(P) ) { proj_log_error(P, "xyzgridshift: could not find required grid(s)."); -- cgit v1.2.3 From fc73b2f2ca673b5121da921bebd96c073f7bc592 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 7 Jan 2020 03:32:20 +0100 Subject: DiskChunkCache::closeAndUnlink(): avoid use-after-free in the destructor --- src/filemanager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 68910a94..2d925f07 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -672,6 +672,7 @@ void DiskChunkCache::closeAndUnlink() { if (hDB_) { sqlite3_exec(hDB_, "COMMIT", nullptr, nullptr, nullptr); sqlite3_close(hDB_); + hDB_ = nullptr; } if (vfs_) { vfs_->raw()->xDelete(vfs_->raw(), path_.c_str(), 0); -- cgit v1.2.3 From 237296b7e84a8bb270e3be06a690737a601d73e7 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 7 Jan 2020 03:37:23 +0100 Subject: Remote grid: add mechanism to re-open a grid if it has changed while being opened --- src/filemanager.cpp | 107 ++++++++++----- src/filemanager.hpp | 1 + src/grids.cpp | 256 ++++++++++++++++++++++++++++------- src/grids.hpp | 4 + src/proj.h | 3 + src/sqlite3.cpp | 11 +- src/transformations/deformation.cpp | 48 +++---- src/transformations/xyzgridshift.cpp | 48 +++---- test/unit/test_network.cpp | 255 +++++++++++++++++++++++++++++++++- 9 files changed, 606 insertions(+), 127 deletions(-) diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 2d925f07..1a94216d 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -135,6 +135,9 @@ class FileStdio : public File { unsigned long long tell() override; void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } + // We may lie, but the real use case is only for network files + bool hasChanged() const override { return false; } + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -198,6 +201,9 @@ class FileLegacyAdapter : public File { unsigned long long tell() override; void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } + // We may lie, but the real use case is only for network files + bool hasChanged() const override { return false; } + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -1337,19 +1343,24 @@ class NetworkFile : public File { unsigned long long m_pos = 0; size_t m_nBlocksToDownload = 1; unsigned long long m_lastDownloadedOffset; - unsigned long long m_filesize; + FileProperties m_props; proj_network_close_cbk_type m_closeCbk; + bool m_hasChanged = false; NetworkFile(const NetworkFile &) = delete; NetworkFile &operator=(const NetworkFile &) = delete; + static bool get_props_from_headers(PJ_CONTEXT *ctx, + PROJ_NETWORK_HANDLE *handle, + FileProperties &props); + protected: NetworkFile(PJ_CONTEXT *ctx, const std::string &url, PROJ_NETWORK_HANDLE *handle, unsigned long long lastDownloadOffset, - unsigned long long filesize) + const FileProperties &props) : File(url), m_ctx(ctx), m_url(url), m_handle(handle), - m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize), + m_lastDownloadedOffset(lastDownloadOffset), m_props(props), m_closeCbk(ctx->networking.close) {} public: @@ -1359,18 +1370,47 @@ class NetworkFile : public File { bool seek(unsigned long long offset, int whence) override; unsigned long long tell() override; void reassign_context(PJ_CONTEXT *ctx) override; + bool hasChanged() const override { return m_hasChanged; } static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; // --------------------------------------------------------------------------- +bool NetworkFile::get_props_from_headers(PJ_CONTEXT *ctx, + PROJ_NETWORK_HANDLE *handle, + FileProperties &props) { + const char *contentRange = ctx->networking.get_header_value( + ctx, handle, "Content-Range", ctx->networking.user_data); + if (contentRange) { + const char *slash = strchr(contentRange, '/'); + if (slash) { + props.size = std::stoull(slash + 1); + + const char *lastModified = ctx->networking.get_header_value( + ctx, handle, "Last-Modified", ctx->networking.user_data); + if (lastModified) + props.lastModified = lastModified; + + const char *etag = ctx->networking.get_header_value( + ctx, handle, "ETag", ctx->networking.user_data); + if (etag) + props.etag = etag; + + return true; + } + } + return false; +} + +// --------------------------------------------------------------------------- + std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { FileProperties props; if (gNetworkChunkCache.get(ctx, filename, 0, props)) { return std::unique_ptr(new NetworkFile( ctx, filename, nullptr, - std::numeric_limits::max(), props.size)); + std::numeric_limits::max(), props)); } else { std::vector buffer(DOWNLOAD_CHUNK_SIZE); size_t size_read = 0; @@ -1387,40 +1427,18 @@ std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { errorBuffer.c_str()); } - unsigned long long filesize = 0; + bool ok = false; if (handle) { - const char *contentRange = ctx->networking.get_header_value( - ctx, handle, "Content-Range", ctx->networking.user_data); - if (contentRange) { - const char *slash = strchr(contentRange, '/'); - if (slash) { - filesize = std::stoull(slash + 1); - - props.size = filesize; - - const char *lastModified = ctx->networking.get_header_value( - ctx, handle, "Last-Modified", - ctx->networking.user_data); - if (lastModified) - props.lastModified = lastModified; - - const char *etag = ctx->networking.get_header_value( - ctx, handle, "ETag", ctx->networking.user_data); - if (etag) - props.etag = etag; - - gNetworkFileProperties.insert(ctx, filename, props); - } - } - if (filesize != 0) { + if (get_props_from_headers(ctx, handle, props)) { + ok = true; + gNetworkFileProperties.insert(ctx, filename, props); gNetworkChunkCache.insert(ctx, filename, 0, std::move(buffer)); } } return std::unique_ptr( - handle != nullptr && filesize != 0 - ? new NetworkFile(ctx, filename, handle, size_read, filesize) - : nullptr); + ok ? new NetworkFile(ctx, filename, handle, size_read, props) + : nullptr); } } @@ -1505,6 +1523,20 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) { } return 0; } + + if (!m_hasChanged) { + FileProperties props; + if (get_props_from_headers(m_ctx, m_handle, props)) { + if (props.size != m_props.size || + props.lastModified != m_props.lastModified || + props.etag != m_props.etag) { + gNetworkFileProperties.insert(m_ctx, m_url, props); + gNetworkChunkCache.clearMemoryCache(); + m_hasChanged = true; + } + } + } + region.resize(nRead); m_lastDownloadedOffset = offsetToDownload + nRead; @@ -1547,7 +1579,7 @@ bool NetworkFile::seek(unsigned long long offset, int whence) { } else { if (offset != 0) return false; - m_pos = m_filesize; + m_pos = m_props.size; } return true; } @@ -1871,6 +1903,7 @@ static size_t pj_curl_read_range(PJ_CONTEXT *ctx, auto hCurlHandle = handle->m_handle; double oldDelay = MIN_RETRY_DELAY_MS; + std::string headers; std::string body; char szBuffer[128]; @@ -1880,6 +1913,12 @@ static size_t pj_curl_read_range(PJ_CONTEXT *ctx, while (true) { curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); + headers.clear(); + headers.reserve(16 * 1024); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &headers); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, + pj_curl_write_func); + body.clear(); body.reserve(size_to_read); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); @@ -1930,6 +1969,8 @@ static size_t pj_curl_read_range(PJ_CONTEXT *ctx, if (!body.empty()) { memcpy(buffer, body.data(), std::min(size_to_read, body.size())); } + handle->m_headers = std::move(headers); + return std::min(size_to_read, body.size()); } diff --git a/src/filemanager.hpp b/src/filemanager.hpp index 993048a7..11fb6356 100644 --- a/src/filemanager.hpp +++ b/src/filemanager.hpp @@ -69,6 +69,7 @@ class File { virtual bool seek(unsigned long long offset, int whence = SEEK_SET) = 0; virtual unsigned long long tell() = 0; virtual void reassign_context(PJ_CONTEXT *ctx) = 0; + virtual bool hasChanged() const = 0; const std::string &name() const { return name_; } }; diff --git a/src/grids.cpp b/src/grids.cpp index 66854188..0904a3c2 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -140,6 +140,7 @@ class NullVerticalShiftGrid : public VerticalShiftGrid { bool valueAt(int, int, float &out) const override; bool isNodata(float, double) const override { return false; } void reassign_context(PJ_CONTEXT *) override {} + bool hasChanged() const override { return false; } }; // --------------------------------------------------------------------------- @@ -177,6 +178,8 @@ class GTXVerticalShiftGrid : public VerticalShiftGrid { m_ctx = ctx; m_fp->reassign_context(ctx); } + + bool hasChanged() const override { return m_fp->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -373,6 +376,7 @@ class GTiffGrid : public Grid { PJ_CONTEXT *m_ctx; // owned by the belonging GTiffDataset TIFF *m_hTIFF; // owned by the belonging GTiffDataset BlockCache &m_cache; // owned by the belonging GTiffDataset + File *m_fp; // owned by the belonging GTiffDataset uint32 m_ifdIdx; TIFFDataType m_dt; uint16 m_samplesPerPixel; @@ -402,9 +406,9 @@ class GTiffGrid : public Grid { uint32 offsetInBlock, uint16 sample) const; public: - GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, uint32 ifdIdx, - const std::string &nameIn, int widthIn, int heightIn, - const ExtentAndRes &extentIn, TIFFDataType dtIn, + GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, File *fp, + uint32 ifdIdx, const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn, TIFFDataType dtIn, uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn); ~GTiffGrid() override; @@ -420,17 +424,19 @@ class GTiffGrid : public Grid { uint32 subfileType() const { return m_subfileType; } void reassign_context(PJ_CONTEXT *ctx) { m_ctx = ctx; } + + bool hasChanged() const override { return m_fp->hasChanged(); } }; // --------------------------------------------------------------------------- -GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, +GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, File *fp, uint32 ifdIdx, const std::string &nameIn, int widthIn, int heightIn, const ExtentAndRes &extentIn, TIFFDataType dtIn, uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn) : Grid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), - m_cache(cache), m_ifdIdx(ifdIdx), m_dt(dtIn), + m_cache(cache), m_fp(fp), m_ifdIdx(ifdIdx), m_dt(dtIn), m_samplesPerPixel(samplesPerPixelIn), m_planarConfig(planarConfig), m_bottomUp(bottomUpIn), m_dirOffset(TIFFCurrentDirOffset(hTIFF)), m_tiled(TIFFIsTiled(hTIFF) != 0) { @@ -1048,8 +1054,8 @@ std::unique_ptr GTiffDataset::nextGrid() { } auto ret = std::unique_ptr(new GTiffGrid( - m_ctx, m_hTIFF, m_cache, m_ifdIdx, m_filename, width, height, extent, - dt, samplesPerPixel, planarConfig, vRes < 0)); + m_ctx, m_hTIFF, m_cache, m_fp.get(), m_ifdIdx, m_filename, width, + height, extent, dt, samplesPerPixel, planarConfig, vRes < 0)); m_ifdIdx++; m_hasNextGrid = TIFFReadDirectory(m_hTIFF) != 0; m_nextDirOffset = TIFFCurrentDirOffset(m_hTIFF); @@ -1058,10 +1064,12 @@ std::unique_ptr GTiffDataset::nextGrid() { // --------------------------------------------------------------------------- -class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { +class GTiffVGridShiftSet : public VerticalShiftGridSet { + + std::unique_ptr m_GTiffDataset; GTiffVGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr &&fp) - : GTiffDataset(ctx, std::move(fp)) {} + : m_GTiffDataset(new GTiffDataset(ctx, std::move(fp))) {} public: ~GTiffVGridShiftSet() override; @@ -1072,7 +1080,26 @@ class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { void reassign_context(PJ_CONTEXT *ctx) override { VerticalShiftGridSet::reassign_context(ctx); - GTiffDataset::reassign_context(ctx); + if (m_GTiffDataset) { + m_GTiffDataset->reassign_context(ctx); + } + } + + bool reopen(PJ_CONTEXT *ctx) override { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + m_grids.clear(); + m_GTiffDataset.reset(); + auto fp = FileManager::open_resource_file(ctx, m_name.c_str()); + if (!fp) { + return false; + } + auto newGS = open(ctx, std::move(fp), m_name); + if (newGS) { + m_grids = std::move(newGS->m_grids); + m_GTiffDataset = std::move(newGS->m_GTiffDataset); + } + return !m_grids.empty(); } }; @@ -1172,6 +1199,8 @@ class GTiffVGrid : public VerticalShiftGrid { void reassign_context(PJ_CONTEXT *ctx) override { m_grid->reassign_context(ctx); } + + bool hasChanged() const override { return m_grid->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -1221,14 +1250,14 @@ GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, new GTiffVGridShiftSet(ctx, std::move(fp))); set->m_name = filename; set->m_format = "gtiff"; - if (!set->openTIFF(filename)) { + if (!set->m_GTiffDataset->openTIFF(filename)) { return nullptr; } uint16 idxSample = 0; std::map mapGrids; for (int ifd = 0;; ++ifd) { - auto grid = set->nextGrid(); + auto grid = set->m_GTiffDataset->nextGrid(); if (!grid) { if (ifd == 0) { return nullptr; @@ -1364,6 +1393,19 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { // --------------------------------------------------------------------------- +bool VerticalShiftGridSet::reopen(PJ_CONTEXT *ctx) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + auto newGS = open(ctx, m_name); + m_grids.clear(); + if (newGS) { + m_grids = std::move(newGS->m_grids); + } + return !m_grids.empty(); +} + +// --------------------------------------------------------------------------- + const VerticalShiftGrid *VerticalShiftGrid::gridAt(double lon, double lat) const { for (const auto &child : m_children) { @@ -1435,6 +1477,8 @@ class NullHorizontalShiftGrid : public HorizontalShiftGrid { float &latShift) const override; void reassign_context(PJ_CONTEXT *) override {} + + bool hasChanged() const override { return false; } }; // --------------------------------------------------------------------------- @@ -1482,6 +1526,8 @@ class NTv1Grid : public HorizontalShiftGrid { m_ctx = ctx; m_fp->reassign_context(ctx); } + + bool hasChanged() const override { return m_fp->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -1603,6 +1649,8 @@ class CTable2Grid : public HorizontalShiftGrid { m_ctx = ctx; m_fp->reassign_context(ctx); } + + bool hasChanged() const override { return m_fp->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -1737,6 +1785,8 @@ class NTv2Grid : public HorizontalShiftGrid { m_ctx = ctx; m_fp->reassign_context(ctx); } + + bool hasChanged() const override { return m_fp->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -1906,10 +1956,12 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- -class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { +class GTiffHGridShiftSet : public HorizontalShiftGridSet { + + std::unique_ptr m_GTiffDataset; GTiffHGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr &&fp) - : GTiffDataset(ctx, std::move(fp)) {} + : m_GTiffDataset(new GTiffDataset(ctx, std::move(fp))) {} public: ~GTiffHGridShiftSet() override; @@ -1920,7 +1972,26 @@ class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { void reassign_context(PJ_CONTEXT *ctx) override { HorizontalShiftGridSet::reassign_context(ctx); - GTiffDataset::reassign_context(ctx); + if (m_GTiffDataset) { + m_GTiffDataset->reassign_context(ctx); + } + } + + bool reopen(PJ_CONTEXT *ctx) override { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + m_grids.clear(); + m_GTiffDataset.reset(); + auto fp = FileManager::open_resource_file(ctx, m_name.c_str()); + if (!fp) { + return false; + } + auto newGS = open(ctx, std::move(fp), m_name); + if (newGS) { + m_grids = std::move(newGS->m_grids); + m_GTiffDataset = std::move(newGS->m_GTiffDataset); + } + return !m_grids.empty(); } }; @@ -1954,6 +2025,8 @@ class GTiffHGrid : public HorizontalShiftGrid { void reassign_context(PJ_CONTEXT *ctx) override { m_grid->reassign_context(ctx); } + + bool hasChanged() const override { return m_grid->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -2024,7 +2097,7 @@ GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, new GTiffHGridShiftSet(ctx, std::move(fp))); set->m_name = filename; set->m_format = "gtiff"; - if (!set->openTIFF(filename)) { + if (!set->m_GTiffDataset->openTIFF(filename)) { return nullptr; } @@ -2037,7 +2110,7 @@ GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, std::map mapGrids; for (int ifd = 0;; ++ifd) { - auto grid = set->nextGrid(); + auto grid = set->m_GTiffDataset->nextGrid(); if (!grid) { if (ifd == 0) { return nullptr; @@ -2269,6 +2342,19 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { // --------------------------------------------------------------------------- +bool HorizontalShiftGridSet::reopen(PJ_CONTEXT *ctx) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + auto newGS = open(ctx, m_name); + m_grids.clear(); + if (newGS) { + m_grids = std::move(newGS->m_grids); + } + return !m_grids.empty(); +} + +// --------------------------------------------------------------------------- + const HorizontalShiftGrid *HorizontalShiftGrid::gridAt(double lon, double lat) const { for (const auto &child : m_children) { @@ -2317,11 +2403,12 @@ void HorizontalShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { #ifdef TIFF_ENABLED // --------------------------------------------------------------------------- -class GTiffGenericGridShiftSet : public GenericShiftGridSet, - public GTiffDataset { +class GTiffGenericGridShiftSet : public GenericShiftGridSet { + + std::unique_ptr m_GTiffDataset; GTiffGenericGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr &&fp) - : GTiffDataset(ctx, std::move(fp)) {} + : m_GTiffDataset(new GTiffDataset(ctx, std::move(fp))) {} public: ~GTiffGenericGridShiftSet() override; @@ -2332,7 +2419,26 @@ class GTiffGenericGridShiftSet : public GenericShiftGridSet, void reassign_context(PJ_CONTEXT *ctx) override { GenericShiftGridSet::reassign_context(ctx); - GTiffDataset::reassign_context(ctx); + if (m_GTiffDataset) { + m_GTiffDataset->reassign_context(ctx); + } + } + + bool reopen(PJ_CONTEXT *ctx) override { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + m_grids.clear(); + m_GTiffDataset.reset(); + auto fp = FileManager::open_resource_file(ctx, m_name.c_str()); + if (!fp) { + return false; + } + auto newGS = open(ctx, std::move(fp), m_name); + if (newGS) { + m_grids = std::move(newGS->m_grids); + m_GTiffDataset = std::move(newGS->m_GTiffDataset); + } + return !m_grids.empty(); } }; @@ -2375,6 +2481,8 @@ class GTiffGenericGrid : public GenericShiftGrid { void reassign_context(PJ_CONTEXT *ctx) override { m_grid->reassign_context(ctx); } + + bool hasChanged() const override { return m_grid->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -2446,6 +2554,8 @@ class NullGenericShiftGrid : public GenericShiftGrid { } void reassign_context(PJ_CONTEXT *) override {} + + bool hasChanged() const override { return false; } }; // --------------------------------------------------------------------------- @@ -2466,13 +2576,13 @@ GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, new GTiffGenericGridShiftSet(ctx, std::move(fp))); set->m_name = filename; set->m_format = "gtiff"; - if (!set->openTIFF(filename)) { + if (!set->m_GTiffDataset->openTIFF(filename)) { return nullptr; } std::map mapGrids; for (int ifd = 0;; ++ifd) { - auto grid = set->nextGrid(); + auto grid = set->m_GTiffDataset->nextGrid(); if (!grid) { if (ifd == 0) { return nullptr; @@ -2574,6 +2684,19 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { // --------------------------------------------------------------------------- +bool GenericShiftGridSet::reopen(PJ_CONTEXT *ctx) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + auto newGS = open(ctx, m_name); + m_grids.clear(); + if (newGS) { + m_grids = std::move(newGS->m_grids); + } + return !m_grids.empty(); +} + +// --------------------------------------------------------------------------- + const GenericShiftGrid *GenericShiftGrid::gridAt(double lon, double lat) const { for (const auto &child : m_children) { const auto &extentChild = child->extentAndRes(); @@ -2646,12 +2769,15 @@ ListOfGenericGrids pj_generic_grid_init(PJ *P, const char *gridkey) { // --------------------------------------------------------------------------- -static const HorizontalShiftGrid *findGrid(const ListOfHGrids &grids, - PJ_LP input) { +static const HorizontalShiftGrid * +findGrid(const ListOfHGrids &grids, const PJ_LP& input, + HorizontalShiftGridSet *&gridSetOut) { for (const auto &gridset : grids) { auto grid = gridset->gridAt(input.lam, input.phi); - if (grid) + if (grid) { + gridSetOut = gridset.get(); return grid; + } } return nullptr; } @@ -2789,11 +2915,14 @@ static PJ_LP pj_hgrid_interpolate(PJ_LP t, const HorizontalShiftGrid *grid, static PJ_LP pj_hgrid_apply_internal(projCtx ctx, PJ_LP in, PJ_DIRECTION direction, const HorizontalShiftGrid *grid, - const ListOfHGrids &grids) { + HorizontalShiftGridSet *gridset, + const ListOfHGrids &grids, + bool &shouldRetry) { PJ_LP t, tb, del, dif; int i = MAX_ITERATIONS; const double toltol = TOL * TOL; + shouldRetry = false; if (in.lam == HUGE_VAL) return in; @@ -2806,6 +2935,10 @@ static PJ_LP pj_hgrid_apply_internal(projCtx ctx, PJ_LP in, tb.lam = adjlon(tb.lam - M_PI) + M_PI; t = pj_hgrid_interpolate(tb, grid, true); + if (grid->hasChanged()) { + shouldRetry = gridset->reopen(ctx); + return t; + } if (t.lam == HUGE_VAL) return t; @@ -2820,6 +2953,10 @@ static PJ_LP pj_hgrid_apply_internal(projCtx ctx, PJ_LP in, do { del = pj_hgrid_interpolate(t, grid, true); + if (grid->hasChanged()) { + shouldRetry = gridset->reopen(ctx); + return t; + } /* We can possibly go outside of the initial guessed grid, so try */ /* to fetch a new grid into which iterate... */ @@ -2827,7 +2964,7 @@ static PJ_LP pj_hgrid_apply_internal(projCtx ctx, PJ_LP in, PJ_LP lp; lp.lam = t.lam + extent->westLon; lp.phi = t.phi + extent->southLat; - auto newGrid = findGrid(grids, lp); + auto newGrid = findGrid(grids, lp, gridset); if (newGrid == nullptr || newGrid == grid || newGrid->isNullGrid()) break; pj_log(ctx, PJ_LOG_DEBUG_MINOR, "Switching from grid %s to grid %s", @@ -2882,16 +3019,24 @@ PJ_LP pj_hgrid_apply(PJ_CONTEXT *ctx, const ListOfHGrids &grids, PJ_LP lp, out.lam = HUGE_VAL; out.phi = HUGE_VAL; - const auto grid = findGrid(grids, lp); - if (!grid) { - pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); - return out; - } - if (grid->isNullGrid()) { - return lp; - } + while (true) { + HorizontalShiftGridSet *gridset = nullptr; + const auto grid = findGrid(grids, lp, gridset); + if (!grid) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return out; + } + if (grid->isNullGrid()) { + return lp; + } - out = pj_hgrid_apply_internal(ctx, lp, direction, grid, grids); + bool shouldRetry = false; + out = pj_hgrid_apply_internal(ctx, lp, direction, grid, gridset, grids, + shouldRetry); + if (!shouldRetry) { + break; + } + } if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) pj_ctx_set_errno(ctx, PJD_ERR_GRID_AREA); @@ -2907,7 +3052,8 @@ PJ_LP pj_hgrid_apply(PJ_CONTEXT *ctx, const ListOfHGrids &grids, PJ_LP lp, PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp) { PJ_LP out = proj_coord_error().lp; - const auto grid = findGrid(grids, lp); + HorizontalShiftGridSet *gridset = nullptr; + const auto grid = findGrid(grids, lp, gridset); if (!grid) { pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); return out; @@ -2921,6 +3067,13 @@ PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp) { lp.lam = adjlon(lp.lam - M_PI) + M_PI; out = pj_hgrid_interpolate(lp, grid, false); + if (grid->hasChanged()) { + if (gridset->reopen(P->ctx)) { + return pj_hgrid_value(P, grids, lp); + } + out.lam = HUGE_VAL; + out.phi = HUGE_VAL; + } if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) { pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); @@ -2931,8 +3084,8 @@ PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp) { // --------------------------------------------------------------------------- -static double read_vgrid_value(const ListOfVGrids &grids, PJ_LP input, - double vmultiplier) { +static double read_vgrid_value(PJ_CONTEXT *ctx, const ListOfVGrids &grids, + const PJ_LP& input, const double vmultiplier) { /* do not deal with NaN coordinates */ /* cppcheck-suppress duplicateExpression */ @@ -2940,11 +3093,14 @@ static double read_vgrid_value(const ListOfVGrids &grids, PJ_LP input, return HUGE_VAL; } + VerticalShiftGridSet *curGridset = nullptr; const VerticalShiftGrid *grid = nullptr; for (const auto &gridset : grids) { grid = gridset->gridAt(input.lam, input.phi); - if (grid) + if (grid) { + curGridset = gridset.get(); break; + } } if (!grid) { return HUGE_VAL; @@ -2985,10 +3141,18 @@ static double read_vgrid_value(const ListOfVGrids &grids, PJ_LP input, float value_b = 0; float value_c = 0; float value_d = 0; - if (!grid->valueAt(grid_ix, grid_iy, value_a) || - !grid->valueAt(grid_ix2, grid_iy, value_b) || - !grid->valueAt(grid_ix, grid_iy2, value_c) || - !grid->valueAt(grid_ix2, grid_iy2, value_d)) { + bool error = (!grid->valueAt(grid_ix, grid_iy, value_a) || + !grid->valueAt(grid_ix2, grid_iy, value_b) || + !grid->valueAt(grid_ix, grid_iy2, value_c) || + !grid->valueAt(grid_ix2, grid_iy2, value_d)); + if (grid->hasChanged()) { + if (curGridset->reopen(ctx)) { + return read_vgrid_value(ctx, grids, input, vmultiplier); + } + error = true; + } + + if (error) { return HUGE_VAL; } @@ -3086,7 +3250,7 @@ double pj_vgrid_value(PJ *P, const ListOfVGrids &grids, PJ_LP lp, double value; - value = read_vgrid_value(grids, lp, vmultiplier); + value = read_vgrid_value(P->ctx, grids, lp, vmultiplier); proj_log_trace(P, "proj_vgrid_value: (%f, %f) = %f", lp.lam * RAD_TO_DEG, lp.phi * RAD_TO_DEG, value); diff --git a/src/grids.hpp b/src/grids.hpp index fbe4e7f8..ef365f06 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -70,6 +70,7 @@ class Grid { const std::string &name() const { return m_name; } virtual bool isNullGrid() const { return false; } + virtual bool hasChanged() const = 0; }; // --------------------------------------------------------------------------- @@ -116,6 +117,7 @@ class VerticalShiftGridSet { const VerticalShiftGrid *gridAt(double lon, double lat) const; virtual void reassign_context(PJ_CONTEXT *ctx); + virtual bool reopen(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- @@ -162,6 +164,7 @@ class HorizontalShiftGridSet { const HorizontalShiftGrid *gridAt(double lon, double lat) const; virtual void reassign_context(PJ_CONTEXT *ctx); + virtual bool reopen(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- @@ -217,6 +220,7 @@ class GenericShiftGridSet { const GenericShiftGrid *gridAt(double lon, double lat) const; virtual void reassign_context(PJ_CONTEXT *ctx); + virtual bool reopen(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- diff --git a/src/proj.h b/src/proj.h index 33057617..5c9e81bb 100644 --- a/src/proj.h +++ b/src/proj.h @@ -401,6 +401,9 @@ typedef const char* (*proj_network_get_header_value_cbk_type)( * * Read size_to_read bytes from handle, starting at offset, into * buffer. + * During this read, the implementation should make sure to store the HTTP + * headers from the server response to be able to respond to + * proj_network_get_header_value_cbk_type callback. * * error_string_max_size should be the maximum size that can be written into * the out_error_string buffer (including terminating nul character). diff --git a/src/sqlite3.cpp b/src/sqlite3.cpp index 0c89c0b9..90e73c2a 100644 --- a/src/sqlite3.cpp +++ b/src/sqlite3.cpp @@ -25,8 +25,17 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Weffc++" +#endif + #include "sqlite3.hpp" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + #include #include #include // std::ostringstream @@ -182,4 +191,4 @@ SQLiteStatement::SQLiteStatement(sqlite3_stmt *hStmtIn) : hStmt(hStmtIn) {} // --------------------------------------------------------------------------- -NS_PROJ_END \ No newline at end of file +NS_PROJ_END diff --git a/src/transformations/deformation.cpp b/src/transformations/deformation.cpp index 993647fc..778c1a2e 100644 --- a/src/transformations/deformation.cpp +++ b/src/transformations/deformation.cpp @@ -81,13 +81,16 @@ struct deformationData { // --------------------------------------------------------------------------- static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids, - const PJ_LP& input) + const PJ_LP& input, + GenericShiftGridSet *&gridSetOut) { for( const auto& gridset: grids ) { auto grid = gridset->gridAt(input.lam, input.phi); - if( grid ) + if( grid ) { + gridSetOut = gridset.get(); return grid; + } } return nullptr; } @@ -101,7 +104,8 @@ static bool get_grid_values(PJ* P, double& vy, double& vz) { - auto grid = findGrid(Q->grids, lp); + GenericShiftGridSet* gridset = nullptr; + auto grid = findGrid(Q->grids, lp, gridset); if( !grid ) { return false; } @@ -145,30 +149,28 @@ static bool get_grid_values(PJ* P, int iy2 = std::min(iy + 1, grid->height() - 1); float dx1, dy1, dz1; - if( !grid->valueAt(ix, iy, sampleE, dx1) || - !grid->valueAt(ix, iy, sampleN, dy1) || - !grid->valueAt(ix, iy, sampleU, dz1) ) { - return false; - } - float dx2, dy2, dz2; - if( !grid->valueAt(ix2, iy, sampleE, dx2) || - !grid->valueAt(ix2, iy, sampleN, dy2) || - !grid->valueAt(ix2, iy, sampleU, dz2) ) { - return false; - } - float dx3, dy3, dz3; - if( !grid->valueAt(ix, iy2, sampleE, dx3) || - !grid->valueAt(ix, iy2, sampleN, dy3) || - !grid->valueAt(ix, iy2, sampleU, dz3) ) { - return false; - } - float dx4, dy4, dz4; - if( !grid->valueAt(ix2, iy2, sampleE, dx4) || + bool error =( !grid->valueAt(ix, iy, sampleE, dx1) || + !grid->valueAt(ix, iy, sampleN, dy1) || + !grid->valueAt(ix, iy, sampleU, dz1) || + !grid->valueAt(ix2, iy, sampleE, dx2) || + !grid->valueAt(ix2, iy, sampleN, dy2) || + !grid->valueAt(ix2, iy, sampleU, dz2) || + !grid->valueAt(ix, iy2, sampleE, dx3) || + !grid->valueAt(ix, iy2, sampleN, dy3) || + !grid->valueAt(ix, iy2, sampleU, dz3) || + !grid->valueAt(ix2, iy2, sampleE, dx4) || !grid->valueAt(ix2, iy2, sampleN, dy4) || - !grid->valueAt(ix2, iy2, sampleU, dz4) ) { + !grid->valueAt(ix2, iy2, sampleU, dz4) ); + if( grid->hasChanged() ) { + if( gridset->reopen(P->ctx) ) { + return get_grid_values( P, Q, lp, vx, vy, vz); + } + error = true; + } + if( error ) { return false; } diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp index 3ec3863c..4a85fb4c 100644 --- a/src/transformations/xyzgridshift.cpp +++ b/src/transformations/xyzgridshift.cpp @@ -55,13 +55,16 @@ struct xyzgridshiftData { // --------------------------------------------------------------------------- static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids, - const PJ_LP& input) + const PJ_LP& input, + GenericShiftGridSet *&gridSetOut) { for( const auto& gridset: grids ) { auto grid = gridset->gridAt(input.lam, input.phi); - if( grid ) + if( grid ) { + gridSetOut = gridset.get(); return grid; + } } return nullptr; } @@ -83,7 +86,8 @@ static bool get_grid_values(PJ* P, } } - auto grid = findGrid(Q->grids, lp); + GenericShiftGridSet* gridset = nullptr; + auto grid = findGrid(Q->grids, lp, gridset); if( !grid ) { return false; } @@ -127,30 +131,28 @@ static bool get_grid_values(PJ* P, int iy2 = std::min(iy + 1, grid->height() - 1); float dx1, dy1, dz1; - if( !grid->valueAt(ix, iy, sampleX, dx1) || - !grid->valueAt(ix, iy, sampleY, dy1) || - !grid->valueAt(ix, iy, sampleZ, dz1) ) { - return false; - } - float dx2, dy2, dz2; - if( !grid->valueAt(ix2, iy, sampleX, dx2) || - !grid->valueAt(ix2, iy, sampleY, dy2) || - !grid->valueAt(ix2, iy, sampleZ, dz2) ) { - return false; - } - float dx3, dy3, dz3; - if( !grid->valueAt(ix, iy2, sampleX, dx3) || - !grid->valueAt(ix, iy2, sampleY, dy3) || - !grid->valueAt(ix, iy2, sampleZ, dz3) ) { - return false; - } - float dx4, dy4, dz4; - if( !grid->valueAt(ix2, iy2, sampleX, dx4) || + bool error =( !grid->valueAt(ix, iy, sampleX, dx1) || + !grid->valueAt(ix, iy, sampleY, dy1) || + !grid->valueAt(ix, iy, sampleZ, dz1) || + !grid->valueAt(ix2, iy, sampleX, dx2) || + !grid->valueAt(ix2, iy, sampleY, dy2) || + !grid->valueAt(ix2, iy, sampleZ, dz2) || + !grid->valueAt(ix, iy2, sampleX, dx3) || + !grid->valueAt(ix, iy2, sampleY, dy3) || + !grid->valueAt(ix, iy2, sampleZ, dz3) || + !grid->valueAt(ix2, iy2, sampleX, dx4) || !grid->valueAt(ix2, iy2, sampleY, dy4) || - !grid->valueAt(ix2, iy2, sampleZ, dz4) ) { + !grid->valueAt(ix2, iy2, sampleZ, dz4) ); + if( grid->hasChanged() ) { + if( gridset->reopen(P->ctx) ) { + return get_grid_values( P, Q, lp, dx, dy, dz); + } + error = true; + } + if( error ) { return false; } diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp index 2ec38e41..e6e7bb7b 100644 --- a/test/unit/test_network.cpp +++ b/test/unit/test_network.cpp @@ -473,7 +473,30 @@ TEST(networking, custom) { } exchange.events.emplace_back(std::move(event)); } - + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } { double lon = 2 / 180. * M_PI; double lat = 49 / 180. * M_PI; @@ -500,6 +523,30 @@ TEST(networking, custom) { } exchange.events.emplace_back(std::move(event)); } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } { double lon = 2 / 180. * M_PI; @@ -743,6 +790,30 @@ TEST(networking, simul_read_range_error) { } exchange.events.emplace_back(std::move(event)); } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } { double lon = 2 / 180. * M_PI; @@ -793,6 +864,188 @@ TEST(networking, simul_read_range_error) { // --------------------------------------------------------------------------- +TEST(networking, simul_file_change_while_opened) { + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/file_change_while_opened.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create(ctx, "+proj=vgridshift " + "+grids=https://foo/file_change_while_opened.tif " + "+multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/file_change_while_opened.tif"; + event->offset = 524288; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 1.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date CHANGED!!!!"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/file_change_while_opened.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 3; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 3; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date CHANGED!!!!"; + event->file_id = 3; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 3; + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 3; + exchange.events.emplace_back(std::move(event)); + } + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + #ifdef CURL_ENABLED TEST(networking, curl_hgridshift) { -- cgit v1.2.3 From e8459c135d22759fbd6b5a77d40ae7fe1a029328 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 7 Jan 2020 14:21:29 +0100 Subject: xyzgridshift.rst: fix typo --- docs/source/operations/transformations/xyzgridshift.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/operations/transformations/xyzgridshift.rst b/docs/source/operations/transformations/xyzgridshift.rst index c49b7592..74d62be5 100644 --- a/docs/source/operations/transformations/xyzgridshift.rst +++ b/docs/source/operations/transformations/xyzgridshift.rst @@ -21,7 +21,7 @@ translation values from a grid. The grid is referenced against either the 2D geographic CRS corresponding to the input (or sometimes output) CRS. This method is described (in French) in :cite:`NTF_88` -and as EPSG operation methode code 9655 in :cite:`IOGP2018` (§2.4.4.1.1 +and as EPSG operation method code 9655 in :cite:`IOGP2018` (§2.4.4.1.1 France geocentric interpolation). The translation in the grids are added to the input coordinates in the forward direction, -- cgit v1.2.3 From 13d651a7305853722780b8ac53c84849ae423714 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 7 Jan 2020 14:36:22 +0100 Subject: geotiff_grids.gie: add comment --- test/gie/geotiff_grids.gie | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/gie/geotiff_grids.gie b/test/gie/geotiff_grids.gie index 920fcf28..85c0cbe9 100644 --- a/test/gie/geotiff_grids.gie +++ b/test/gie/geotiff_grids.gie @@ -6,6 +6,10 @@ Test GeoTIFF grids +# Those first tests using +proj=vgridshift only test the capability of reading +# correctly a value from various formulations of GeoTIFF file, hence only the +# forward path is tested (reverse path is tested in other files) + ------------------------------------------------------------------------------- operation +proj=vgridshift +grids=tests/test_vgrid_pixelispoint.tif +multiplier=1 ------------------------------------------------------------------------------- -- cgit v1.2.3 From 32fc01c08f0fdd154c3b515a3c24165afa981ce3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 7 Jan 2020 14:50:44 +0100 Subject: CMake: use more neutral DISABLE_TIFF switch (refs https://github.com/OSGeo/PROJ/pull/1790#discussion_r363750361) --- CMakeLists.txt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2125b93a..0d5524db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -140,11 +140,15 @@ ENDIF() # Check for libtiff ################################################################################ -option(DISABLE_TIFF_IS_STRONGLY_DISCOURAGED "Disable TIFF support (strongly discouraged !)" OFF) -mark_as_advanced(DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) -if(NOT DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) +option(DISABLE_TIFF "Disable TIFF support" OFF) +mark_as_advanced(DISABLE_TIFF) +if(DISABLE_TIFF) + message(WARNING "TIFF support has been disabled and will result in the inability to read some grids") +else() find_package(TIFF REQUIRED) - if(NOT TIFF_FOUND) + if(TIFF_FOUND) + boost_report_value(TIFF_FOUND) + else() message(SEND_ERROR "libtiff dependency not found!") endif() add_definitions(-DTIFF_ENABLED) -- cgit v1.2.3 From 408cb1e4279786e3729e942c2f8b2ee2ad435c94 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 7 Jan 2020 15:01:36 +0100 Subject: deformation.cpp and xyzgridshift.cpp: factor out common code (addresses https://github.com/OSGeo/PROJ/pull/1790#discussion_r363748615) --- src/grids.cpp | 78 +++++++++++++++++++++++++++++++++++- src/grids.hpp | 9 +++++ src/transformations/deformation.cpp | 74 +++++----------------------------- src/transformations/xyzgridshift.cpp | 74 +++++----------------------------- 4 files changed, 107 insertions(+), 128 deletions(-) diff --git a/src/grids.cpp b/src/grids.cpp index 0904a3c2..092b6903 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -2770,7 +2770,7 @@ ListOfGenericGrids pj_generic_grid_init(PJ *P, const char *gridkey) { // --------------------------------------------------------------------------- static const HorizontalShiftGrid * -findGrid(const ListOfHGrids &grids, const PJ_LP& input, +findGrid(const ListOfHGrids &grids, const PJ_LP &input, HorizontalShiftGridSet *&gridSetOut) { for (const auto &gridset : grids) { auto grid = gridset->gridAt(input.lam, input.phi); @@ -3085,7 +3085,7 @@ PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp) { // --------------------------------------------------------------------------- static double read_vgrid_value(PJ_CONTEXT *ctx, const ListOfVGrids &grids, - const PJ_LP& input, const double vmultiplier) { + const PJ_LP &input, const double vmultiplier) { /* do not deal with NaN coordinates */ /* cppcheck-suppress duplicateExpression */ @@ -3257,6 +3257,80 @@ double pj_vgrid_value(PJ *P, const ListOfVGrids &grids, PJ_LP lp, return value; } +// --------------------------------------------------------------------------- + +const GenericShiftGrid *pj_find_generic_grid(const ListOfGenericGrids &grids, + const PJ_LP &input, + GenericShiftGridSet *&gridSetOut) { + for (const auto &gridset : grids) { + auto grid = gridset->gridAt(input.lam, input.phi); + if (grid) { + gridSetOut = gridset.get(); + return grid; + } + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +bool pj_bilinear_interpolation_three_samples(const GenericShiftGrid *grid, + const PJ_LP &lp, int idx1, + int idx2, int idx3, double &v1, + double &v2, double &v3, + bool &must_retry) { + must_retry = false; + + const auto &extent = grid->extentAndRes(); + double grid_x = (lp.lam - extent.westLon) / extent.resLon; + double grid_y = (lp.phi - extent.southLat) / extent.resLat; + int ix = static_cast(grid_x); + int iy = static_cast(grid_y); + int ix2 = std::min(ix + 1, grid->width() - 1); + int iy2 = std::min(iy + 1, grid->height() - 1); + + float dx1, dy1, dz1; + float dx2, dy2, dz2; + float dx3, dy3, dz3; + float dx4, dy4, dz4; + bool error = (!grid->valueAt(ix, iy, idx1, dx1) || + !grid->valueAt(ix, iy, idx2, dy1) || + !grid->valueAt(ix, iy, idx3, dz1) || + !grid->valueAt(ix2, iy, idx1, dx2) || + !grid->valueAt(ix2, iy, idx2, dy2) || + !grid->valueAt(ix2, iy, idx3, dz2) || + !grid->valueAt(ix, iy2, idx1, dx3) || + !grid->valueAt(ix, iy2, idx2, dy3) || + !grid->valueAt(ix, iy2, idx3, dz3) || + !grid->valueAt(ix2, iy2, idx1, dx4) || + !grid->valueAt(ix2, iy2, idx2, dy4) || + !grid->valueAt(ix2, iy2, idx3, dz4)); + if (grid->hasChanged()) { + must_retry = true; + return false; + } + if (error) { + return false; + } + + double frct_lam = grid_x - ix; + double frct_phi = grid_y - iy; + double m10 = frct_lam; + double m11 = m10; + double m01 = 1. - frct_lam; + double m00 = m01; + m11 *= frct_phi; + m01 *= frct_phi; + frct_phi = 1. - frct_phi; + m00 *= frct_phi; + m10 *= frct_phi; + + v1 = m00 * dx1 + m10 * dx2 + m01 * dx3 + m11 * dx4; + v2 = m00 * dy1 + m10 * dy2 + m01 * dy3 + m11 * dy4; + v3 = m00 * dz1 + m10 * dz2 + m01 * dz3 + m11 * dz4; + return true; +} + NS_PROJ_END /************************************************************************/ diff --git a/src/grids.hpp b/src/grids.hpp index ef365f06..6b6ee0d2 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -239,6 +239,15 @@ double pj_vgrid_value(PJ *P, const ListOfVGrids &, PJ_LP lp, PJ_LP pj_hgrid_apply(PJ_CONTEXT *ctx, const ListOfHGrids &grids, PJ_LP lp, PJ_DIRECTION direction); +const GenericShiftGrid *pj_find_generic_grid(const ListOfGenericGrids &grids, + const PJ_LP &input, + GenericShiftGridSet *&gridSetOut); +bool pj_bilinear_interpolation_three_samples(const GenericShiftGrid *grid, + const PJ_LP &lp, int idx1, + int idx2, int idx3, double &v1, + double &v2, double &v3, + bool &must_retry); + NS_PROJ_END #endif // GRIDS_HPP_INCLUDED diff --git a/src/transformations/deformation.cpp b/src/transformations/deformation.cpp index 778c1a2e..8aee50c9 100644 --- a/src/transformations/deformation.cpp +++ b/src/transformations/deformation.cpp @@ -80,23 +80,6 @@ struct deformationData { // --------------------------------------------------------------------------- -static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids, - const PJ_LP& input, - GenericShiftGridSet *&gridSetOut) -{ - for( const auto& gridset: grids ) - { - auto grid = gridset->gridAt(input.lam, input.phi); - if( grid ) { - gridSetOut = gridset.get(); - return grid; - } - } - return nullptr; -} - -// --------------------------------------------------------------------------- - static bool get_grid_values(PJ* P, deformationData* Q, const PJ_LP& lp, @@ -105,7 +88,7 @@ static bool get_grid_values(PJ* P, double& vz) { GenericShiftGridSet* gridset = nullptr; - auto grid = findGrid(Q->grids, lp, gridset); + auto grid = pj_find_generic_grid(Q->grids, lp, gridset); if( !grid ) { return false; } @@ -140,55 +123,20 @@ static bool get_grid_values(PJ* P, return false; } - const auto& extent = grid->extentAndRes(); - double grid_x = (lp.lam - extent.westLon) / extent.resLon; - double grid_y = (lp.phi - extent.southLat) / extent.resLat; - int ix = static_cast(grid_x); - int iy = static_cast(grid_y); - int ix2 = std::min(ix + 1, grid->width() - 1); - int iy2 = std::min(iy + 1, grid->height() - 1); - - float dx1, dy1, dz1; - float dx2, dy2, dz2; - float dx3, dy3, dz3; - float dx4, dy4, dz4; - bool error =( !grid->valueAt(ix, iy, sampleE, dx1) || - !grid->valueAt(ix, iy, sampleN, dy1) || - !grid->valueAt(ix, iy, sampleU, dz1) || - !grid->valueAt(ix2, iy, sampleE, dx2) || - !grid->valueAt(ix2, iy, sampleN, dy2) || - !grid->valueAt(ix2, iy, sampleU, dz2) || - !grid->valueAt(ix, iy2, sampleE, dx3) || - !grid->valueAt(ix, iy2, sampleN, dy3) || - !grid->valueAt(ix, iy2, sampleU, dz3) || - !grid->valueAt(ix2, iy2, sampleE, dx4) || - !grid->valueAt(ix2, iy2, sampleN, dy4) || - !grid->valueAt(ix2, iy2, sampleU, dz4) ); - if( grid->hasChanged() ) { - if( gridset->reopen(P->ctx) ) { + bool must_retry = false; + if( !pj_bilinear_interpolation_three_samples(grid, lp, + sampleE, sampleN, sampleU, + vx, vy, vz, + must_retry) ) + { + if( must_retry ) return get_grid_values( P, Q, lp, vx, vy, vz); - } - error = true; - } - if( error ) { return false; } - - double frct_lam = grid_x - ix; - double frct_phi = grid_y - iy; - double m10 = frct_lam; - double m11 = m10; - double m01 = 1. - frct_lam; - double m00 = m01; - m11 *= frct_phi; - m01 *= frct_phi; - frct_phi = 1. - frct_phi; - m00 *= frct_phi; - m10 *= frct_phi; // divide by 1000 to get m/year - vx = (m00 * dx1 + m10 * dx2 + m01 * dx3 + m11 * dx4) / 1000; - vy = (m00 * dy1 + m10 * dy2 + m01 * dy3 + m11 * dy4) / 1000; - vz = (m00 * dz1 + m10 * dz2 + m01 * dz3 + m11 * dz4) / 1000; + vx /= 1000; + vy /= 1000; + vz /= 1000; return true; } diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp index 4a85fb4c..e1c76518 100644 --- a/src/transformations/xyzgridshift.cpp +++ b/src/transformations/xyzgridshift.cpp @@ -51,24 +51,6 @@ struct xyzgridshiftData { }; } // anonymous namespace - -// --------------------------------------------------------------------------- - -static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids, - const PJ_LP& input, - GenericShiftGridSet *&gridSetOut) -{ - for( const auto& gridset: grids ) - { - auto grid = gridset->gridAt(input.lam, input.phi); - if( grid ) { - gridSetOut = gridset.get(); - return grid; - } - } - return nullptr; -} - // --------------------------------------------------------------------------- static bool get_grid_values(PJ* P, @@ -87,7 +69,7 @@ static bool get_grid_values(PJ* P, } GenericShiftGridSet* gridset = nullptr; - auto grid = findGrid(Q->grids, lp, gridset); + auto grid = pj_find_generic_grid(Q->grids, lp, gridset); if( !grid ) { return false; } @@ -122,54 +104,20 @@ static bool get_grid_values(PJ* P, return false; } - const auto& extent = grid->extentAndRes(); - double grid_x = (lp.lam - extent.westLon) / extent.resLon; - double grid_y = (lp.phi - extent.southLat) / extent.resLat; - int ix = static_cast(grid_x); - int iy = static_cast(grid_y); - int ix2 = std::min(ix + 1, grid->width() - 1); - int iy2 = std::min(iy + 1, grid->height() - 1); - - float dx1, dy1, dz1; - float dx2, dy2, dz2; - float dx3, dy3, dz3; - float dx4, dy4, dz4; - bool error =( !grid->valueAt(ix, iy, sampleX, dx1) || - !grid->valueAt(ix, iy, sampleY, dy1) || - !grid->valueAt(ix, iy, sampleZ, dz1) || - !grid->valueAt(ix2, iy, sampleX, dx2) || - !grid->valueAt(ix2, iy, sampleY, dy2) || - !grid->valueAt(ix2, iy, sampleZ, dz2) || - !grid->valueAt(ix, iy2, sampleX, dx3) || - !grid->valueAt(ix, iy2, sampleY, dy3) || - !grid->valueAt(ix, iy2, sampleZ, dz3) || - !grid->valueAt(ix2, iy2, sampleX, dx4) || - !grid->valueAt(ix2, iy2, sampleY, dy4) || - !grid->valueAt(ix2, iy2, sampleZ, dz4) ); - if( grid->hasChanged() ) { - if( gridset->reopen(P->ctx) ) { + bool must_retry = false; + if( !pj_bilinear_interpolation_three_samples(grid, lp, + sampleX, sampleY, sampleZ, + dx, dy, dz, + must_retry) ) + { + if( must_retry ) return get_grid_values( P, Q, lp, dx, dy, dz); - } - error = true; - } - if( error ) { return false; } - double frct_lam = grid_x - ix; - double frct_phi = grid_y - iy; - double m10 = frct_lam; - double m11 = m10; - double m01 = 1. - frct_lam; - double m00 = m01; - m11 *= frct_phi; - m01 *= frct_phi; - frct_phi = 1. - frct_phi; - m00 *= frct_phi; - m10 *= frct_phi; - dx = (m00 * dx1 + m10 * dx2 + m01 * dx3 + m11 * dx4) * Q->multiplier; - dy = (m00 * dy1 + m10 * dy2 + m01 * dy3 + m11 * dy4) * Q->multiplier; - dz = (m00 * dz1 + m10 * dz2 + m01 * dz3 + m11 * dz4) * Q->multiplier; + dx *= Q->multiplier; + dy *= Q->multiplier; + dz *= Q->multiplier; return true; } -- cgit v1.2.3 From 316ca96fc9e6317247ef30e2926b47ec5544eb98 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 7 Jan 2020 21:52:32 +0100 Subject: network: handle opening remote grids whose local name has no extension --- src/open_lib.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/open_lib.cpp b/src/open_lib.cpp index 23ee79a0..222dd757 100644 --- a/src/open_lib.cpp +++ b/src/open_lib.cpp @@ -381,6 +381,23 @@ std::unique_ptr NS_PROJ::FileManager::open_resource_file( "Using %s", remote_file.c_str() ); pj_ctx_set_errno( ctx, 0 ); } + } else { + // For example for resource files like 'alaska' + auto remote_file_tif = remote_file + ".tif"; + file = open(ctx, remote_file_tif.c_str()); + if( file ) { + pj_log( ctx, PJ_LOG_DEBUG_MAJOR, + "Using %s", remote_file_tif.c_str() ); + pj_ctx_set_errno( ctx, 0 ); + } else { + // Init files + file = open(ctx, remote_file.c_str()); + if( file ) { + pj_log( ctx, PJ_LOG_DEBUG_MAJOR, + "Using %s", remote_file.c_str() ); + pj_ctx_set_errno( ctx, 0 ); + } + } } } } -- cgit v1.2.3 From 8821b3591ea6598e764b1b749749e424a52feaf2 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 7 Jan 2020 22:40:47 +0100 Subject: Doc: address code review comments --- docs/source/operations/transformations/deformation.rst | 2 +- docs/source/operations/transformations/xyzgridshift.rst | 6 +++--- docs/source/references.bib | 2 +- docs/source/resource_files.rst | 6 ++++++ docs/source/usage/environmentvars.rst | 2 +- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/source/operations/transformations/deformation.rst b/docs/source/operations/transformations/deformation.rst index b45c5e2b..aefda3a6 100644 --- a/docs/source/operations/transformations/deformation.rst +++ b/docs/source/operations/transformations/deformation.rst @@ -92,7 +92,7 @@ Parameters .. option:: +xy_grids= - Comma-separated list of grids to load. If a grid is prefixed by an `@` the + Comma-separated list of grids to load. If a grid is prefixed by an ``@`` the grid is considered optional and PROJ will the not complain if the grid is not available. diff --git a/docs/source/operations/transformations/xyzgridshift.rst b/docs/source/operations/transformations/xyzgridshift.rst index 74d62be5..30537032 100644 --- a/docs/source/operations/transformations/xyzgridshift.rst +++ b/docs/source/operations/transformations/xyzgridshift.rst @@ -26,11 +26,11 @@ France geocentric interpolation). The translation in the grids are added to the input coordinates in the forward direction, and subtracted in the reverse direction. -By default (if grid_ref=input_crs), in the forward direction, the input coordinates +By default (if ``grid_ref=input_crs``), in the forward direction, the input coordinates are converted to their geographic equivalent to directly read and interpolate from the grid. In the reverse direction, an iterative method is used to be able to find the grid locations to read. -If grid_ref=output_crs is used, then the reverse strategy is applied: iterative +If ``grid_ref=output_crs`` is used, then the reverse strategy is applied: iterative method in the forward direction, and direct read in the reverse direction. Example @@ -62,7 +62,7 @@ Required .. option:: +grids= - Comma-separated list of grids to load. If a grid is prefixed by an `@` the + Comma-separated list of grids to load. If a grid is prefixed by an ``@`` the grid is considered optional and PROJ will the not complain if the grid is not available. diff --git a/docs/source/references.bib b/docs/source/references.bib index 8e527ea1..912929c3 100644 --- a/docs/source/references.bib +++ b/docs/source/references.bib @@ -231,7 +231,7 @@ } @TechReport{NTF_88, - Title = {Grille de parametres de transformation de coordonnees - GR3DF97A - Notice d'utilisation}, + Title = {Grille de parametres de transformation de coordonnees - {GR3DF97A} - Notice d'utilisation}, Author = {IGN}, Institution = {Service de Geodesie et Nivellement, Institut Geographique National}, Year = {1997}, diff --git a/docs/source/resource_files.rst b/docs/source/resource_files.rst index ea02fd4b..51db94fb 100644 --- a/docs/source/resource_files.rst +++ b/docs/source/resource_files.rst @@ -77,6 +77,12 @@ Its default content is: ; Can be overriden with the PROJ_NETWORK_ENDPOINT environment variable. cdn_endpoint = https://cdn.proj.org + cache_enabled = on + + cache_size_MB = 100 + + cache_ttl_sec = 86400 + Transformation grids ------------------------------------------------------------------------------- diff --git a/docs/source/usage/environmentvars.rst b/docs/source/usage/environmentvars.rst index 24ae4a45..31a74e9a 100644 --- a/docs/source/usage/environmentvars.rst +++ b/docs/source/usage/environmentvars.rst @@ -71,6 +71,6 @@ done by setting the variable with no content:: .. versionadded:: 7.0.0 Define the endpoint of the CDN storage. Normally defined through the proj.ini - configuration file locale in PROJ_LIB. + configuration file locale in :envvar:`PROJ_LIB`. Alternatively, the :c:func:`proj_context_set_url_endpoint` function can be used. -- cgit v1.2.3 From dd9afa39d785b6fcfbcdb5492bb4f3b448b8a183 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 7 Jan 2020 22:48:27 +0100 Subject: pj_bilinear_interpolation_three_samples(): fix MSVC warning about uninitialized variables --- src/grids.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/grids.cpp b/src/grids.cpp index 092b6903..edde66ba 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -3289,10 +3289,10 @@ bool pj_bilinear_interpolation_three_samples(const GenericShiftGrid *grid, int ix2 = std::min(ix + 1, grid->width() - 1); int iy2 = std::min(iy + 1, grid->height() - 1); - float dx1, dy1, dz1; - float dx2, dy2, dz2; - float dx3, dy3, dz3; - float dx4, dy4, dz4; + float dx1 = 0.0f, dy1 = 0.0f, dz1 = 0.0f; + float dx2 = 0.0f, dy2 = 0.0f, dz2 = 0.0f; + float dx3 = 0.0f, dy3 = 0.0f, dz3 = 0.0f; + float dx4 = 0.0f, dy4 = 0.0f, dz4 = 0.0f; bool error = (!grid->valueAt(ix, iy, idx1, dx1) || !grid->valueAt(ix, iy, idx2, dy1) || !grid->valueAt(ix, iy, idx3, dz1) || -- cgit v1.2.3 From da93fe3bea35ae8d2383e6006b7775bb96af6885 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 8 Jan 2020 12:00:51 +0100 Subject: NTv2GridSet::open(): reject files with GS_TYPE!=SECONDS (fixes #1294) --- src/grids.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/grids.cpp b/src/grids.cpp index edde66ba..3007fedc 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -1840,6 +1840,11 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } + if (memcmp(header + 56, "SECONDS", 7) != 0) { + pj_log(ctx, PJ_LOG_ERROR, "Only GS_TYPE=SECONDS is supported"); + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } const bool must_swap = (header[8] == 11) ? !IS_LSB : IS_LSB; if (must_swap) { -- cgit v1.2.3 From 9263e1d36eec53ee3c4e4d04da93a032c0596eec Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 8 Jan 2020 22:32:54 +0100 Subject: Add capability to read resource files from the user writable directory --- cmake/ProjTest.cmake | 8 ++-- docs/source/resource_files.rst | 13 ++++++ scripts/reference_exported_symbols.txt | 1 + src/4D_api.cpp | 11 +++++ src/filemanager.cpp | 77 ++++++++++++++++++++-------------- src/filemanager.hpp | 8 ++-- src/open_lib.cpp | 21 ++++++++++ src/proj_internal.h | 4 +- test/cli/Makefile.am | 12 +++--- test/gie/Makefile.am | 20 ++++----- test/gigs/Makefile.am | 34 +++++++-------- test/unit/CMakeLists.txt | 18 ++++---- test/unit/Makefile.am | 10 ++--- test/unit/gie_self_tests.cpp | 4 ++ test/unit/proj_context_test.cpp | 62 ++++++++++++++++++++------- 15 files changed, 201 insertions(+), 102 deletions(-) diff --git a/cmake/ProjTest.cmake b/cmake/ProjTest.cmake index 6abfbcd7..a016cd9a 100644 --- a/cmake/ProjTest.cmake +++ b/cmake/ProjTest.cmake @@ -28,10 +28,10 @@ function(proj_add_test_script_sh SH_NAME BIN_USE) ) if(MSVC) set_tests_properties( ${testname} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data") + PROPERTIES ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data") else() set_tests_properties( ${testname} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data") + PROPERTIES ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data") endif() endif() @@ -51,10 +51,10 @@ function(proj_add_gie_test TESTNAME TESTCASE) if(MSVC) set_tests_properties( ${TESTNAME} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data") + PROPERTIES ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data") else() set_tests_properties( ${TESTNAME} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data") + PROPERTIES ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data") endif() diff --git a/docs/source/resource_files.rst b/docs/source/resource_files.rst index 51db94fb..3684a948 100644 --- a/docs/source/resource_files.rst +++ b/docs/source/resource_files.rst @@ -23,21 +23,34 @@ The following paths are checked in order: - For transformation grids that have an explict relative or absolute path, the directory specified in the grid filename. + - Path resolved by the callback function set with the :c:func:`proj_context_set_file_finder`. If it is set, the next tests will not be run. + - Path(s) set with the :c:func:`proj_context_set_search_paths`. If set, the next tests will not be run. + +- The PROJ user writable directory, which is : + + * on Windows, ${LOCALAPPDATA}/proj + * on MacOSX, ${HOME}/Library/Logs/proj + * on other platforms (Linux), ${XDG_DATA_HOME}/proj if :envvar:`XDG_DATA_HOME` + is defined. Else ${HOME}/.local/share/proj + - Path(s) set with by the environment variable :envvar:`PROJ_LIB`. On Linux/MacOSX/Unix, use ``:`` to separate paths. On Windows, ``;`` + - On Windows, the *..\\share\\proj\\* and its contents are found automatically at run-time if the installation respects the build structure. That is, the binaries and proj.dll are installed under *..\\bin\\*, and resource files are in *..\\share\\proj\\*. + - A path built into PROJ as its resource installation directory (whose value is $(pkgdatadir)), for builds using the Makefile build system. Note, however, that since this is a hard-wired path setting, it only works if the whole PROJ installation is not moved somewhere else. + - The current directory When networking capabilities are enabled, either by API with the diff --git a/scripts/reference_exported_symbols.txt b/scripts/reference_exported_symbols.txt index 90a1d3cf..a1c03e99 100644 --- a/scripts/reference_exported_symbols.txt +++ b/scripts/reference_exported_symbols.txt @@ -708,6 +708,7 @@ pj_cleanup_lock pj_clear_initcache pj_compare_datums pj_context_get_grid_cache_filename(projCtx_t*) +pj_context_get_user_writable_directory(projCtx_t*, bool) pj_context_is_network_enabled(projCtx_t*) pj_ctx_alloc pj_ctx_fclose diff --git a/src/4D_api.cpp b/src/4D_api.cpp index cee8262e..9107723d 100644 --- a/src/4D_api.cpp +++ b/src/4D_api.cpp @@ -48,6 +48,7 @@ #include #include "geodesic.h" #include "grids.hpp" +#include "filemanager.hpp" #include "proj/common.hpp" #include "proj/coordinateoperation.hpp" @@ -1450,6 +1451,16 @@ PJ_INFO proj_info (void) { /* build search path string */ auto ctx = pj_get_default_ctx(); if (!ctx || ctx->search_paths.empty()) { + // Env var mostly for testing purposes and being independent from + // an existing installation + const char* ignoreUserWritableDirectory = + getenv("PROJ_IGNORE_USER_WRITABLE_DIRECTORY"); + if( ignoreUserWritableDirectory == nullptr || + ignoreUserWritableDirectory[0] == '\0' ) { + buf = path_append(buf, + pj_context_get_user_writable_directory(ctx, false).c_str(), + &buf_size); + } const char *envPROJ_LIB = getenv("PROJ_LIB"); buf = path_append(buf, envPROJ_LIB, &buf_size); #ifdef PROJ_LIB diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 1a94216d..1c49a16b 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -2294,48 +2294,61 @@ static void CreateDirectory(const std::string &path) { // --------------------------------------------------------------------------- -std::string pj_context_get_grid_cache_filename(PJ_CONTEXT *ctx) { - pj_load_ini(ctx); - if (!ctx->gridChunkCache.filename.empty()) { - return ctx->gridChunkCache.filename; - } - std::string path; +std::string pj_context_get_user_writable_directory(PJ_CONTEXT *ctx, + bool create) { + if (ctx->user_writable_directory.empty()) { + std::string path; #ifdef _WIN32 - std::wstring wPath; - wPath.resize(MAX_PATH); - if (SHGetFolderPathW(nullptr, CSIDL_LOCAL_APPDATA, nullptr, 0, &wPath[0]) == - S_OK) { - wPath.resize(wcslen(wPath.data())); - path = WStringToUTF8(wPath); - } else { - const char *local_app_data = getenv("LOCALAPPDATA"); - if (!local_app_data) { - local_app_data = getenv("TEMP"); + std::wstring wPath; + wPath.resize(MAX_PATH); + if (SHGetFolderPathW(nullptr, CSIDL_LOCAL_APPDATA, nullptr, 0, + &wPath[0]) == S_OK) { + wPath.resize(wcslen(wPath.data())); + path = WStringToUTF8(wPath); + } else { + const char *local_app_data = getenv("LOCALAPPDATA"); if (!local_app_data) { - local_app_data = "c:/users"; + local_app_data = getenv("TEMP"); + if (!local_app_data) { + local_app_data = "c:/users"; + } } + path = local_app_data; } - path = local_app_data; - } #else - const char *xdg_data_home = getenv("XDG_DATA_HOME"); - if (xdg_data_home != nullptr) { - path = xdg_data_home; - } else { - const char *home = getenv("HOME"); - if (home) { + const char *xdg_data_home = getenv("XDG_DATA_HOME"); + if (xdg_data_home != nullptr) { + path = xdg_data_home; + } else { + const char *home = getenv("HOME"); + if (home) { #if defined(__MACH__) && defined(__APPLE__) - path = std::string(home) + "/Library/Logs"; + path = std::string(home) + "/Library/Logs"; #else - path = std::string(home) + "/.local/share"; + path = std::string(home) + "/.local/share"; #endif - } else { - path = "/tmp"; + } else { + path = "/tmp"; + } } - } #endif - path += "/proj"; - CreateDirectory(path); + path += "/proj"; + ctx->user_writable_directory = path; + } + if (create) { + CreateDirectory(ctx->user_writable_directory); + } + return ctx->user_writable_directory; +} + +// --------------------------------------------------------------------------- + +std::string pj_context_get_grid_cache_filename(PJ_CONTEXT *ctx) { + pj_load_ini(ctx); + if (!ctx->gridChunkCache.filename.empty()) { + return ctx->gridChunkCache.filename; + } + const std::string path(pj_context_get_user_writable_directory(ctx, true)); ctx->gridChunkCache.filename = path + "/cache.db"; return ctx->gridChunkCache.filename; } diff --git a/src/filemanager.hpp b/src/filemanager.hpp index 11fb6356..9793267c 100644 --- a/src/filemanager.hpp +++ b/src/filemanager.hpp @@ -33,10 +33,10 @@ #include "proj.h" #include "proj/util.hpp" -NS_PROJ_START - //! @cond Doxygen_Suppress +NS_PROJ_START + class File; class FileManager { @@ -74,8 +74,8 @@ class File { const std::string &name() const { return name_; } }; -//! @endcond Doxygen_Suppress - NS_PROJ_END +//! @endcond Doxygen_Suppress + #endif // FILEMANAGER_HPP_INCLUDED \ No newline at end of file diff --git a/src/open_lib.cpp b/src/open_lib.cpp index 222dd757..ae387281 100644 --- a/src/open_lib.cpp +++ b/src/open_lib.cpp @@ -224,6 +224,16 @@ static bool is_rel_or_absolute_filename(const char *name) || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])); } +static bool ignoreUserWritableDirectory() +{ + // Env var mostly for testing purposes and being independent from + // an existing installation + const char* envVarIgnoreUserWritableDirectory = + getenv("PROJ_IGNORE_USER_WRITABLE_DIRECTORY"); + return envVarIgnoreUserWritableDirectory != nullptr && + envVarIgnoreUserWritableDirectory[0] != '\0'; +} + static void* pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, void* (*open_file)(projCtx, const char*, const char*), @@ -279,6 +289,17 @@ pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, break; } } + + else if( !ignoreUserWritableDirectory() && + (fid = open_file(ctx, + (pj_context_get_user_writable_directory(ctx, false) + + DIR_CHAR + name).c_str(), mode)) != nullptr ) { + fname = pj_context_get_user_writable_directory(ctx, false); + fname += DIR_CHAR; + fname += name; + sysname = fname.c_str(); + } + /* if is environment PROJ_LIB defined */ else if ((sysname = getenv("PROJ_LIB")) != nullptr) { auto paths = NS_PROJ::internal::split(std::string(sysname), dirSeparator); diff --git a/src/proj_internal.h b/src/proj_internal.h index 63c53551..ce7b9d74 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -709,6 +709,7 @@ struct projCtx_t { bool iniFileLoaded = false; std::string endpoint{}; + std::string user_writable_directory{}; projGridChunkCache gridChunkCache{}; int projStringParserCreateFromPROJStringRecursionCounter = 0; // to avoid potential infinite recursion in PROJStringParser::createFromPROJString() @@ -844,8 +845,9 @@ std::string pj_context_get_url_endpoint(PJ_CONTEXT* ctx); void pj_load_ini(PJ_CONTEXT* ctx); -// For testing purposes +// Exported for testing purposes only std::string PROJ_DLL pj_context_get_grid_cache_filename(PJ_CONTEXT *ctx); +std::string PROJ_DLL pj_context_get_user_writable_directory(PJ_CONTEXT *ctx, bool create); /* classic public API */ #include "proj_api.h" diff --git a/test/cli/Makefile.am b/test/cli/Makefile.am index 758352c6..253b85d8 100644 --- a/test/cli/Makefile.am +++ b/test/cli/Makefile.am @@ -27,7 +27,7 @@ EXTRA_DIST = pj_out27.dist pj_out83.dist td_out.dist \ CMakeLists.txt testprojinfo-check: - PROJ_LIB=$(PROJ_LIB) $(TESTPROJINFO) $(PROJINFOEXE) + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTPROJINFO) $(PROJINFOEXE) test27-check: $(TEST27) $(PROJEXE) @@ -36,24 +36,24 @@ test83-check: $(TEST83) $(PROJEXE) testvarious-check: - PROJ_LIB=$(PROJ_LIB) $(TESTVARIOUS) $(CS2CSEXE) + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTVARIOUS) $(CS2CSEXE) testdatumfile-check: @if [ -f $(PROJ_LIB)/conus -a -f $(PROJ_LIB)/ntv1_can.dat -a -f $(PROJ_LIB)/MD -a -f $(PROJ_LIB)/ntf_r93.gsb -a -f $(PROJ_LIB)/egm96_15.gtx ]; then \ - PROJ_LIB=$(PROJ_LIB) $(TESTDATUMFILE) $(CS2CSEXE) ; \ + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTDATUMFILE) $(CS2CSEXE) ; \ fi testign-check: @if [ -f $(PROJ_LIB)/ntf_r93.gsb ] ; then \ - PROJ_LIB=$(PROJ_LIB) $(TESTIGN) $(CS2CSEXE) ; \ + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTIGN) $(CS2CSEXE) ; \ fi testntv2-check: @if [ -f $(PROJ_LIB)/ntv2_0.gsb -a -f $(PROJ_LIB)/conus -a -f $(PROJ_LIB)/ntv1_can.dat ] ; then \ - PROJ_LIB=$(PROJ_LIB) $(TESTNTV2) $(CS2CSEXE) ; \ + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTNTV2) $(CS2CSEXE) ; \ fi testcct-check: - PROJ_LIB=$(PROJ_LIB) $(TESTCCT) $(CCTEXE) + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTCCT) $(CCTEXE) check-local: testprojinfo-check test27-check test83-check testvarious-check testdatumfile-check testign-check testntv2-check testcct-check diff --git a/test/gie/Makefile.am b/test/gie/Makefile.am index ff333a14..1539cb2e 100644 --- a/test/gie/Makefile.am +++ b/test/gie/Makefile.am @@ -15,33 +15,33 @@ EXTRA_DIST = 4D-API_cs2cs-style.gie \ PROJ_LIB ?= ../../data 4D-API-cs2cs-style: 4D-API_cs2cs-style.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< GDA: GDA.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< axisswap: axisswap.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< builtins: builtins.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< deformation: deformation.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< ellipsoid: ellipsoid.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< more_builtins: more_builtins.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< unitconvert: unitconvert.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< DHDN_ETRS89: DHDN_ETRS89.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< geotiff_grids: geotiff_grids.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< check-local: 4D-API-cs2cs-style GDA axisswap builtins deformation ellipsoid more_builtins unitconvert DHDN_ETRS89 geotiff_grids diff --git a/test/gigs/Makefile.am b/test/gigs/Makefile.am index bea3be59..56493550 100644 --- a/test/gigs/Makefile.am +++ b/test/gigs/Makefile.am @@ -23,54 +23,54 @@ EXTRA_DIST = \ PROJ_LIB ?= ../../data 5101.1: 5101.1-jhs.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5101.2: 5101.2-jhs.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5101.3: 5101.3-jhs.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5101.4: 5101.4-jhs-etmerc.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5102.1: 5102.1.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5103.1: 5103.1.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5103.2: 5103.2.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5103.3: 5103.3.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5105.2: 5105.2.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5106: 5106.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5107: 5107.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5109: 5109.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5111.1: 5111.1.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5112: 5112.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5113: 5113.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5201: 5201.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5208: 5208.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< check-local: 5101.1 5101.2 5101.3 5101.4 5102.1 5103.1 5103.2 5103.3 5105.2 5106 5107 5109 5111.1 5112 5113 5201 5208 diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 35a6e4c1..e1eefcf0 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -71,7 +71,7 @@ target_link_libraries(proj_pj_transform_test ${PROJ_LIBRARIES}) add_test(NAME proj_pj_transform_test COMMAND proj_pj_transform_test) set_property(TEST proj_pj_transform_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") add_executable(proj_errno_string_test @@ -82,7 +82,7 @@ target_link_libraries(proj_errno_string_test ${PROJ_LIBRARIES}) add_test(NAME proj_errno_string_test COMMAND proj_errno_string_test) set_property(TEST proj_errno_string_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") add_executable(proj_angular_io_test main.cpp @@ -92,7 +92,7 @@ target_link_libraries(proj_angular_io_test ${PROJ_LIBRARIES}) add_test(NAME proj_angular_io_test COMMAND proj_angular_io_test) set_property(TEST proj_angular_io_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") add_executable(proj_context_test main.cpp @@ -102,7 +102,7 @@ target_link_libraries(proj_context_test ${PROJ_LIBRARIES}) add_test(NAME proj_context_test COMMAND proj_context_test) set_property(TEST proj_context_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") if(MSVC AND BUILD_LIBPROJ_SHARED) # ph_phi2_test not compatible of a .dll build @@ -115,7 +115,7 @@ else() ${PROJ_LIBRARIES}) add_test(NAME pj_phi2_test COMMAND pj_phi2_test) set_property(TEST pj_phi2_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") endif() add_executable(proj_test_cpp_api @@ -135,7 +135,7 @@ target_link_libraries(proj_test_cpp_api ${SQLITE3_LIBRARY}) add_test(NAME proj_test_cpp_api COMMAND proj_test_cpp_api) set_property(TEST proj_test_cpp_api - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") add_executable(gie_self_tests @@ -146,7 +146,7 @@ target_link_libraries(gie_self_tests ${PROJ_LIBRARIES}) add_test(NAME gie_self_tests COMMAND gie_self_tests) set_property(TEST gie_self_tests - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") add_executable(test_network @@ -163,8 +163,8 @@ target_link_libraries(test_network add_test(NAME test_network COMMAND test_network) if(MSVC) set_property(TEST test_network - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") else() set_property(TEST test_network - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") endif() diff --git a/test/unit/Makefile.am b/test/unit/Makefile.am index 422fe687..7ffb06ae 100644 --- a/test/unit/Makefile.am +++ b/test/unit/Makefile.am @@ -23,7 +23,7 @@ pj_transform_test_SOURCES = pj_transform_test.cpp main.cpp pj_transform_test_LDADD = ../../src/libproj.la @GTEST_LIBS@ pj_transform_test-check: pj_transform_test - PROJ_LIB=$(PROJ_LIB) ./pj_transform_test + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) ./pj_transform_test pj_phi2_test_SOURCES = pj_phi2_test.cpp main.cpp pj_phi2_test_LDADD = ../../src/libproj.la @GTEST_LIBS@ @@ -47,19 +47,19 @@ proj_context_test_SOURCES = proj_context_test.cpp main.cpp proj_context_test_LDADD = ../../src/libproj.la @GTEST_LIBS@ proj_context_test-check: proj_context_test - ./proj_context_test + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES ./proj_context_test test_cpp_api_SOURCES = test_util.cpp test_common.cpp test_crs.cpp test_metadata.cpp test_io.cpp test_operation.cpp test_datum.cpp test_factory.cpp test_c_api.cpp main.cpp test_cpp_api_LDADD = ../../src/libproj.la @GTEST_LIBS@ @SQLITE3_LIBS@ test_cpp_api-check: test_cpp_api - PROJ_LIB=$(PROJ_LIB) ./test_cpp_api + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) ./test_cpp_api gie_self_tests_SOURCES = gie_self_tests.cpp main.cpp gie_self_tests_LDADD = ../../src/libproj.la @GTEST_LIBS@ @SQLITE3_LIBS@ gie_self_tests-check: gie_self_tests - PROJ_LIB=$(PROJ_LIB) ./gie_self_tests + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) ./gie_self_tests include_proj_h_from_c_SOURCES = include_proj_h_from_c.c @@ -68,6 +68,6 @@ test_network_CXXFLAGS = @CURL_CFLAGS@ @CURL_ENABLED_FLAGS@ test_network_LDADD = ../../src/libproj.la @GTEST_LIBS@ @SQLITE3_LIBS@ @CURL_LIBS@ test_network-check: test_network - PROJ_LIB=$(PROJ_LIB) PROJ_SOURCE_DATA=$(PROJ_LIB) ./test_network + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) PROJ_SOURCE_DATA=$(PROJ_LIB) ./test_network check-local: pj_transform_test-check pj_phi2_test-check proj_errno_string_test-check proj_angular_io_test-check proj_context_test-check test_cpp_api-check gie_self_tests-check test_network-check diff --git a/test/unit/gie_self_tests.cpp b/test/unit/gie_self_tests.cpp index a738db75..a3b41fb0 100644 --- a/test/unit/gie_self_tests.cpp +++ b/test/unit/gie_self_tests.cpp @@ -348,7 +348,9 @@ TEST(gie, info_functions) { /* proj_info() */ /* this one is difficult to test, since the output changes with the setup */ + putenv(const_cast("PROJ_IGNORE_USER_WRITABLE_DIRECTORY=")); info = proj_info(); + putenv(const_cast("PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES")); if (info.version[0] != '\0') { char tmpstr[64]; @@ -360,6 +362,8 @@ TEST(gie, info_functions) { ASSERT_NE(std::string(info.searchpath), std::string()); } + ASSERT_TRUE(std::string(info.searchpath).find("/proj") != std::string::npos); + /* proj_pj_info() */ { P = proj_create(PJ_DEFAULT_CTX, diff --git a/test/unit/proj_context_test.cpp b/test/unit/proj_context_test.cpp index 23c46f29..ec59590d 100644 --- a/test/unit/proj_context_test.cpp +++ b/test/unit/proj_context_test.cpp @@ -40,7 +40,20 @@ namespace { -static std::string createTempDict(std::string &dirname) { +static bool createTmpFile(const std::string &filename) { + FILE *f = fopen(filename.c_str(), "wt"); + if (!f) + return false; + fprintf( + f, + " +proj=pipeline +step +proj=utm +zone=31 +ellps=GRS80\n"); + fclose(f); + return true; +} + +// --------------------------------------------------------------------------- + +static std::string createTempDict(std::string &dirname, const char *filename) { const char *temp_dir = getenv("TEMP"); if (!temp_dir) { temp_dir = getenv("TMP"); @@ -58,16 +71,9 @@ static std::string createTempDict(std::string &dirname) { std::string tmpFilename; tmpFilename = temp_dir; tmpFilename += DIR_CHAR; - tmpFilename += "temp_proj_dic"; + tmpFilename += filename; - FILE *f = fopen(tmpFilename.c_str(), "wt"); - if (!f) - return std::string(); - fprintf( - f, - " +proj=pipeline +step +proj=utm +zone=31 +ellps=GRS80\n"); - fclose(f); - return tmpFilename; + return createTmpFile(tmpFilename) ? tmpFilename : std::string(); } // --------------------------------------------------------------------------- @@ -85,7 +91,7 @@ static int MyUnlink(const std::string &filename) { TEST(proj_context, proj_context_set_file_finder) { std::string dirname; - auto filename = createTempDict(dirname); + auto filename = createTempDict(dirname, "temp_proj_dic1"); if (filename.empty()) return; @@ -111,7 +117,7 @@ TEST(proj_context, proj_context_set_file_finder) { finderData.dirname = dirname; proj_context_set_file_finder(ctx, finder, &finderData); - auto P = proj_create(ctx, "+init=temp_proj_dic:MY_PIPELINE"); + auto P = proj_create(ctx, "+init=temp_proj_dic1:MY_PIPELINE"); EXPECT_NE(P, nullptr); proj_destroy(P); @@ -125,7 +131,7 @@ TEST(proj_context, proj_context_set_file_finder) { TEST(proj_context, proj_context_set_search_paths) { std::string dirname; - auto filename = createTempDict(dirname); + auto filename = createTempDict(dirname, "temp_proj_dic2"); if (filename.empty()) return; @@ -134,7 +140,7 @@ TEST(proj_context, proj_context_set_search_paths) { const char *path = dirname.c_str(); proj_context_set_search_paths(ctx, 1, &path); - auto P = proj_create(ctx, "+init=temp_proj_dic:MY_PIPELINE"); + auto P = proj_create(ctx, "+init=temp_proj_dic2:MY_PIPELINE"); EXPECT_NE(P, nullptr); proj_destroy(P); @@ -143,4 +149,32 @@ TEST(proj_context, proj_context_set_search_paths) { MyUnlink(filename); } +// --------------------------------------------------------------------------- + +TEST(proj_context, read_grid_from_user_writable_directory) { + + auto ctx = proj_context_create(); + auto path = pj_context_get_user_writable_directory(ctx, true); + EXPECT_TRUE(!path.empty()); + auto filename = path + DIR_CHAR + "temp_proj_dic3"; + EXPECT_TRUE(createTmpFile(filename)); + { + // Check that with PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES (set by + // calling script), we cannot find the file + auto P = proj_create(ctx, "+init=temp_proj_dic3:MY_PIPELINE"); + EXPECT_EQ(P, nullptr); + proj_destroy(P); + } + { + // Cancel the effect of PROJ_IGNORE_USER_WRITABLE_DIRECTORY + putenv(const_cast("PROJ_IGNORE_USER_WRITABLE_DIRECTORY=")); + auto P = proj_create(ctx, "+init=temp_proj_dic3:MY_PIPELINE"); + EXPECT_NE(P, nullptr); + putenv(const_cast("PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES")); + proj_destroy(P); + } + proj_context_destroy(ctx); + MyUnlink(filename); +} + } // namespace -- cgit v1.2.3 From 90b6685a990b8c4931aafb508853401a89163e78 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 9 Jan 2020 23:27:43 +0100 Subject: Add proj_is_download_needed() and proj_download_file() --- scripts/reference_exported_symbols.txt | 2 + src/filemanager.cpp | 420 ++++++++++++++++++++++++++++++++- src/proj.h | 9 + test/unit/gie_self_tests.cpp | 3 +- test/unit/test_network.cpp | 131 ++++++++++ 5 files changed, 560 insertions(+), 5 deletions(-) diff --git a/scripts/reference_exported_symbols.txt b/scripts/reference_exported_symbols.txt index a1c03e99..d10f3bc1 100644 --- a/scripts/reference_exported_symbols.txt +++ b/scripts/reference_exported_symbols.txt @@ -931,6 +931,7 @@ proj_cs_get_axis_info proj_cs_get_type proj_destroy proj_dmstor +proj_download_file proj_ellipsoid_get_parameters proj_errno proj_errno_reset @@ -970,6 +971,7 @@ proj_int_list_destroy proj_is_crs proj_is_deprecated proj_is_derived_crs +proj_is_download_needed proj_is_equivalent_to proj_list_angular_units proj_list_destroy diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 1c49a16b..9acea83e 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -468,6 +468,13 @@ static const char *cache_db_structure_sql = " lastModified TEXT," " etag TEXT" ");" + "CREATE TABLE downloaded_file_properties(" + " url TEXT PRIMARY KEY NOT NULL," + " lastChecked TIMESTAMP NOT NULL," + " fileSize INTEGER NOT NULL," + " lastModified TEXT," + " etag TEXT" + ");" "CREATE TABLE chunk_data(" " id INTEGER PRIMARY KEY AUTOINCREMENT CHECK (id > 0)," " data BLOB NOT NULL" @@ -1350,10 +1357,6 @@ class NetworkFile : public File { NetworkFile(const NetworkFile &) = delete; NetworkFile &operator=(const NetworkFile &) = delete; - static bool get_props_from_headers(PJ_CONTEXT *ctx, - PROJ_NETWORK_HANDLE *handle, - FileProperties &props); - protected: NetworkFile(PJ_CONTEXT *ctx, const std::string &url, PROJ_NETWORK_HANDLE *handle, @@ -1373,6 +1376,10 @@ class NetworkFile : public File { bool hasChanged() const override { return m_hasChanged; } static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); + + static bool get_props_from_headers(PJ_CONTEXT *ctx, + PROJ_NETWORK_HANDLE *handle, + FileProperties &props); }; // --------------------------------------------------------------------------- @@ -2296,6 +2303,15 @@ static void CreateDirectory(const std::string &path) { std::string pj_context_get_user_writable_directory(PJ_CONTEXT *ctx, bool create) { + if (ctx->user_writable_directory.empty()) { + // For testing purposes only + const char *env_var_PROJ_USER_WRITABLE_DIRECTORY = + getenv("PROJ_USER_WRITABLE_DIRECTORY"); + if (env_var_PROJ_USER_WRITABLE_DIRECTORY && + env_var_PROJ_USER_WRITABLE_DIRECTORY[0] != '\0') { + ctx->user_writable_directory = env_var_PROJ_USER_WRITABLE_DIRECTORY; + } + } if (ctx->user_writable_directory.empty()) { std::string path; #ifdef _WIN32 @@ -2353,4 +2369,400 @@ std::string pj_context_get_grid_cache_filename(PJ_CONTEXT *ctx) { return ctx->gridChunkCache.filename; } +// --------------------------------------------------------------------------- + +#ifdef WIN32 +static const char dir_chars[] = "/\\"; +#else +static const char dir_chars[] = "/"; +#endif + +static bool is_tilde_slash(const char *name) { + return *name == '~' && strchr(dir_chars, name[1]); +} + +static bool is_rel_or_absolute_filename(const char *name) { + return strchr(dir_chars, *name) || + (*name == '.' && strchr(dir_chars, name[1])) || + (!strncmp(name, "..", 2) && strchr(dir_chars, name[2])) || + (name[0] != '\0' && name[1] == ':' && strchr(dir_chars, name[2])); +} + +static std::string build_url(PJ_CONTEXT *ctx, const char *name) { + if (!is_tilde_slash(name) && !is_rel_or_absolute_filename(name) && + !starts_with(name, "http://") && !starts_with(name, "https://")) { + std::string remote_file(pj_context_get_url_endpoint(ctx)); + if (!remote_file.empty()) { + if (remote_file.back() != '/') { + remote_file += '/'; + } + remote_file += name; + auto pos = remote_file.rfind('.'); + if (pos + 4 == remote_file.size()) { + remote_file = remote_file.substr(0, pos) + ".tif"; + } else { + // For example for resource files like 'alaska' + remote_file += ".tif"; + } + } + return remote_file; + } + return name; +} + //! @endcond + +// --------------------------------------------------------------------------- + +/** Return if a file must be downloaded or is already available in the + * PROJ user-writable directory. + * + * The file will be determinted to have to be downloaded if it does not exist + * yet in the user-writable directory, or if it is determined that a more recent + * version exists. To determine if a more recent version exists, PROJ will + * use the "downloaded_file_properties" table of its grid cache database. + * Consequently files manually placed in the user-writable + * directory without using this function would be considered as + * non-existing/obsolete and would be unconditionnaly downloaded again. + * + * This function can only be used if networking is enabled, and either + * the default curl network API or a custom one have been installed. + * + * @param ctx PROJ context, or NULL + * @param url_or_filename URL or filename (without directory component) + * @param ignore_ttl_setting If set to FALSE, PROJ will only check the + * recentness of an already downloaded file, if + * the delay between the last time it has been + * verified and the current time exceeds the TTL + * setting. This can save network accesses. + * If set to TRUE, PROJ will unconditionnally + * check from the server the recentness of the file. + * @return TRUE if the file must be downloaded with proj_download_file() + * @since 7.0 + */ + +int proj_is_download_needed(PJ_CONTEXT *ctx, const char *url_or_filename, + int ignore_ttl_setting) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (!pj_context_is_network_enabled(ctx)) { + pj_log(ctx, PJ_LOG_ERROR, "Networking capabilities are not enabled"); + return false; + } + + const auto url(build_url(ctx, url_or_filename)); + const char *filename = strrchr(url.c_str(), '/'); + if (filename == nullptr) + return false; + const auto localFilename( + pj_context_get_user_writable_directory(ctx, false) + filename); + + auto f = NS_PROJ::FileManager::open(ctx, localFilename.c_str()); + if (!f) { + return true; + } + f.reset(); + + auto diskCache = NS_PROJ::DiskChunkCache::open(ctx); + if (!diskCache) + return false; + auto stmt = + diskCache->prepare("SELECT lastChecked, fileSize, lastModified, etag " + "FROM downloaded_file_properties WHERE url = ?"); + if (!stmt) + return true; + stmt->bindText(url.c_str()); + if (stmt->execute() != SQLITE_ROW) { + return true; + } + + NS_PROJ::FileProperties cachedProps; + cachedProps.lastChecked = stmt->getInt64(); + cachedProps.size = stmt->getInt64(); + const char *lastModified = stmt->getText(); + cachedProps.lastModified = lastModified ? lastModified : std::string(); + const char *etag = stmt->getText(); + cachedProps.etag = etag ? etag : std::string(); + + if (!ignore_ttl_setting) { + const auto ttl = NS_PROJ::pj_context_get_grid_cache_ttl(ctx); + if (ttl > 0) { + time_t curTime; + time(&curTime); + if (curTime > cachedProps.lastChecked + ttl) { + + unsigned char dummy; + size_t size_read = 0; + std::string errorBuffer; + errorBuffer.resize(1024); + auto handle = ctx->networking.open( + ctx, url.c_str(), 0, 1, &dummy, &size_read, + errorBuffer.size(), &errorBuffer[0], + ctx->networking.user_data); + if (!handle) { + errorBuffer.resize(strlen(errorBuffer.data())); + pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", url.c_str(), + errorBuffer.c_str()); + return false; + } + NS_PROJ::FileProperties props; + if (!NS_PROJ::NetworkFile::get_props_from_headers(ctx, handle, + props)) { + ctx->networking.close(ctx, handle, + ctx->networking.user_data); + return false; + } + ctx->networking.close(ctx, handle, ctx->networking.user_data); + + if (props.size != cachedProps.size || + props.lastModified != cachedProps.lastModified || + props.etag != cachedProps.etag) { + return true; + } + + stmt = diskCache->prepare( + "UPDATE downloaded_file_properties SET lastChecked = ? " + "WHERE url = ?"); + if (!stmt) + return false; + stmt->bindInt64(curTime); + stmt->bindText(url.c_str()); + if (stmt->execute() != SQLITE_DONE) { + auto hDB = diskCache->handle(); + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return false; + } + } + } + } + + return false; +} + +// --------------------------------------------------------------------------- + +/** Download a file in the PROJ user-writable directory. + * + * The file will only be downloaded if it does not exist yet in the + * user-writable directory, or if it is determined that a more recent + * version exists. To determine if a more recent version exists, PROJ will + * use the "downloaded_file_properties" table of its grid cache database. + * Consequently files manually placed in the user-writable + * directory without using this function would be considered as + * non-existing/obsolete and would be unconditionnaly downloaded again. + * + * This function can only be used if networking is enabled, and either + * the default curl network API or a custom one have been installed. + * + * @param ctx PROJ context, or NULL + * @param url_or_filename URL or filename (without directory component) + * @param ignore_ttl_setting If set to FALSE, PROJ will only check the + * recentness of an already downloaded file, if + * the delay between the last time it has been + * verified and the current time exceeds the TTL + * setting. This can save network accesses. + * If set to TRUE, PROJ will unconditionnally + * check from the server the recentness of the file. + * @param progress_cbk Progress callback, or NULL. + * The passed percentage is in the [0, 1] range. + * The progress callback must return TRUE + * if download must be continued. + * @param user_data User data to provide to the progress callback, or NULL + * @return TRUE if the download was successful (or not needed) + * @since 7.0 + */ + +int proj_download_file(PJ_CONTEXT *ctx, const char *url_or_filename, + int ignore_ttl_setting, + int (*progress_cbk)(PJ_CONTEXT *, double pct, + void *user_data), + void *user_data) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (!pj_context_is_network_enabled(ctx)) { + pj_log(ctx, PJ_LOG_ERROR, "Networking capabilities are not enabled"); + return false; + } + if (!proj_is_download_needed(ctx, url_or_filename, ignore_ttl_setting)) { + return true; + } + + const auto url(build_url(ctx, url_or_filename)); + const char *filename = strrchr(url.c_str(), '/'); + if (filename == nullptr) + return false; + const auto localFilename(pj_context_get_user_writable_directory(ctx, true) + + filename); + +#ifdef _WIN32 + const int nPID = GetCurrentProcessId(); +#else + const int nPID = getpid(); +#endif + char szUniqueSuffix[128]; + snprintf(szUniqueSuffix, sizeof(szUniqueSuffix), "%d_%p", nPID, &url); + const auto localFilenameTmp(localFilename + szUniqueSuffix); + FILE *f = fopen(localFilenameTmp.c_str(), "wb"); + if (!f) { + pj_log(ctx, PJ_LOG_ERROR, "Cannot create %s", localFilenameTmp.c_str()); + return false; + } + + constexpr size_t FULL_FILE_CHUNK_SIZE = 1024 * 1024; + std::vector buffer(FULL_FILE_CHUNK_SIZE); + // For testing purposes only + const char *env_var_PROJ_FULL_FILE_CHUNK_SIZE = + getenv("PROJ_FULL_FILE_CHUNK_SIZE"); + if (env_var_PROJ_FULL_FILE_CHUNK_SIZE && + env_var_PROJ_FULL_FILE_CHUNK_SIZE[0] != '\0') { + buffer.resize(atoi(env_var_PROJ_FULL_FILE_CHUNK_SIZE)); + } + size_t size_read = 0; + std::string errorBuffer; + errorBuffer.resize(1024); + auto handle = ctx->networking.open( + ctx, url.c_str(), 0, buffer.size(), &buffer[0], &size_read, + errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data); + if (!handle) { + errorBuffer.resize(strlen(errorBuffer.data())); + pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", url.c_str(), + errorBuffer.c_str()); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + + time_t curTime; + time(&curTime); + NS_PROJ::FileProperties props; + if (!NS_PROJ::NetworkFile::get_props_from_headers(ctx, handle, props)) { + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + + if (size_read < + std::min(static_cast(buffer.size()), props.size)) { + pj_log(ctx, PJ_LOG_ERROR, "Did not get as many bytes as expected"); + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + if (fwrite(buffer.data(), size_read, 1, f) != 1) { + pj_log(ctx, PJ_LOG_ERROR, "Write error"); + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + + unsigned long long totalDownloaded = size_read; + while (totalDownloaded < props.size) { + if (totalDownloaded + buffer.size() > props.size) { + buffer.resize(static_cast(props.size - totalDownloaded)); + } + errorBuffer.resize(1024); + size_read = ctx->networking.read_range( + ctx, handle, totalDownloaded, buffer.size(), &buffer[0], + errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data); + + if (size_read < buffer.size()) { + pj_log(ctx, PJ_LOG_ERROR, "Did not get as many bytes as expected"); + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + if (fwrite(buffer.data(), size_read, 1, f) != 1) { + pj_log(ctx, PJ_LOG_ERROR, "Write error"); + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + + totalDownloaded += size_read; + if (progress_cbk && + !progress_cbk(ctx, double(totalDownloaded) / props.size, + user_data)) { + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + } + + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + + unlink(localFilename.c_str()); + if (rename(localFilenameTmp.c_str(), localFilename.c_str()) != 0) { + pj_log(ctx, PJ_LOG_ERROR, "Cannot rename %s to %s", + localFilenameTmp.c_str(), localFilename.c_str()); + return false; + } + + auto diskCache = NS_PROJ::DiskChunkCache::open(ctx); + if (!diskCache) + return false; + auto stmt = + diskCache->prepare("SELECT lastChecked, fileSize, lastModified, etag " + "FROM downloaded_file_properties WHERE url = ?"); + if (!stmt) + return false; + stmt->bindText(url.c_str()); + + props.lastChecked = curTime; + auto hDB = diskCache->handle(); + + if (stmt->execute() == SQLITE_ROW) { + stmt = diskCache->prepare( + "UPDATE downloaded_file_properties SET lastChecked = ?, " + "fileSize = ?, lastModified = ?, etag = ? " + "WHERE url = ?"); + if (!stmt) + return false; + stmt->bindInt64(props.lastChecked); + stmt->bindInt64(props.size); + if (props.lastModified.empty()) + stmt->bindNull(); + else + stmt->bindText(props.lastModified.c_str()); + if (props.etag.empty()) + stmt->bindNull(); + else + stmt->bindText(props.etag.c_str()); + stmt->bindText(url.c_str()); + if (stmt->execute() != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return false; + } + } else { + stmt = diskCache->prepare( + "INSERT INTO downloaded_file_properties (url, lastChecked, " + "fileSize, lastModified, etag) VALUES " + "(?,?,?,?,?)"); + if (!stmt) + return false; + stmt->bindText(url.c_str()); + stmt->bindInt64(props.lastChecked); + stmt->bindInt64(props.size); + if (props.lastModified.empty()) + stmt->bindNull(); + else + stmt->bindText(props.lastModified.c_str()); + if (props.etag.empty()) + stmt->bindNull(); + else + stmt->bindText(props.etag.c_str()); + if (stmt->execute() != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return false; + } + } + return true; +} diff --git a/src/proj.h b/src/proj.h index 5c9e81bb..8add29ac 100644 --- a/src/proj.h +++ b/src/proj.h @@ -443,6 +443,15 @@ void PROJ_DLL proj_grid_cache_set_ttl(PJ_CONTEXT* ctx, int ttl_seconds); void PROJ_DLL proj_grid_cache_clear(PJ_CONTEXT* ctx); +int PROJ_DLL proj_is_download_needed(PJ_CONTEXT* ctx, + const char* url_or_filename, + int ignore_ttl_setting); +int PROJ_DLL proj_download_file(PJ_CONTEXT *ctx, const char *url_or_filename, + int ignore_ttl_setting, + int (*progress_cbk)(PJ_CONTEXT *, double pct, + void *user_data), + void *user_data); + /*! @cond Doxygen_Suppress */ /* Manage the transformation definition object PJ */ diff --git a/test/unit/gie_self_tests.cpp b/test/unit/gie_self_tests.cpp index a3b41fb0..fc0f0748 100644 --- a/test/unit/gie_self_tests.cpp +++ b/test/unit/gie_self_tests.cpp @@ -362,7 +362,8 @@ TEST(gie, info_functions) { ASSERT_NE(std::string(info.searchpath), std::string()); } - ASSERT_TRUE(std::string(info.searchpath).find("/proj") != std::string::npos); + ASSERT_TRUE(std::string(info.searchpath).find("/proj") != + std::string::npos); /* proj_pj_info() */ { diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp index e6e7bb7b..4e66d8c5 100644 --- a/test/unit/test_network.cpp +++ b/test/unit/test_network.cpp @@ -38,6 +38,12 @@ #include #include +#ifdef _WIN32 +#include +#else +#include +#endif + #ifdef CURL_ENABLED #include #endif @@ -1558,6 +1564,131 @@ TEST(networking, cache_lock) { proj_context_destroy(ctx); } + +// --------------------------------------------------------------------------- + +TEST(networking, download_whole_files) { + if (!networkAccessOK) { + return; + } + + proj_cleanup(); + unlink("proj_test_tmp/cache.db"); + unlink("proj_test_tmp/ntf_r93.tif"); + rmdir("proj_test_tmp"); + + putenv(const_cast("PROJ_IGNORE_USER_WRITABLE_DIRECTORY=")); + putenv(const_cast("PROJ_USER_WRITABLE_DIRECTORY=./proj_test_tmp")); + putenv(const_cast("PROJ_FULL_FILE_CHUNK_SIZE=30000")); + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + ASSERT_TRUE(proj_is_download_needed(ctx, "ntf_r93.gsb", false)); + + ASSERT_TRUE( + proj_download_file(ctx, "ntf_r93.gsb", false, nullptr, nullptr)); + + FILE *f = fopen("proj_test_tmp/ntf_r93.tif", "rb"); + ASSERT_NE(f, nullptr); + fseek(f, 0, SEEK_END); + ASSERT_EQ(ftell(f), 93581); + fclose(f); + + ASSERT_FALSE(proj_is_download_needed(ctx, "ntf_r93.gsb", false)); + + { + sqlite3 *hDB = nullptr; + sqlite3_open_v2("proj_test_tmp/cache.db", &hDB, SQLITE_OPEN_READWRITE, + nullptr); + ASSERT_NE(hDB, nullptr); + // Force lastChecked to the Epoch so that data is expired. + sqlite3_stmt *hStmt = nullptr; + sqlite3_prepare_v2( + hDB, "UPDATE downloaded_file_properties SET lastChecked = 0", -1, + &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_DONE); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + } + + // If we ignore TTL settings, then no network access will be done + ASSERT_FALSE(proj_is_download_needed(ctx, "ntf_r93.gsb", true)); + + { + sqlite3 *hDB = nullptr; + sqlite3_open_v2("proj_test_tmp/cache.db", &hDB, SQLITE_OPEN_READWRITE, + nullptr); + ASSERT_NE(hDB, nullptr); + // Check that the lastChecked timestamp is still 0 + sqlite3_stmt *hStmt = nullptr; + sqlite3_prepare_v2(hDB, + "SELECT lastChecked FROM downloaded_file_properties", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(hStmt, 0), 0); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + } + + // Should recheck from the CDN, update last_checked and do nothing + ASSERT_FALSE(proj_is_download_needed(ctx, "ntf_r93.gsb", false)); + + { + sqlite3 *hDB = nullptr; + sqlite3_open_v2("proj_test_tmp/cache.db", &hDB, SQLITE_OPEN_READWRITE, + nullptr); + ASSERT_NE(hDB, nullptr); + sqlite3_stmt *hStmt = nullptr; + // Check that the lastChecked timestamp has been updated + sqlite3_prepare_v2(hDB, + "SELECT lastChecked FROM downloaded_file_properties", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_ROW); + ASSERT_NE(sqlite3_column_int64(hStmt, 0), 0); + sqlite3_finalize(hStmt); + hStmt = nullptr; + + // Now invalid lastModified. This should trigger a new download + sqlite3_prepare_v2( + hDB, "UPDATE downloaded_file_properties SET lastChecked = 0, " + "lastModified = 'foo'", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_DONE); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + } + + ASSERT_TRUE(proj_is_download_needed(ctx, "ntf_r93.gsb", false)); + + // Redo download with a progress callback this time. + unlink("proj_test_tmp/ntf_r93.tif"); + + const auto cbk = [](PJ_CONTEXT *l_ctx, double pct, void *user_data) -> int { + auto vect = static_cast> *>( + user_data); + vect->push_back(std::pair(l_ctx, pct)); + return true; + }; + + std::vector> vectPct; + ASSERT_TRUE(proj_download_file(ctx, "ntf_r93.gsb", false, cbk, &vectPct)); + ASSERT_EQ(vectPct.size(), 3U); + ASSERT_EQ(vectPct.back().first, ctx); + ASSERT_EQ(vectPct.back().second, 1.0); + + proj_context_destroy(ctx); + putenv(const_cast("PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES")); + putenv(const_cast("PROJ_USER_WRITABLE_DIRECTORY=")); + putenv(const_cast("PROJ_FULL_FILE_CHUNK_SIZE=")); + unlink("proj_test_tmp/cache.db"); + unlink("proj_test_tmp/ntf_r93.tif"); + rmdir("proj_test_tmp"); +} + #endif } // namespace -- cgit v1.2.3