1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
|
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>PROJ RFC 4: Remote access to grids and GeoTIFF grids — PROJ 9.0.0 documentation</title>
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<link rel="shortcut icon" href="../../_static/favicon.png"/>
<link rel="canonical" href="https://proj.orgcommunity/rfc/rfc-4.html"/>
<!--[if lt IE 9]>
<script src="../../_static/js/html5shiv.min.js"></script>
<![endif]-->
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
<script src="../../_static/jquery.js"></script>
<script src="../../_static/underscore.js"></script>
<script src="../../_static/doctools.js"></script>
<script src="../../_static/js/theme.js"></script>
<link rel="author" title="About these documents" href="../../about.html" />
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<link rel="next" title="PROJ RFC 5: Adopt GeoTIFF-based grids for grids delivered with PROJ" href="rfc-5.html" />
<link rel="prev" title="PROJ RFC 3: Dependency management" href="rfc-3.html" />
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" style="background: #353130" >
<a href="../../index.html">
<img src="../../_static/logo.png" class="logo" alt="Logo"/>
</a>
<div class="version">
9.0.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../about.html">About</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../news.html">News</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../download.html">Download</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../install.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../usage/index.html">Using PROJ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../apps/index.html">Applications</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../operations/index.html">Coordinate operations</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../resource_files.html">Resource files</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../geodesic.html">Geodesic calculations</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../development/index.html">Development</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../specifications/index.html">Specifications</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">Community</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../channels.html">Communication channels</a></li>
<li class="toctree-l2"><a class="reference internal" href="../contributing.html">Contributing</a></li>
<li class="toctree-l2"><a class="reference internal" href="../code_contributions.html">Guidelines for PROJ code contributors</a></li>
<li class="toctree-l2"><a class="reference internal" href="../code_of_conduct.html">Code of Conduct</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">Request for Comments</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="rfc-1.html">PROJ RFC 1: Project Committee Guidelines</a></li>
<li class="toctree-l3"><a class="reference internal" href="rfc-2.html">PROJ RFC 2: Initial integration of “GDAL SRS barn” work</a></li>
<li class="toctree-l3"><a class="reference internal" href="rfc-3.html">PROJ RFC 3: Dependency management</a></li>
<li class="toctree-l3 current"><a class="current reference internal" href="#">PROJ RFC 4: Remote access to grids and GeoTIFF grids</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#motivation">Motivation</a></li>
<li class="toctree-l4"><a class="reference internal" href="#summary-of-work-planned-by-this-rfc">Summary of work planned by this RFC</a></li>
<li class="toctree-l4"><a class="reference internal" href="#network-access-to-grids">Network access to grids</a></li>
<li class="toctree-l4"><a class="reference internal" href="#grids-in-geotiff-format">Grids in GeoTIFF format</a></li>
<li class="toctree-l4"><a class="reference internal" href="#dropping-grid-catalog-functionality">Dropping grid catalog functionality</a></li>
<li class="toctree-l4"><a class="reference internal" href="#backward-compatibility-issues">Backward compatibility issues</a></li>
<li class="toctree-l4"><a class="reference internal" href="#potential-future-related-work">Potential future related work</a></li>
<li class="toctree-l4"><a class="reference internal" href="#documentation">Documentation</a></li>
<li class="toctree-l4"><a class="reference internal" href="#testing">Testing</a></li>
<li class="toctree-l4"><a class="reference internal" href="#proposed-implementation">Proposed implementation</a></li>
<li class="toctree-l4"><a class="reference internal" href="#adoption-status">Adoption status</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="rfc-5.html">PROJ RFC 5: Adopt GeoTIFF-based grids for grids delivered with PROJ</a></li>
<li class="toctree-l3"><a class="reference internal" href="rfc-6.html">PROJ RFC 6: Triangulation-based transformations</a></li>
<li class="toctree-l3"><a class="reference internal" href="rfc-7.html">PROJ RFC 7: Drop Autotools, maintain CMake</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../index.html#conference">Conference</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../glossary.html">Glossary</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../zreferences.html">References</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" style="background: #353130" >
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../index.html">PROJ</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="Page navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../index.html" class="icon icon-home"></a> »</li>
<li><a href="../index.html">Community</a> »</li>
<li><a href="index.html">Request for Comments</a> »</li>
<li>PROJ RFC 4: Remote access to grids and GeoTIFF grids</li>
<li class="wy-breadcrumbs-aside">
<a href="https://github.com/OSGeo/PROJ/edit/8.2/docs/source/community/rfc/rfc-4.rst" class="fa fa-github"> Edit on GitHub</a>
</li>
</ul><div class="rst-breadcrumbs-buttons" role="navigation" aria-label="Sequential page navigation">
<a href="rfc-3.html" class="btn btn-neutral float-left" title="PROJ RFC 3: Dependency management" accesskey="p"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
<a href="rfc-5.html" class="btn btn-neutral float-right" title="PROJ RFC 5: Adopt GeoTIFF-based grids for grids delivered with PROJ" accesskey="n">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
</div>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<section id="proj-rfc-4-remote-access-to-grids-and-geotiff-grids">
<span id="rfc4"></span><h1>PROJ RFC 4: Remote access to grids and GeoTIFF grids<a class="headerlink" href="#proj-rfc-4-remote-access-to-grids-and-geotiff-grids" title="Permalink to this headline">¶</a></h1>
<dl class="field-list simple">
<dt class="field-odd">Author</dt>
<dd class="field-odd"><p>Even Rouault, Howard Butler</p>
</dd>
<dt class="field-even">Contact</dt>
<dd class="field-even"><p><a class="reference external" href="mailto:even.rouault%40spatialys.com">even<span>.</span>rouault<span>@</span>spatialys<span>.</span>com</a>, <a class="reference external" href="mailto:howard%40hobu.co">howard<span>@</span>hobu<span>.</span>co</a></p>
</dd>
<dt class="field-odd">Status</dt>
<dd class="field-odd"><p>Adopted</p>
</dd>
<dt class="field-even">Implementation target</dt>
<dd class="field-even"><p>PROJ 7</p>
</dd>
<dt class="field-odd">Last Updated</dt>
<dd class="field-odd"><p>2020-01-10</p>
</dd>
</dl>
<section id="motivation">
<h2>Motivation<a class="headerlink" href="#motivation" title="Permalink to this headline">¶</a></h2>
<p>PROJ 6 brings undeniable advances in the management of coordinate
transformations between datums by relying and applying information available in
the PROJ database. PROJ’s rapid evolution from a cartographic projections
library with a little bit of geodetic capability to a full geodetic
transformation and description environment has highlighted the importance of
the support data. Users desire the convenience of software doing the right
thing with the least amount of fuss, and survey organizations wish to deliver
their models across as wide a software footprint as possible. To get results
with the highest precision, a grid file that defines a model that provides
dimension shifts is often needed. The proj-datumgrid project centralizes grids
available under an open data license and bundles them in different archives
split along major geographical regions of the world .</p>
<p>It is assumed that a PROJ user has downloaded and installed grid files that are
referred to in the PROJ database. These files can be quite large in aggregate,
and packaging support by major distribution channels is somewhat uneven due to
their size, sometimes ambiguous licensing story, and difficult-to-track
versioning and lineage. It is not always clear to the user, especially to
those who may not be so familiar with geodetic operations, that the highest
precision transformation may not always being applied if grid data is not
available. Users want both convenience and correctness, and management of the
shift files can be challenging to those who may not be aware of their
importance to the process.</p>
<p>The computing environment in which PROJ operates is also changing. Because the
shift data can be so large (currently more than 700 MB of uncompressed data,
and growing), deployment of high accuracy operations can be limited due to
deployment size constraints (serverless operations, for example). Changing to a
delivery format that supports incremental access over a network along with
convenient access and compression will ease the resource burden the shift files
present while allowing the project to deliver transformation capability with
the highest known precision provided by the survey organizations.</p>
<p>Adjustment grids also tend to be provided in many different formats depending
on the organization and country that produced them. In PROJ, we have over time
“standardized” on using horizontal shift grids as NTv2 and vertical shift grids
using GTX. Both have poor general support as dedicated formats, limited
metadata capabilities, and neither are not necessarily “cloud optimized” for
incremental access across a network.</p>
</section>
<section id="summary-of-work-planned-by-this-rfc">
<h2>Summary of work planned by this RFC<a class="headerlink" href="#summary-of-work-planned-by-this-rfc" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>Grids will be hosted by one or several Content Delivery Networks (CDN)</p></li>
<li><p>Grid loading mechanism will be reworked to be able to download grids or parts
of grids from a online repository. When opted in, users will no longer have to
manually fetch grid files and place them in PROJ_LIB.
Full and accurate capability of the software will no longer require hundreds
of megabytes of grid shift files in advance, even if only just a few of them
are needed for the transformations done by the user.</p></li>
<li><p>Local caching of grid files, or even part of files, so that users end up
mirroring what they actually use.</p></li>
<li><p>A grid shift format, for both horizontal and vertical shift grids (and in
potential future steps, for other needs, such as deformation models) will be
implemented.</p></li>
</ul>
<p>The use of grids locally available will of course still be available, and will
be the default behavior.</p>
</section>
<section id="network-access-to-grids">
<h2>Network access to grids<a class="headerlink" href="#network-access-to-grids" title="Permalink to this headline">¶</a></h2>
<p>curl will be an optional build dependency of PROJ, added in autoconf and cmake
build systems. It can be disabled at build time, but this must be
an explicit setting of configure/cmake as the resulting builds have less functionality.
When curl is enabled at build time, download of grids themselves will not be
enabled by default at runtime. It will require explicit consent of the user, either
through the API
(<a class="reference internal" href="../../development/reference/functions.html#c.proj_context_set_enable_network" title="proj_context_set_enable_network"><code class="xref c c-func docutils literal notranslate"><span class="pre">proj_context_set_enable_network()</span></code></a>) through the PROJ_NETWORK=ON
environment variable, or the <code class="docutils literal notranslate"><span class="pre">network</span> <span class="pre">=</span> <span class="pre">on</span></code> setting of proj.ini.</p>
<p>Regarding the minimum version of libcurl required, given GDAL experience that
can build with rather ancient libcurl for similar functionality, we can aim for
libcurl >= 7.29.0 (as being available in RHEL 7).</p>
<p>An alternate pluggable network interface can also be set by the user in case
support for libcurl was not built in, or if for the desired context of use, the
user wishes to provide the network implementation (a typical use case could be
QGIS that would use its QT-based networking facilities to solve issues with
SSL, proxy, authentication, etc.)</p>
<p>A text configuration file, installed in ${installation_prefix}/share/proj/proj.ini
(or ${PROJ_LIB}/proj.ini)
will contain the URL of the CDN that will be used.
The user may also override this setting with the
<a class="reference internal" href="../../development/reference/functions.html#c.proj_context_set_url_endpoint" title="proj_context_set_url_endpoint"><code class="xref c c-func docutils literal notranslate"><span class="pre">proj_context_set_url_endpoint()</span></code></a> or through the PROJ_NETWORK_ENDPOINT
environment variable.</p>
<p>The rationale for putting proj.ini in that location is
that it is a well-known place by PROJ users, with the existing PROJ_LIB mechanics
for systems like Windows where hardcoded paths at runtime aren’t generally usable.</p>
<section id="c-api">
<h3>C API<a class="headerlink" href="#c-api" title="Permalink to this headline">¶</a></h3>
<p>The preliminary C API for the above is:</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="cm">/** Enable or disable network access.</span>
<span class="cm">*</span>
<span class="cm">* @param ctx PROJ context, or NULL</span>
<span class="cm">* @return TRUE if network access is possible. That is either libcurl is</span>
<span class="cm">* available, or an alternate interface has been set.</span>
<span class="cm">*/</span><span class="w"></span>
<span class="kt">int</span><span class="w"> </span><span class="nf">proj_context_set_enable_network</span><span class="p">(</span><span class="n">PJ_CONTEXT</span><span class="o">*</span><span class="w"> </span><span class="n">ctx</span><span class="p">,</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">enable</span><span class="p">);</span><span class="w"></span>
<span class="cm">/** Define URL endpoint to query for remote grids.</span>
<span class="cm">*</span>
<span class="cm">* This overrides the default endpoint in the PROJ configuration file or with</span>
<span class="cm">* the PROJ_NETWORK_ENDPOINT environment variable.</span>
<span class="cm">*</span>
<span class="cm">* @param ctx PROJ context, or NULL</span>
<span class="cm">* @param url Endpoint URL. Must NOT be NULL.</span>
<span class="cm">*/</span><span class="w"></span>
<span class="kt">void</span><span class="w"> </span><span class="nf">proj_context_set_url_endpoint</span><span class="p">(</span><span class="n">PJ_CONTEXT</span><span class="o">*</span><span class="w"> </span><span class="n">ctx</span><span class="p">,</span><span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="o">*</span><span class="w"> </span><span class="n">url</span><span class="p">);</span><span class="w"></span>
<span class="cm">/** Opaque structure for PROJ. Implementations might cast it to their</span>
<span class="cm"> * structure/class of choice. */</span><span class="w"></span>
<span class="k">typedef</span><span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">PROJ_NETWORK_HANDLE</span><span class="w"> </span><span class="n">PROJ_NETWORK_HANDLE</span><span class="p">;</span><span class="w"></span>
<span class="cm">/** Network access: open callback</span>
<span class="cm">*</span>
<span class="cm">* Should try to read the size_to_read first bytes at the specified offset of</span>
<span class="cm">* the file given by URL url,</span>
<span class="cm">* and write them to buffer. *out_size_read should be updated with the actual</span>
<span class="cm">* amount of bytes read (== size_to_read if the file is larger than size_to_read).</span>
<span class="cm">* During this read, the implementation should make sure to store the HTTP</span>
<span class="cm">* headers from the server response to be able to respond to</span>
<span class="cm">* proj_network_get_header_value_cbk_type callback.</span>
<span class="cm">*</span>
<span class="cm">* error_string_max_size should be the maximum size that can be written into</span>
<span class="cm">* the out_error_string buffer (including terminating nul character).</span>
<span class="cm">*</span>
<span class="cm">* @return a non-NULL opaque handle in case of success.</span>
<span class="cm">*/</span><span class="w"></span>
<span class="k">typedef</span><span class="w"> </span><span class="n">PROJ_NETWORK_HANDLE</span><span class="o">*</span><span class="w"> </span><span class="p">(</span><span class="o">*</span><span class="n">proj_network_open_cbk_type</span><span class="p">)(</span><span class="w"></span>
<span class="w"> </span><span class="n">PJ_CONTEXT</span><span class="o">*</span><span class="w"> </span><span class="n">ctx</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="o">*</span><span class="w"> </span><span class="n">url</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">long</span><span class="w"> </span><span class="kt">long</span><span class="w"> </span><span class="n">offset</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">size_t</span><span class="w"> </span><span class="n">size_to_read</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">void</span><span class="o">*</span><span class="w"> </span><span class="n">buffer</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">size_t</span><span class="o">*</span><span class="w"> </span><span class="n">out_size_read</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">size_t</span><span class="w"> </span><span class="n">error_string_max_size</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">char</span><span class="o">*</span><span class="w"> </span><span class="n">out_error_string</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">void</span><span class="o">*</span><span class="w"> </span><span class="n">user_data</span><span class="p">);</span><span class="w"></span>
<span class="cm">/** Network access: close callback */</span><span class="w"></span>
<span class="k">typedef</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="p">(</span><span class="o">*</span><span class="n">proj_network_close_cbk_type</span><span class="p">)(</span><span class="n">PJ_CONTEXT</span><span class="o">*</span><span class="w"> </span><span class="n">ctx</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">PROJ_NETWORK_HANDLE</span><span class="o">*</span><span class="w"> </span><span class="n">handle</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">void</span><span class="o">*</span><span class="w"> </span><span class="n">user_data</span><span class="p">);</span><span class="w"></span>
<span class="cm">/** Network access: get HTTP headers */</span><span class="w"></span>
<span class="k">typedef</span><span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="o">*</span><span class="w"> </span><span class="p">(</span><span class="o">*</span><span class="n">proj_network_get_header_value_cbk_type</span><span class="p">)(</span><span class="w"></span>
<span class="w"> </span><span class="n">PJ_CONTEXT</span><span class="o">*</span><span class="w"> </span><span class="n">ctx</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">PROJ_NETWORK_HANDLE</span><span class="o">*</span><span class="w"> </span><span class="n">handle</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="o">*</span><span class="w"> </span><span class="n">header_name</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">void</span><span class="o">*</span><span class="w"> </span><span class="n">user_data</span><span class="p">);</span><span class="w"></span>
<span class="cm">/** Network access: read range</span>
<span class="cm">*</span>
<span class="cm">* Read size_to_read bytes from handle, starting at offset, into</span>
<span class="cm">* buffer.</span>
<span class="cm">* During this read, the implementation should make sure to store the HTTP</span>
<span class="cm">* headers from the server response to be able to respond to</span>
<span class="cm">* proj_network_get_header_value_cbk_type callback.</span>
<span class="cm">*</span>
<span class="cm">* error_string_max_size should be the maximum size that can be written into</span>
<span class="cm">* the out_error_string buffer (including terminating nul character).</span>
<span class="cm">*</span>
<span class="cm">* @return the number of bytes actually read (0 in case of error)</span>
<span class="cm">*/</span><span class="w"></span>
<span class="k">typedef</span><span class="w"> </span><span class="kt">size_t</span><span class="w"> </span><span class="p">(</span><span class="o">*</span><span class="n">proj_network_read_range_type</span><span class="p">)(</span><span class="w"></span>
<span class="w"> </span><span class="n">PJ_CONTEXT</span><span class="o">*</span><span class="w"> </span><span class="n">ctx</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">PROJ_NETWORK_HANDLE</span><span class="o">*</span><span class="w"> </span><span class="n">handle</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">long</span><span class="w"> </span><span class="kt">long</span><span class="w"> </span><span class="n">offset</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">size_t</span><span class="w"> </span><span class="n">size_to_read</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">void</span><span class="o">*</span><span class="w"> </span><span class="n">buffer</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">size_t</span><span class="w"> </span><span class="n">error_string_max_size</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">char</span><span class="o">*</span><span class="w"> </span><span class="n">out_error_string</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">void</span><span class="o">*</span><span class="w"> </span><span class="n">user_data</span><span class="p">);</span><span class="w"></span>
<span class="cm">/** Define a custom set of callbacks for network access.</span>
<span class="cm">*</span>
<span class="cm">* All callbacks should be provided (non NULL pointers).</span>
<span class="cm">*</span>
<span class="cm">* @param ctx PROJ context, or NULL</span>
<span class="cm">* @param open_cbk Callback to open a remote file given its URL</span>
<span class="cm">* @param close_cbk Callback to close a remote file.</span>
<span class="cm">* @param get_header_value_cbk Callback to get HTTP headers</span>
<span class="cm">* @param read_range_cbk Callback to read a range of bytes inside a remote file.</span>
<span class="cm">* @param user_data Arbitrary pointer provided by the user, and passed to the</span>
<span class="cm">* above callbacks. May be NULL.</span>
<span class="cm">* @return TRUE in case of success.</span>
<span class="cm">*/</span><span class="w"></span>
<span class="kt">int</span><span class="w"> </span><span class="nf">proj_context_set_network_callbacks</span><span class="p">(</span><span class="w"></span>
<span class="w"> </span><span class="n">PJ_CONTEXT</span><span class="o">*</span><span class="w"> </span><span class="n">ctx</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">proj_network_open_cbk_type</span><span class="w"> </span><span class="n">open_cbk</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">proj_network_close_cbk_type</span><span class="w"> </span><span class="n">close_cbk</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">proj_network_get_header_value_cbk_type</span><span class="w"> </span><span class="n">get_header_value_cbk</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">proj_network_read_range_type</span><span class="w"> </span><span class="n">read_range_cbk</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="kt">void</span><span class="o">*</span><span class="w"> </span><span class="n">user_data</span><span class="p">);</span><span class="w"></span>
</pre></div>
</div>
<p>To make network access efficient, PROJ will internally have a in-memory cache
of file ranges to only issue network requests by chunks of 16 KB or multiple of them,
to limit the number of HTTP GET requests and minimize latency caused by network
access. This is very similar to the behavior of the GDAL
<a class="reference external" href="https://gdal.org/user/virtual_file_systems.html#vsicurl-http-https-ftp-files-random-access">/vsicurl/</a>
I/O layer. The plan is to mostly copy GDAL’s vsicurl implementation inside PROJ, with
needed adjustments and proper namespacing of it.</p>
<p>A retry strategy (typically a delay with an exponential back-off and some random
jitter) will be added to account for intermittent network or server-side failure.</p>
</section>
<section id="url-building">
<h3>URL building<a class="headerlink" href="#url-building" title="Permalink to this headline">¶</a></h3>
<p>The PROJ database has a <code class="docutils literal notranslate"><span class="pre">grid_transformation</span></code> grid whose column <code class="docutils literal notranslate"><span class="pre">grid_name</span></code>
(and possibly <code class="docutils literal notranslate"><span class="pre">grid2_name</span></code>) contain the name of the grid as indicated by the
authority having registered the transformation (typically EPSG). As those
grid names are not generally directly usable by PROJ, the PROJ database has
also a <code class="docutils literal notranslate"><span class="pre">grid_alternatives</span></code> table that link original grid names to the ones used
by PROJ. When network access will be available and needed due to lack of a
local grid, the full URL will be the
endpoint from the configuration or set by the user, the basename of the PROJ
usable filename, and the “tif” suffix. So if the CDN is at <a class="reference external" href="http://example.com">http://example.com</a>
and the name from <code class="docutils literal notranslate"><span class="pre">grid_alternatives</span></code> is egm96_15.gtx, then the URL will
be <a class="reference external" href="http://example.com/egm96_15.tif">http://example.com/egm96_15.tif</a></p>
</section>
<section id="grid-loading">
<h3>Grid loading<a class="headerlink" href="#grid-loading" title="Permalink to this headline">¶</a></h3>
<p>The following files will be affected, in one way or another, by the above describes
changes:
nad_cvt.cpp, nad_intr.cpp, nad_init.cpp, grid_info.cpp, grid_list.cpp, apply_gridshift.cpp,
apply_vgridshift.cpp.</p>
<p>In particular the current logic that consists to ingest all the values of a
grid/subgrid in the ct->cvs array will be completely modified, to enable
access to grid values at a specified (x,y) location.</p>
</section>
<section id="proj-create-crs-to-crs-proj-create-operations-impacts">
<h3>proj_create_crs_to_crs() / proj_create_operations() impacts<a class="headerlink" href="#proj-create-crs-to-crs-proj-create-operations-impacts" title="Permalink to this headline">¶</a></h3>
<p>Once network access is available, all grids known to the PROJ database
(grid_transformation + grid_alternatives table) will be assumed to be available,
when computing the potential pipelines between two CRS.</p>
<p>Concretely, this will be equivalent to calling
<code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">proj_operation_factory_context_set_grid_availability_use()</span></code>
with the <code class="docutils literal notranslate"><span class="pre">use</span></code> argument set to a new enumeration value</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="cm">/** Results will be presented as if grids known to PROJ (that is</span>
<span class="cm">* registered in the grid_alternatives table of its database) were</span>
<span class="cm">* available. Used typically when networking is enabled.</span>
<span class="cm">*/</span><span class="w"></span>
<span class="n">PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE</span><span class="w"></span>
</pre></div>
</div>
</section>
<section id="local-on-disk-caching-of-remote-grids">
<h3>Local on-disk caching of remote grids<a class="headerlink" href="#local-on-disk-caching-of-remote-grids" title="Permalink to this headline">¶</a></h3>
<p>As many workflows will tend to use the same grids over and over, a local
on-disk caching of remote grids will be added. The cache will be a single
SQLite3 database, in a user-writable directory shared by all applications using
PROJ.</p>
<p>Its total size will be configurable, with a default maximum size of 100 MB
in proj.ini. The cache will also keep the timestamp of the last time it checked
various global properties of the file (its size, Last-Modified and ETag headers).
A time-to-live parameter, with a default of 1 day in proj.ini, will be used to
determine whether the CDN should be hit to verify if the information in the
cache is still up-to-date.</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="cm">/** Enable or disable the local cache of grid chunks</span>
<span class="cm">*</span>
<span class="cm">* This overrides the setting in the PROJ configuration file.</span>
<span class="cm">*</span>
<span class="cm">* @param ctx PROJ context, or NULL</span>
<span class="cm">* @param enabled TRUE if the cache is enabled.</span>
<span class="cm">*/</span><span class="w"></span>
<span class="kt">void</span><span class="w"> </span><span class="nf">proj_grid_cache_set_enable</span><span class="p">(</span><span class="n">PJ_CONTEXT</span><span class="w"> </span><span class="o">*</span><span class="n">ctx</span><span class="p">,</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">enabled</span><span class="p">);</span><span class="w"></span>
<span class="cm">/** Override, for the considered context, the path and file of the local</span>
<span class="cm">* cache of grid chunks.</span>
<span class="cm">*</span>
<span class="cm">* @param ctx PROJ context, or NULL</span>
<span class="cm">* @param fullname Full name to the cache (encoded in UTF-8). If set to NULL,</span>
<span class="cm">* caching will be disabled.</span>
<span class="cm">*/</span><span class="w"></span>
<span class="kt">void</span><span class="w"> </span><span class="nf">proj_grid_cache_set_filename</span><span class="p">(</span><span class="n">PJ_CONTEXT</span><span class="o">*</span><span class="w"> </span><span class="n">ctx</span><span class="p">,</span><span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="o">*</span><span class="w"> </span><span class="n">fullname</span><span class="p">);</span><span class="w"></span>
<span class="cm">/** Override, for the considered context, the maximum size of the local</span>
<span class="cm">* cache of grid chunks.</span>
<span class="cm">*</span>
<span class="cm">* @param ctx PROJ context, or NULL</span>
<span class="cm">* @param max_size_MB Maximum size, in mega-bytes (1024*1024 bytes), or</span>
<span class="cm">* negative value to set unlimited size.</span>
<span class="cm">*/</span><span class="w"></span>
<span class="kt">void</span><span class="w"> </span><span class="nf">proj_grid_cache_set_max_size</span><span class="p">(</span><span class="n">PJ_CONTEXT</span><span class="o">*</span><span class="w"> </span><span class="n">ctx</span><span class="p">,</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">max_size_MB</span><span class="p">);</span><span class="w"></span>
<span class="cm">/** Override, for the considered context, the time-to-live delay for</span>
<span class="cm">* re-checking if the cached properties of files are still up-to-date.</span>
<span class="cm">*</span>
<span class="cm">* @param ctx PROJ context, or NULL</span>
<span class="cm">* @param ttl_seconds Delay in seconds. Use negative value for no expiration.</span>
<span class="cm">*/</span><span class="w"></span>
<span class="kt">void</span><span class="w"> </span><span class="nf">proj_grid_cache_set_ttl</span><span class="p">(</span><span class="n">PJ_CONTEXT</span><span class="o">*</span><span class="w"> </span><span class="n">ctx</span><span class="p">,</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">ttl_seconds</span><span class="p">);</span><span class="w"></span>
<span class="cm">/** Clear the local cache of grid chunks.</span>
<span class="cm"> *</span>
<span class="cm"> * @param ctx PROJ context, or NULL.</span>
<span class="cm"> */</span><span class="w"></span>
<span class="kt">void</span><span class="w"> </span><span class="nf">proj_grid_cache_clear</span><span class="p">(</span><span class="n">PJ_CONTEXT</span><span class="o">*</span><span class="w"> </span><span class="n">ctx</span><span class="p">);</span><span class="w"></span>
</pre></div>
</div>
<p>The planned database structure is:</p>
<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="c1">-- General properties on a file</span>
<span class="k">CREATE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">properties</span><span class="p">(</span><span class="w"></span>
<span class="w"> </span><span class="n">url</span><span class="w"> </span><span class="nb">TEXT</span><span class="w"> </span><span class="k">PRIMARY</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">lastChecked</span><span class="w"> </span><span class="k">TIMESTAMP</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">fileSize</span><span class="w"> </span><span class="nb">INTEGER</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">lastModified</span><span class="w"> </span><span class="nb">TEXT</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">etag</span><span class="w"> </span><span class="nb">TEXT</span><span class="w"></span>
<span class="p">);</span><span class="w"></span>
<span class="c1">-- Store chunks of data. To avoid any potential fragmentation of the</span>
<span class="c1">-- cache, the data BLOB is always set to the maximum chunk size of 16 KB</span>
<span class="c1">-- (right padded with 0-byte)</span>
<span class="c1">-- The actual size is stored in chunks.data_size</span>
<span class="k">CREATE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">chunk_data</span><span class="p">(</span><span class="w"></span>
<span class="w"> </span><span class="n">id</span><span class="w"> </span><span class="nb">INTEGER</span><span class="w"> </span><span class="k">PRIMARY</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="n">AUTOINCREMENT</span><span class="w"> </span><span class="k">CHECK</span><span class="w"> </span><span class="p">(</span><span class="n">id</span><span class="w"> </span><span class="o">></span><span class="w"> </span><span class="mi">0</span><span class="p">),</span><span class="w"></span>
<span class="w"> </span><span class="k">data</span><span class="w"> </span><span class="nb">BLOB</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="w"></span>
<span class="p">);</span><span class="w"></span>
<span class="c1">-- Record chunks of data by (url, offset)</span>
<span class="k">CREATE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">chunks</span><span class="p">(</span><span class="w"></span>
<span class="w"> </span><span class="n">id</span><span class="w"> </span><span class="nb">INTEGER</span><span class="w"> </span><span class="k">PRIMARY</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="n">AUTOINCREMENT</span><span class="w"> </span><span class="k">CHECK</span><span class="w"> </span><span class="p">(</span><span class="n">id</span><span class="w"> </span><span class="o">></span><span class="w"> </span><span class="mi">0</span><span class="p">),</span><span class="w"></span>
<span class="w"> </span><span class="n">url</span><span class="w"> </span><span class="nb">TEXT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="k">offset</span><span class="w"> </span><span class="nb">INTEGER</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">data_id</span><span class="w"> </span><span class="nb">INTEGER</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">data_size</span><span class="w"> </span><span class="nb">INTEGER</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="k">CONSTRAINT</span><span class="w"> </span><span class="n">fk_chunks_url</span><span class="w"> </span><span class="k">FOREIGN</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="w"> </span><span class="k">REFERENCES</span><span class="w"> </span><span class="n">properties</span><span class="p">(</span><span class="n">url</span><span class="p">),</span><span class="w"></span>
<span class="w"> </span><span class="k">CONSTRAINT</span><span class="w"> </span><span class="n">fk_chunks_data</span><span class="w"> </span><span class="k">FOREIGN</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="p">(</span><span class="n">data_id</span><span class="p">)</span><span class="w"> </span><span class="k">REFERENCES</span><span class="w"> </span><span class="n">chunk_data</span><span class="p">(</span><span class="n">id</span><span class="p">)</span><span class="w"></span>
<span class="p">);</span><span class="w"></span>
<span class="k">CREATE</span><span class="w"> </span><span class="k">INDEX</span><span class="w"> </span><span class="n">idx_chunks</span><span class="w"> </span><span class="k">ON</span><span class="w"> </span><span class="n">chunks</span><span class="p">(</span><span class="n">url</span><span class="p">,</span><span class="w"> </span><span class="k">offset</span><span class="p">);</span><span class="w"></span>
<span class="c1">-- Doubly linked list of chunks. The next link is to go to the least-recently</span>
<span class="c1">-- used entries.</span>
<span class="k">CREATE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">linked_chunks</span><span class="p">(</span><span class="w"></span>
<span class="w"> </span><span class="n">id</span><span class="w"> </span><span class="nb">INTEGER</span><span class="w"> </span><span class="k">PRIMARY</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="n">AUTOINCREMENT</span><span class="w"> </span><span class="k">CHECK</span><span class="w"> </span><span class="p">(</span><span class="n">id</span><span class="w"> </span><span class="o">></span><span class="w"> </span><span class="mi">0</span><span class="p">),</span><span class="w"></span>
<span class="w"> </span><span class="n">chunk_id</span><span class="w"> </span><span class="nb">INTEGER</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">prev</span><span class="w"> </span><span class="nb">INTEGER</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="k">next</span><span class="w"> </span><span class="nb">INTEGER</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="k">CONSTRAINT</span><span class="w"> </span><span class="n">fk_links_chunkid</span><span class="w"> </span><span class="k">FOREIGN</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="p">(</span><span class="n">chunk_id</span><span class="p">)</span><span class="w"> </span><span class="k">REFERENCES</span><span class="w"> </span><span class="n">chunks</span><span class="p">(</span><span class="n">id</span><span class="p">),</span><span class="w"></span>
<span class="w"> </span><span class="k">CONSTRAINT</span><span class="w"> </span><span class="n">fk_links_prev</span><span class="w"> </span><span class="k">FOREIGN</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="p">(</span><span class="n">prev</span><span class="p">)</span><span class="w"> </span><span class="k">REFERENCES</span><span class="w"> </span><span class="n">linked_chunks</span><span class="p">(</span><span class="n">id</span><span class="p">),</span><span class="w"></span>
<span class="w"> </span><span class="k">CONSTRAINT</span><span class="w"> </span><span class="n">fk_links_next</span><span class="w"> </span><span class="k">FOREIGN</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="p">(</span><span class="k">next</span><span class="p">)</span><span class="w"> </span><span class="k">REFERENCES</span><span class="w"> </span><span class="n">linked_chunks</span><span class="p">(</span><span class="n">id</span><span class="p">)</span><span class="w"></span>
<span class="p">);</span><span class="w"></span>
<span class="k">CREATE</span><span class="w"> </span><span class="k">INDEX</span><span class="w"> </span><span class="n">idx_linked_chunks_chunk_id</span><span class="w"> </span><span class="k">ON</span><span class="w"> </span><span class="n">linked_chunks</span><span class="p">(</span><span class="n">chunk_id</span><span class="p">);</span><span class="w"></span>
<span class="c1">-- Head and tail pointers of the linked_chunks. The head pointer is for</span>
<span class="c1">-- the most-recently used chunk.</span>
<span class="c1">-- There should be just one row in this table.</span>
<span class="k">CREATE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">linked_chunks_head_tail</span><span class="p">(</span><span class="w"></span>
<span class="w"> </span><span class="n">head</span><span class="w"> </span><span class="nb">INTEGER</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="n">tail</span><span class="w"> </span><span class="nb">INTEGER</span><span class="p">,</span><span class="w"></span>
<span class="w"> </span><span class="k">CONSTRAINT</span><span class="w"> </span><span class="n">lht_head</span><span class="w"> </span><span class="k">FOREIGN</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="p">(</span><span class="n">head</span><span class="p">)</span><span class="w"> </span><span class="k">REFERENCES</span><span class="w"> </span><span class="n">linked_chunks</span><span class="p">(</span><span class="n">id</span><span class="p">),</span><span class="w"></span>
<span class="w"> </span><span class="k">CONSTRAINT</span><span class="w"> </span><span class="n">lht_tail</span><span class="w"> </span><span class="k">FOREIGN</span><span class="w"> </span><span class="k">KEY</span><span class="w"> </span><span class="p">(</span><span class="n">tail</span><span class="p">)</span><span class="w"> </span><span class="k">REFERENCES</span><span class="w"> </span><span class="n">linked_chunks</span><span class="p">(</span><span class="n">id</span><span class="p">)</span><span class="w"></span>
<span class="p">);</span><span class="w"></span>
<span class="k">INSERT</span><span class="w"> </span><span class="k">INTO</span><span class="w"> </span><span class="n">linked_chunks_head_tail</span><span class="w"> </span><span class="k">VALUES</span><span class="w"> </span><span class="p">(</span><span class="k">NULL</span><span class="p">,</span><span class="w"> </span><span class="k">NULL</span><span class="p">);</span><span class="w"></span>
</pre></div>
</div>
<p>The chunks table will store 16 KB chunks (or less for terminating chunks).
The linked_chunks and linked_chunks_head_tail table swill act as a doubly linked
list of chunks, with the least recently used ones at the end of the list, which
will be evicted when the cache saturates.</p>
<p>The directory used to locate this database will be ${XDG_DATA_HOME}/proj
(per <a class="reference external" href="https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html">https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html</a>)
where ${XDG_DATA_HOME} defaults to ${HOME}/.local/share on Unix builds
and ${LOCALAPPDATA} on Windows builds. Exact details to be sorted out, but
<a class="reference external" href="https://github.com/ActiveState/appdirs/blob/a54ea98feed0a7593475b94de3a359e9e1fe8fdb/appdirs.py#L45-L97">https://github.com/ActiveState/appdirs/blob/a54ea98feed0a7593475b94de3a359e9e1fe8fdb/appdirs.py#L45-L97</a>
can be a good reference.</p>
<p>As this database might be accessed by several threads or processes at the same
time, the code accessing to it will carefully honour SQLite3 errors regarding
to locks, to do appropriate retries if another thread/process is currently
locking the database. Accesses requiring a modification of the database will
start with a BEGIN IMMEDIATE transaction so as to acquire a write lock.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This database should be hosted on a local disk, not a network one.
Otherwise SQLite3 locking issues are to be expected.</p>
</div>
</section>
<section id="cdn-provider">
<h3>CDN provider<a class="headerlink" href="#cdn-provider" title="Permalink to this headline">¶</a></h3>
<p><a class="reference external" href="https://aws.amazon.com/opendata/public-datasets/">Amazon Public Datasets</a>
has offered to be a storage and CDN provider.</p>
<p>The program covers storage and egress (bandwidth) of the data.
They generally don’t allow usage of CloudFront
(their CDN) as part of the program (we would usually look to have it covered
by credits), but in this instance, they would be fine to provide it.
They’d only ask that we keep the CloudFront URL “visible” (as appropriate for
the use case) so people can see where the data is hosted in case they go looking.
Their terms can be seen at <a class="reference external" href="https://aws.amazon.com/service-terms/">https://aws.amazon.com/service-terms/</a> and CloudFront
has its own, small section. Those terms may change a bit from time to time for
minor changes. Major changing service terms is assumed to be unfrequent.
There are also the Public Dataset Program terms at <a class="reference external" href="http://aws.amazon.com/public-datasets/terms/">http://aws.amazon.com/public-datasets/terms/</a>.
Those also do not effectively change over time and are renewed on a 2 year basis.</p>
</section>
<section id="criteria-for-grid-hosting">
<h3>Criteria for grid hosting<a class="headerlink" href="#criteria-for-grid-hosting" title="Permalink to this headline">¶</a></h3>
<p>The grids hosted on the CDN will be exactly the ones collected,
currently and in the future, by the <a class="reference external" href="https://github.com/OSGeo/proj-datumgrid/">proj-datumgrid</a>
initiative. In particular, new grids are accepted as long as
they are released under a license that is compatible with the
<a class="reference external" href="https://opensource.org/osd-annotated">Open Source Definition</a> and the source
of the grid is clearly stated and verifiable. Suitable licenses include:</p>
<ul class="simple">
<li><p>Public domain</p></li>
<li><p>X/MIT</p></li>
<li><p>BSD 2/3/4 clause</p></li>
<li><p>CC0</p></li>
<li><p>CC-BY (v3.0 or later)</p></li>
<li><p>CC-BY-SA (v3.0 or later)</p></li>
</ul>
<p>For new grids to be transparently used by the proj_create_crs_to_crs() mechanics,
they must be registered in the PROJ database (proj.db) in the <code class="docutils literal notranslate"><span class="pre">grid_transformation</span></code> and
<code class="docutils literal notranslate"><span class="pre">grid_alternatives</span></code> table. The nominal path to have a new record in the grid_transformation
is to have a transformation being registered in the EPSG dataset (if there is no
existing one), which will be subsequently imported into the PROJ database.</p>
</section>
<section id="versioning-historical-preservation-of-grids">
<h3>Versioning, historical preservation of grids<a class="headerlink" href="#versioning-historical-preservation-of-grids" title="Permalink to this headline">¶</a></h3>
<p>The policy regarding this should be similar to the one applied to
<a class="reference external" href="https://github.com/OSGeo/proj-datumgrid/">proj-datumgrid</a>, which even if
not formalized, is around the following lines:</p>
<ul class="simple">
<li><p>Geodetic agencies release regularly new version of grids. Typically for the
USA, NOAA has released GEOID99, GEOID03, GEOID06, GEOID09, GEOID12A, GEOID12B,
GEOID18 for the NAVD88 to NAD83/NAD83(2011) vertical adjustments. Each of these
grids is considered by EPSG and PROJ has a separate object, with distinct filenames.
The release of a new version does not cause the old grid to be automatically removed.
That said, due to advertized accuracies and supersession rules of the EPSG dataset, the
most recent grid will generally be used for a CRS -> CRS transformation if the
user uses proj_create_crs_to_crs() (with the exception that if a VERT_CRS WKT
includes a GEOID_MODEL known to PROJ, an old version of the grid will be used).
If the user specifies a whole pipeline with an explicit grid name, it will be
of course strictly honoured.
As time goes, the size of the datasets managed by proj-datumgrid will be increasing,
we will have to explore on we managed that for the distributed .zip / .tar.gz
archives. This should not be a concern for CDN hosted content.</p></li>
<li><p>In case software-related conversion errors from the original grid format to the
one used by PROJ (be it GTX, NTv2 or GeoTIFF) would happen, the previous erroneous
version of the dataset would be replaced by the corrected one. In that situation,
this might have an effect with the local on-disk caching of remote grids. We will
have to see with the CDN providers used if we can use for example the ETag HTTP header
on the client to detect a change, so that old cached content is not erroneously
reused (if not possible, we’ll have to use some text file listing the grid names and their
current md5sum)</p></li>
</ul>
</section>
</section>
<section id="grids-in-geotiff-format">
<h2>Grids in GeoTIFF format<a class="headerlink" href="#grids-in-geotiff-format" title="Permalink to this headline">¶</a></h2>
<section id="limitations-of-current-formats">
<h3>Limitations of current formats<a class="headerlink" href="#limitations-of-current-formats" title="Permalink to this headline">¶</a></h3>
<p>Several formats exist depending on the ad-hoc needs and ideas of the original
data producer. It would be appropriate to converge on a common format able to
address the different use cases.</p>
<ul class="simple">
<li><p>Not tiled. Tiling is a nice to have property for cloud-friendly access to
large files.</p></li>
<li><p>No support for compression</p></li>
<li><p>The NTv2 structures is roughly: header of main grid, data of main grid,
header of subgrid 1, data of subgrid 1, header of subgrid 2, data of subgrid 2,
etc.Due to the headers being scattered through the file, it is not possibly
to retrieve with a single HTTP GET request all header information.</p></li>
<li><p>GTX format has no provision to store metadata besides the minimum georeferencing
of the grid. NTv2 is a bit richer, but no extensible metadata possible.</p></li>
</ul>
</section>
<section id="discussion-on-choice-of-format">
<h3>Discussion on choice of format<a class="headerlink" href="#discussion-on-choice-of-format" title="Permalink to this headline">¶</a></h3>
<p>We have been made recently aware of other initiatives from the industry to come
with a common format to store geodetic adjustment data. Some discussions have
happen recently within the OGC CRS Working group. Past efforts include the
Esri’s proposed Geodetic data Grid eXchange Format, GGXF, briefly mentioned at
page 86 of
<a class="reference external" href="https://iag.dgfi.tum.de/fileadmin/IAG-docs/Travaux2015/01_Travaux_Template_Comm_1_tvd.pdf">https://iag.dgfi.tum.de/fileadmin/IAG-docs/Travaux2015/01_Travaux_Template_Comm_1_tvd.pdf</a>
and page 66 of <a class="reference external" href="ftp://ftp.iaspei.org/pub/meetings/2010-2019/2015-Prague/IAG-Geodesy.pdf">ftp://ftp.iaspei.org/pub/meetings/2010-2019/2015-Prague/IAG-Geodesy.pdf</a>
The current trend of those works would be to use a netCDF / HDF5 container.</p>
<p>So, for the sake of completeness, we list hereafter a few potential candidate
formats and their pros and cons.</p>
<section id="tiff-geotiff">
<h4>TIFF/GeoTIFF<a class="headerlink" href="#tiff-geotiff" title="Permalink to this headline">¶</a></h4>
<p>Strong points:</p>
<ul class="simple">
<li><p>TIFF is a well-known and widespread format.</p></li>
<li><p>The GeoTIFF encoding is a widely industry supported scheme to encode georeferencing.
It is now a <a class="reference external" href="https://www.opengeospatial.org/standards/geotiff">OGC standard</a></p></li>
<li><p>There are independent initiatives to share grids as GeoTIFF, like
<a class="reference external" href="https://www.agisoft.com/downloads/geoids/">that one</a></p></li>
<li><p>TIFF can contain multiple images (IFD: Image File Directory) chained together.
This is the mechanism used for multiple-page scanned TIFF files, or in the
geospatial field to store multi-resolution/pyramid rasters. So it can be
used with sub-grids as in the NTv2 format.</p></li>
<li><p>Extensive experience with the TIFF format, and its appropriateness for network
access, in particular through the <a class="reference external" href="https://www.cogeo.org/">Cloud Optimized GeoTIFF initiative</a>
whose layout can make use of sub-grids efficient from a network access
perspective, because grid headers can be put at the beginning of the file, and
so being retrieved in a single HTTP GET request.</p></li>
<li><p>TIFF can be tiled.</p></li>
<li><p>TIFF can be compressed. Commonly found compression formats are DEFLATE, LZW,
combined with differential integer or floating point predictors</p></li>
<li><p>A TIFF image can contain a configurable number of channels/bands/samples.
In the rest of the document, we will use the sample terminology for this concept.</p></li>
<li><p>TIFF sample organization can be configured: either the values of different
samples are packed together (<a class="reference external" href="https://www.awaresystems.be/imaging/tiff/tifftags/planarconfiguration.html">PlanarConfiguration</a> = Contig), or put in separate tiles/strips
(PlanarConfiguration = Separate)</p></li>
<li><p>libtiff is a dependency commonly found in binary distributions of the
“ecosystem” to which PROJ belongs too</p></li>
<li><p>libtiff benefits from many years of efforts to increase its security, for
example being integrated to the oss-fuzz initiative. Given the potential
fetching of grids, using security tested components is an important concern.</p></li>
<li><p>Browser-side: there are “ports” of libtiff/libgeotiff in the browser such
as <a class="reference external" href="https://geotiffjs.github.io/">https://geotiffjs.github.io/</a> which could potentially make a port of PROJ
easier.</p></li>
</ul>
<p>Weak points:</p>
<ul class="simple">
<li><p>we cannot use libgeotiff, since it depends itself on PROJ (to resolve CRS
or components of CRS from their EPSG codes). That said, for PROJ intended
use, we only need to decode the ModelTiepointTag and ModelPixelScaleTag TIFF
tags, so this can be done “at hand”</p></li>
<li><p>the metadata capabilities of TIFF baseline are limited. The TIFF format comes
with a predefined set of metadata items whose keys have numeric values. That
said, GDAL has used for the last 20 years or so a dedicated tag,
<a class="reference external" href="https://www.awaresystems.be/imaging/tiff/tifftags/gdal_metadata.html">GDAL_METADATA</a>
of code 42112 that holds a XML-formatted string being able to store arbitrary
key-pair values.</p></li>
</ul>
</section>
<section id="netcdf-v3">
<h4>netCDF v3<a class="headerlink" href="#netcdf-v3" title="Permalink to this headline">¶</a></h4>
<p>Strong points:</p>
<ul class="simple">
<li><p>The binary format description as given in
<a class="reference external" href="http://portal.opengeospatial.org/files/?artifact_id=43734">OGC 10-092r3</a> is relatively simple,
but it would still probably be necessary to use libnetcdf-c to access it</p></li>
<li><p>Metadata can be stored easily in netCDF attributes</p></li>
</ul>
<p>Weak points:</p>
<ul class="simple">
<li><p>No compression in netCDF v3</p></li>
<li><p>No tiling in netCDF v3</p></li>
<li><p>Multi-samples variables are located in different sections of the files
(correspond to TIFF PlanarConfiguration = Separate)</p></li>
<li><p>No natural way of having hierarchical / multigrids. They must be encoded as
separate variables</p></li>
<li><p>georeferencing in netCDF is somewhat less standardized than TIFF/GeoTIFF.
The generally used model is <a class="reference external" href="http://cfconventions.org/">the conventions for CF (Climate and Forecast)
metadata</a>
but there is nothing really handy in them for simple georeferencing with
the coordinate of the upper-left pixel and the resolution. The practice is
to write explicit lon and lat variables with all values taken by the grid.
GDAL has for many years supported a simpler syntax, using a GeoTransform
attribute.</p></li>
<li><p>From the format description, its layout could be relatively cloud friendly,
except that libnetcdf has no API to plug an alternate I/O layer.</p></li>
<li><p>Most binary distributions of netCDF nowadays are based on libnetcdf v4, which
implies the HDF5 dependency.</p></li>
<li><p>From a few issues we identified a few years ago regarding crashes on corrupted
datasets, we contacted libnetcdf upstream, but they did not seem to be
interested in addressing those security issues.</p></li>
</ul>
</section>
<section id="netcdf-v4-hdf5">
<h4>netCDF v4 / HDF5<a class="headerlink" href="#netcdf-v4-hdf5" title="Permalink to this headline">¶</a></h4>
<p>Note: The netCDF v4 format is a profile of the HDF5 file format.</p>
<p>Strong points:</p>
<ul class="simple">
<li><p>Compression supported (ZLIB and SZIP predefined)</p></li>
<li><p>Tiling (chunking) supported</p></li>
<li><p>Values of Multi-sample variables can be interleaved together (similarly
to TIFF PlanarConfiguration = Contig) by using compound data types.</p></li>
<li><p>Hierarchical organization with groups</p></li>
<li><p>While the netCDF API does not provide an alternate I/O layer, this is
possible with the HDF5 API.</p></li>
<li><p>Grids can be indexed by more than 2 dimensions (for current needs, we
don’t need more than 2D support)</p></li>
</ul>
<p>Weak points:</p>
<ul class="simple">
<li><p>The <a class="reference external" href="https://support.hdfgroup.org/HDF5/doc/H5.format.html">HDF 5 File format</a>
is more complex than netCDF v3, and likely more than TIFF. We do not have
in-depth expertise of it to assess its cloud-friendliness.</p></li>
<li><p>The ones mentioned for netCDF v3 regarding georeferencing and security apply.</p></li>
</ul>
</section>
<section id="geopackage">
<h4>GeoPackage<a class="headerlink" href="#geopackage" title="Permalink to this headline">¶</a></h4>
<p>As PROJ has already a SQLite3 dependency, GeoPackage could be examined as a
potential solution.</p>
<p>Strong points:</p>
<ul class="simple">
<li><p>SQLite3 dependency</p></li>
<li><p>OGC standard</p></li>
<li><p>Multi-grid capabilities</p></li>
<li><p>Tiling</p></li>
<li><p>Compression</p></li>
<li><p>Metadata capabilities</p></li>
</ul>
<p>Weak points:</p>
<ul class="simple">
<li><p>GeoPackage mostly address the RGB(A) Byte use case, or via the tile gridded
data extension, single-sample non-Byte data. No native support for multi-sample
non-Byte data: each sample should be put in a separate raster table.</p></li>
<li><p>Experience shows that SQLite3 layout (at least the layout adopted when using
the standard libsqlite3) is not cloud friendly. Indices may be scattered in
different places of the file.</p></li>
</ul>
</section>
<section id="conclusions">
<h4>Conclusions<a class="headerlink" href="#conclusions" title="Permalink to this headline">¶</a></h4>
<p>The 2 major contenders regarding our goals and constraints are GeoTIFF and HDF5.
Given past positive experience and its long history, GeoTIFF remains our preferred
choice.</p>
</section>
</section>
<section id="format-description">
<span id="description-geotiff-format"></span><h3>Format description<a class="headerlink" href="#format-description" title="Permalink to this headline">¶</a></h3>
<p>The format description is available in a dedicated <a class="reference internal" href="../../specifications/geodetictiffgrids.html#geodetictiffgrids"><span class="std std-ref">Geodetic TIFF grids (GTG)</span></a>
document.</p>
</section>
<section id="tooling">
<h3>Tooling<a class="headerlink" href="#tooling" title="Permalink to this headline">¶</a></h3>
<p>A script will be developed to accept a list of individual grids to combine
together into a single file.</p>
<p>A ntv2_to_gtiff.py convenience script will be created to convert NTv2 grids,
including their subgrids, to the above
described GeoTIFF layout.</p>
<p>A validation Python script will be created to check that a file meets the above
described requirements and recommendations.</p>
</section>
<section id="build-requirements">
<h3>Build requirements<a class="headerlink" href="#build-requirements" title="Permalink to this headline">¶</a></h3>
<p>The minimum libtiff version will be 4.0 (RHEL 7 ships with libtiff 4.0.3).
To be able to read grids stored on the CDN, libtiff will need to build against
zlib to have DEFLATE and LZW support, which is met by all known binary distributions
of libtiff.</p>
<p>The libtiff dependency can be disabled at build time, but this must be
an explicit setting of configure/cmake as the resulting builds have less functionality.</p>
</section>
</section>
<section id="dropping-grid-catalog-functionality">
<h2>Dropping grid catalog functionality<a class="headerlink" href="#dropping-grid-catalog-functionality" title="Permalink to this headline">¶</a></h2>
<p>While digging through existing code, I more or less discovered that the PROJ
code base has the concept of a grid catalog. This is a feature apparently triggered by
using the +catalog=somefilename.csv in a PROJ string, where the CSV file list
grid names, their extent, priority and date. It seems to be an alternative to using
+nadgrids with multiple grids, with the extra ability to interpolate shift values between
several grids if a +date parameter is provided and the grid catalog mentions a
date for each grids.
It was added in June 2012 per <a class="reference external" href="https://github.com/OSGeo/PROJ/commit/fcb186942ec8532655ff6cf4cc990e5da669a3bc">commit fcb186942ec8532655ff6cf4cc990e5da669a3bc</a></p>
<p>This feature is likely unknown to most users as there is no known documentation for
it (neither in current documentation, nor in <a class="reference external" href="https://web.archive.org/web/20160601000000*/http://trac.osgeo.org/proj/wiki/GenParms">historic one</a>).
It is not either tested by PROJ tests, so its working status is unknown. It would
likely make implementation of this RFC easier if this was removed. This would result in
completely dropping the gridcatalog.cpp and gc_reader.cpp files, their call sites
and the catalog_name and datum_date parameter from the PJ structure.</p>
<p>In case similar functionality would be be needed, it might be later reintroduced
as an extra mode of <a class="reference internal" href="../../operations/transformations/hgridshift.html#hgridshift"><span class="std std-ref">Horizontal grid shift</span></a>, or using a dedicated transformation method,
similarly to the <a class="reference internal" href="../../operations/transformations/deformation.html#deformation"><span class="std std-ref">Kinematic datum shifting utilizing a deformation model</span></a> one,
and possibly combining the several grids to interpolate among in the same file,
with a date metadata item.</p>
</section>
<section id="backward-compatibility-issues">
<h2>Backward compatibility issues<a class="headerlink" href="#backward-compatibility-issues" title="Permalink to this headline">¶</a></h2>
<p>None anticipated, except the removal of the (presumably little used) grid catalog
functionality.</p>
</section>
<section id="potential-future-related-work">
<h2>Potential future related work<a class="headerlink" href="#potential-future-related-work" title="Permalink to this headline">¶</a></h2>
<p>The foundations set in the definition of the GeoTIFF grid format should hopefully
be reused to extend them to support deformation models (was initially discussed
per <a class="reference external" href="https://github.com/OSGeo/PROJ/issues/1001">https://github.com/OSGeo/PROJ/issues/1001</a>).</p>
<p>Definition of such an extension is out of scope of this RFC.</p>
</section>
<section id="documentation">
<h2>Documentation<a class="headerlink" href="#documentation" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>New API function will be documented.</p></li>
<li><p>A dedicated documentation page will be created to explain the working of
network-based access.</p></li>
<li><p>A dedicated documentation page will be created to describe the GeoTIFF based
grid format. Mostly reusing above material.</p></li>
</ul>
</section>
<section id="testing">
<h2>Testing<a class="headerlink" href="#testing" title="Permalink to this headline">¶</a></h2>
<p>Number of GeoTIFF formulations (tiled vs untiled, PlanarConfiguration Separate vs
Contig, data types, scale+offset vs not, etc.) will be tested.</p>
<p>For testing of network capabilities, a mix of real hits to the CDN and use of
the alternate pluggable network interface to test edge cases will be used.</p>
</section>
<section id="proposed-implementation">
<h2>Proposed implementation<a class="headerlink" href="#proposed-implementation" title="Permalink to this headline">¶</a></h2>
<p>A proposed implementation is available at <a class="reference external" href="https://github.com/OSGeo/PROJ/pull/1817">https://github.com/OSGeo/PROJ/pull/1817</a></p>
<p>Tooling scripts are currently available at <a class="reference external" href="https://github.com/rouault/sample_proj_gtiff_grids/">https://github.com/rouault/sample_proj_gtiff_grids/</a>
(will be ultimately stored in PROJ repository)</p>
</section>
<section id="adoption-status">
<h2>Adoption status<a class="headerlink" href="#adoption-status" title="Permalink to this headline">¶</a></h2>
<p>The RFC was adopted on 2020-01-10 with +1’s from the following PSC members</p>
<ul class="simple">
<li><p>Kristian Evers</p></li>
<li><p>Even Rouault</p></li>
<li><p>Thomas Knudsen</p></li>
<li><p>Howard Butler</p></li>
<li><p>Kurt Schwehr</p></li>
</ul>
</section>
</section>
</div>
</div>
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
<a href="rfc-3.html" class="btn btn-neutral float-left" title="PROJ RFC 3: Dependency management" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
<a href="rfc-5.html" class="btn btn-neutral float-right" title="PROJ RFC 5: Adopt GeoTIFF-based grids for grids delivered with PROJ" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
</div>
<hr/>
<div role="contentinfo">
<p>© Copyright 1983-2022.
<span class="lastupdated">Last updated on 22 Mar 2022.
</span></p>
</div>
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script>
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>
|
