-
Notifications
You must be signed in to change notification settings - Fork 266
/
Copy pathnch5s3comms.c
2913 lines (2541 loc) · 101 KB
/
nch5s3comms.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* ********************************************************************/
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://www.hdfgroup.org/licenses. *
* If you do not have access to either file, you may request a copy from *
* [email protected]. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*****************************************************************************
* Read-Only S3 Virtual File Driver (VFD)
* Source for S3 Communications module
* ***NOT A FILE DRIVER***
* Provide functions and structures required for interfacing with Amazon
* Simple Storage Service (S3).
* Provide S3 object access as if it were a local file.
* Connect to remote host, send and receive HTTP requests and responses
* as part of the AWS REST API, authenticating requests as appropriate.
* Programmer: Jacob Smith
* 2017-11-30
*****************************************************************************/
/**
* Unidata Changes:
* Derived from HDF5-1.14.0 H5FDs3comms.[ch]
* Modified to support Write operations and support NCZarr.
* Primary Changes:
* - rename H5FD_s3comms to NCH5_s3comms to avoid name conflicts
* - Remove HDF5 dependencies
* - Support zmap API
*
* Note that this code is very ugly because it is the bastard
* child of the HDF5 coding style and the NetCDF-C coding style
* and some libcurl as well.
*
* A note about the nccurl_hmac.c and nccurl_sha256.c files.
* The code in this file depends on having access to two
* cryptographic functions:
* 1. HMAC signing function
* 2. SHA256 digest function
*
* There are a number of libraries providing these functions.
* For example, OPENSSL, WOLFSSL, GNUTLS, Windows crypto package
* etc. It turns out that libcurl has identified all of these
* possible sources and set up a wrapper to handle the
* possibilities. So, this code copies the libcurl wrapper to
* inherit its multi-source capabilities.
*
* Author: Dennis Heimbigner
* Creation Date: 2/12/2023
* Last Modified: 5/1/2023
*/
/****************/
/* Module Setup */
/****************/
/***********/
/* Headers */
/***********/
/*****************/
#include "config.h"
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif
#ifdef HAVE_STRING_H
#include <string.h>
#endif
#ifdef HAVE_CTYPE_H
#include <ctype.h>
#endif
#include <assert.h>
#include "nccurl_sha256.h"
#include "nccurl_hmac.h"
/* Necessary S3 headers */
#include <curl/curl.h>
//#include <openssl/evp.h>
//#include <openssl/hmac.h>
//#include <openssl/sha.h>
#include "netcdf.h"
#include "ncuri.h"
#include "ncutil.h"
/*****************/
#include "ncs3sdk.h"
#include "nch5s3comms.h" /* S3 Communications */
/****************/
/* Local Macros */
/****************/
#undef TRACING
#undef DEBUG
#define SUCCEED NC_NOERR
#ifndef TRUE
#define TRUE 1
#define FALSE 0
#endif
/* enable debugging */
#define S3COMMS_DEBUG 0
#define S3COMMS_DEBUG_TRACE 0
/* manipulate verbosity of CURL output
* operates separately from S3COMMS_DEBUG
* 0 -> no explicit curl output
* 1 -> on error, print failure info to stderr
* 2 -> in addition to above, print information for all performs; sets all
* curl handles with CURLOPT_VERBOSE
*/
#define S3COMMS_CURL_VERBOSITY 0
/* Apparently Apple/OSX C Compiler does not (yet) accept __VA_OPT__(,),
so we have to case it out (ugh!)
*/
#if S3COMMS_CURL_VERBOSITY > 1
#define HDONE_ERROR(ignore1,ncerr,ignore2,msg) do {ret_value=report(ncerr,__func__,__LINE__,msg);} while(0)
#define HDONE_ERRORVA(ignore1,ncerr,ignore2,msg,...) do {ret_value=report(ncerr,__func__,__LINE__,msg, __VA_ARGS__);} while(0)
#define HGOTO_ERROR(ignore1,ncerr,ignore2,msg,...) do {ret_value=report(ncerr,__func__,__LINE__,msg); goto done;} while(0)
#define HGOTO_ERRORVA(ignore1,ncerr,ignore2,msg,...) do {ret_value=report(ncerr,__func__,__LINE__,msg, __VA_ARGS__); goto done;} while(0)
#else /*S3COMMS_CURL_VERBOSITY*/
#define HDONE_ERROR(ignore1,ncerr,ignore2,msg,...) do {ret_value=(ncerr);} while(0)
#define HDONE_ERRORVA(ignore1,ncerr,ignore2,msg,...) HDONE_ERROR(ignore1,ncerr,ignore2,msg)
#define HGOTO_ERROR(ignore1,ncerr,ignore2,msg,...) do {ret_value=(ncerr);; goto done;} while(0)
#define HGOTO_ERRORVA(ignore1,ncerr,ignore2,msg,...) HGOTO_ERROR(ignore1,ncerr,ignore2,msg)
#endif /*S3COMMS_CURL_VERBOSITY*/
/* size to allocate for "bytes=<first_byte>[-<last_byte>]" HTTP Range value
*/
#define S3COMMS_MAX_RANGE_STRING_SIZE 128
#define SNULL(x) ((x)==NULL?"NULL":(x))
#define INULL(x) ((x)==NULL?-1:(int)(*x))
#ifdef TRACING
#define TRACE(level,fmt,...) s3trace((level),__func__,fmt,##__VA_ARGS__)
#define TRACEMORE(level,fmt,...) s3tracemore((level),fmt,##__VA_ARGS__)
#define UNTRACE(e) s3untrace(__func__,NCTHROW(e),NULL)
#define UNTRACEX(e,fmt,...) s3untrace(__func__,NCTHROW(e),fmt,##__VA_ARGS__)
#else
#define TRACE(level,fmt,...)
#define TRACEMORE(level,fmt,...)
#define UNTRACE(e) (e)
#define UNTRACEX(e,fmt,...) (e)
#endif
#ifdef TRACING
static struct S3LOGGLOBAL {
FILE* stream;
int depth;
struct Frame {
const char* fcn;
int level;
int depth;
} frames[1024];
} s3log_global = {NULL,0};
static int
s3breakpoint(int err)
{
return err;
}
static void
s3vtrace(int level, const char* fcn, const char* fmt, va_list ap)
{
struct Frame* frame;
if(s3log_global.stream == NULL) s3log_global.stream = stderr;
if(fcn != NULL) {
frame = &s3log_global.frames[s3log_global.depth];
frame->fcn = fcn;
frame->level = level;
frame->depth = s3log_global.depth;
}
{
if(fcn != NULL)
fprintf(s3log_global.stream,"%s: (%d): %s:","Enter",level,fcn);
if(fmt != NULL)
vfprintf(s3log_global.stream, fmt, ap);
fprintf(s3log_global.stream, "\n" );
fflush(s3log_global.stream);
}
if(fcn != NULL) s3log_global.depth++;
}
static void
s3trace(int level, const char* fcn, const char* fmt, ...)
{
va_list args;
va_start(args, fmt);
s3vtrace(level,fcn,fmt,args);
va_end(args);
}
static void
s3tracemore(int level, const char* fmt, ...)
{
va_list args;
va_start(args, fmt);
s3vtrace(level,NULL,fmt,args);
va_end(args);
}
static int
s3untrace(const char* fcn, int err, const char* fmt, ...)
{
va_list args;
struct Frame* frame;
va_start(args, fmt);
if(s3log_global.depth == 0) {
fprintf(s3log_global.stream,"*** Unmatched untrace: %s: depth==0\n",fcn);
goto done;
}
s3log_global.depth--;
frame = &s3log_global.frames[s3log_global.depth];
if(frame->depth != s3log_global.depth || strcmp(frame->fcn,fcn) != 0) {
fprintf(s3log_global.stream,"*** Unmatched untrace: fcn=%s expected=%s\n",frame->fcn,fcn);
goto done;
}
{
fprintf(s3log_global.stream,"%s: (%d): %s: ","Exit",frame->level,frame->fcn);
if(err)
fprintf(s3log_global.stream,"err=(%d) '%s':",err,nc_strerror(err));
if(fmt != NULL)
vfprintf(s3log_global.stream, fmt, args);
fprintf(s3log_global.stream, "\n" );
fflush(s3log_global.stream);
}
done:
va_end(args);
if(err != 0)
return s3breakpoint(err);
else
return err;
}
#endif
/******************/
/* Local Decls*/
/******************/
#define S3COMMS_VERB_MAX 16
/********************/
/* Local Structures */
/********************/
/* Provide a single, unified argument for curl callbacks */
/* struct s3r_cbstruct
* Structure passed to curl callback
*/
struct s3r_cbstruct {
unsigned long magic;
VString* data;
const char* key; /* headcallback: header search key */
size_t pos; /* readcallback: write from this point in data */
};
#define S3COMMS_CALLBACK_STRUCT_MAGIC 0x28c2b2ul
/********************/
/* Local Prototypes */
/********************/
/* Forward */
static int NCH5_s3comms_s3r_execute(s3r_t *handle, const char* url, HTTPVerb verb, const char* byterange, const char* header, const char** otherheaders, long* httpcodep, VString* data);
static size_t curlwritecallback(char *ptr, size_t size, size_t nmemb, void *userdata);
static size_t curlheadercallback(char *ptr, size_t size, size_t nmemb, void *userdata);
static int curl_reset(s3r_t* handle);
static int perform_request(s3r_t* handle, long* httpcode);
static int build_request(s3r_t* handle, NCURI* purl, const char* byterange, const char** otherheaders, VString* payload, HTTPVerb verb);
static int request_setup(s3r_t* handle, const char* url, HTTPVerb verb, struct s3r_cbstruct*);
static int validate_handle(s3r_t* handle, const char* url);
static int validate_url(NCURI* purl);
static int build_range(size_t offset, size_t len, char** rangep);
static const char* verbtext(HTTPVerb verb);
static int trace(CURL* curl, int onoff);
static int sortheaders(VList* headers);
static int httptonc(long httpcode);
static void hrb_node_free(hrb_node_t *node);
#if S3COMMS_DEBUG_HRB
static void dumphrbnodes(VList* nodes);
static void dumphrb(hrb_t* hrb);
#endif
/*********************/
/* Package Variables */
/*********************/
/*****************************/
/* Library Private Variables */
/*****************************/
/*******************/
/* Local Variables */
/*******************/
/*************/
/* Functions */
/*************/
#if S3COMMS_CURL_VERBOSITY > 0
static void
nch5breakpoint(int stat)
{
if(stat == -78) abort();
ncbreakpoint(stat);
}
static int
report(int stat, const char* fcn, int lineno, const char* fmt, ...)
{
va_list args;
char bigfmt[1024];
if(stat == NC_NOERR) goto done;
snprintf(bigfmt,sizeof(bigfmt),"(%d)%s ; fcn=%s line=%d ; %s",stat,nc_strerror(stat),fcn,lineno,fmt);
va_start(args,fmt);
ncvlog(NCLOGERR,bigfmt,args);
nch5breakpoint(stat);
done:
va_end(args);
return stat;
}
#endif
/*----------------------------------------------------------------------------
* Function: curlwritecallback()
* Purpose:
* Function called by CURL to write received data.
* Writes bytes to `userdata`.
* Internally manages number of bytes processed.
* Return:
* - Number of bytes processed.
* - Should equal number of bytes passed to callback.
* - Failure will result in curl error: CURLE_WRITE_ERROR.
* Programmer: Jacob Smith
* 2017-08-17
*----------------------------------------------------------------------------
*/
static size_t
curlwritecallback(char *ptr, size_t size, size_t nmemb, void *userdata)
{
struct s3r_cbstruct *sds = (struct s3r_cbstruct *)userdata;
size_t product = (size * nmemb);
size_t written = 0;
if (sds->magic != S3COMMS_CALLBACK_STRUCT_MAGIC)
return written;
if (product > 0) {
vsappendn(sds->data,ptr,product);
written = product;
}
return written;
} /* end curlwritecallback() */
/*----------------------------------------------------------------------------
* Function: curlreadcallback()
* Purpose:
* Function called by CURL to write PUT data.
* Reads bytes from `userdata`.
* Internally manages number of bytes processed.
* Return:
* - Number of bytes processed.
* - Should equal number of bytes passed to callback.
* - Failure will result in curl error: CURLE_WRITE_ERROR.
* Programmer: Dennis Heimbigner
*----------------------------------------------------------------------------
*/
static size_t
curlreadcallback(char *ptr, size_t size, size_t nmemb, void *userdata)
{
struct s3r_cbstruct *sds = (struct s3r_cbstruct *)userdata;
size_t product = (size * nmemb);
size_t written = 0;
size_t avail = 0;
size_t towrite = 0;
if (sds->magic != S3COMMS_CALLBACK_STRUCT_MAGIC)
return CURL_READFUNC_ABORT;
avail = (vslength(sds->data) - sds->pos);
towrite = (product > avail ? avail : product);
if (towrite > 0) {
const char* data = vscontents(sds->data);
memcpy(ptr,&data[sds->pos],towrite);
}
sds->pos += towrite;
written = towrite;
return written;
} /* end curlreadcallback() */
/*----------------------------------------------------------------------------
* Function: curlheadercallback()
* Purpose:
* Function called by CURL to write headers.
* Writes target header line to value field;
* Internally manages number of bytes processed.
* Return:
* - Number of bytes processed.
* - Should equal number of bytes passed to callback.
* - Failure will result in curl error: CURLE_WRITE_ERROR.
* Programmer: Dennis Heimbigner
* 2017-08-17
*----------------------------------------------------------------------------
*/
static size_t
curlheadercallback(char *ptr, size_t size, size_t nmemb, void *userdata)
{
struct s3r_cbstruct *sds = (struct s3r_cbstruct *)userdata;
size_t len = (size * nmemb);
char* line = ptr;
size_t i,j;
if (sds->magic != S3COMMS_CALLBACK_STRUCT_MAGIC)
return 0;
if(vslength(sds->data) > 0)
goto done; /* already found */
/* skip leading white space */
for(j=0,i=0;i<len;i++) {if(!isspace(line[i])) {j = i; break;}}
line = line + j;
len -= j;
if(sds->key && strncasecmp(line,sds->key,strlen(sds->key)) == 0) {
vsappendn(sds->data,line,len);
}
done:
return size * nmemb;
} /* end curlwritecallback() */
/*----------------------------------------------------------------------------
* Function: NCH5_s3comms_hrb_node_insert()
* Purpose:
* Insert elements in a field node list.
* `name` cannot be null; will return FAIL and list will be unaltered.
* Entries are accessed via the lowercase representation of their name:
* "Host", "host", and "hOSt" would all access the same node,
* but name's case is relevant in HTTP request output.
*----------------------------------------------------------------------------
*/
int
NCH5_s3comms_hrb_node_insert(VList* list, const char *name, const char *value)
{
size_t i = 0;
int ret,ret_value = SUCCEED;
size_t catlen, namelen;
size_t catwrite;
char* lowername = NULL;
char* nvcat = NULL;
hrb_node_t* new_node = NULL;
#if S3COMMS_DEBUG_HRB
fprintf(stdout, "called NCH5_s3comms_hrb_node_insert.");
printf("NAME: %s\n", name);
printf("VALUE: %s\n", value);
printf("LIST:\n->");
dumphrbnodes(list);
fflush(stdout);
#endif
if (name == NULL)
HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to operate on null name");
namelen = nulllen(name);
/* get lowercase name */
lowername = (char *)malloc(sizeof(char) * (namelen + 1));
if (lowername == NULL)
HGOTO_ERROR(H5E_RESOURCE, NC_ENOMEM, FAIL, "cannot make space for lowercase name copy.");
for (i = 0; i < namelen; i++)
lowername[i] = (char)tolower((int)name[i]);
lowername[namelen] = 0;
if(value == NULL) value = "";
/* create new_node */
new_node = (hrb_node_t *)calloc(1,sizeof(hrb_node_t));
if (new_node == NULL)
HGOTO_ERROR(H5E_RESOURCE, NC_ENOMEM, FAIL, "cannot make space for new set.");
new_node->magic = S3COMMS_HRB_NODE_MAGIC;
new_node->name = strdup(name);
new_node->value = strdup(value);
catlen = namelen + strlen(value) + 2; /* +2 from ": " */
catwrite = catlen + 3; /* 3 not 1 to quiet compiler warning */
nvcat = (char *)malloc(catwrite);
if (nvcat == NULL)
HGOTO_ERROR(H5E_RESOURCE, NC_ENOMEM, FAIL, "cannot make space for concatenated string.");
ret = snprintf(nvcat, catwrite, "%s: %s", lowername, value);
if (ret < 0 || (size_t)ret > catlen)
HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "cannot concatenate `%s: %s", name, value);
assert(catlen == nulllen(nvcat));
new_node->cat = nvcat; nvcat = NULL;
new_node->lowername = lowername; lowername = NULL;
vlistpush(list,new_node); new_node = NULL;
done:
/* clean up */
if (nvcat != NULL) free(nvcat);
if (lowername != NULL) free(lowername);
hrb_node_free(new_node);
return (ret_value);
}
/*----------------------------------------------------------------------------
* Function: NCH5_s3comms_hrb_destroy()
* Purpose:
* Destroy and free resources _directly_ associated with an HTTP Buffer.
* Takes a pointer to pointer to the buffer structure.
* This allows for the pointer itself to be NULLed from within the call.
* If buffer or buffer pointer is NULL, there is no effect.
* Headers list at `first_header` is not touched.
* - Programmer should re-use or destroy `first_header` pointer
* (hrb_node_t *) as suits their purposes.
* - Recommend fetching prior to destroy()
* e.g., `reuse_node = hrb_to_die->first_header; destroy(hrb_to_die);`
* or maintaining an external reference.
* - Destroy node/list separately as appropriate
* - Failure to account for this will result in a memory leak.
* Return:
* - SUCCESS: `SUCCEED`
* - successfully released buffer resources
* - if `buf` is NULL or `*buf` is NULL, no effect
* - FAILURE: `FAIL`
* - `buf->magic != S3COMMS_HRB_MAGIC`
* Programmer: Jacob Smith
* 2017-07-21
*----------------------------------------------------------------------------
*/
int
NCH5_s3comms_hrb_destroy(hrb_t *buf)
{
int ret_value = SUCCEED;
size_t i;
#if S3COMMS_DEBUG_TRACE
fprintf(stdout, "called NCH5_s3comms_hrb_destroy.\n");
#endif
if(buf == NULL) return ret_value;
if (buf->magic != S3COMMS_HRB_MAGIC)
HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "pointer's magic does not match.");
free(buf->version);
free(buf->resource);
buf->magic += 1ul;
vsfree(buf->body);
for(i=0;i<vlistlength(buf->headers);i++) {
hrb_node_t* node = (hrb_node_t*)vlistget(buf->headers,i);
hrb_node_free(node);
}
vlistfree(buf->headers);
free(buf);
done:
return (ret_value);
} /* end NCH5_s3comms_hrb_destroy() */
/*----------------------------------------------------------------------------
* Function: NCH5_s3comms_hrb_init_request()
* Purpose:
* Create a new HTTP Request Buffer
* All non-null arguments should be null-terminated strings.
* If `verb` is NULL, defaults to "GET".
* If `http_version` is NULL, defaults to "HTTP/1.1".
* `resource` cannot be NULL; should be string beginning with slash
* character ('/').
* All strings are copied into the structure, making them safe from
* modification in source strings.
* Return:
* - SUCCESS: pointer to new `hrb_t`
* - FAILURE: `NULL`
* Programmer: Jacob Smith
* 2017-07-21
*----------------------------------------------------------------------------
*/
hrb_t *
NCH5_s3comms_hrb_init_request(const char *_resource, const char *_http_version)
{
hrb_t *request = NULL;
char *res = NULL;
size_t reslen = 0;
int ret_value = SUCCEED;
char *vrsn = NULL;
size_t vrsnlen = 0;
#if S3COMMS_DEBUG_TRACE
fprintf(stdout, "called NCH5_s3comms_hrb_init_request.\n");
#endif
if (_resource == NULL)
HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "resource string cannot be null.");
/* populate valid NULLs with defaults */
if (_http_version == NULL)
_http_version = "HTTP/1.1";
/* malloc space for and prepare structure */
request = (hrb_t *)malloc(sizeof(hrb_t));
if (request == NULL)
HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "no space for request structure");
request->magic = S3COMMS_HRB_MAGIC;
request->body = vsnew();
request->headers = vlistnew();
/* malloc and copy strings for the structure */
reslen = nulllen(_resource);
if (_resource[0] == '/') {
res = (char *)malloc(sizeof(char) * (reslen + 1));
if (res == NULL)
HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "no space for resource string");
memcpy(res, _resource, (reslen + 1));
}
else {
res = (char *)malloc(sizeof(char) * (reslen + 2));
if (res == NULL)
HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "no space for resource string");
*res = '/';
memcpy((&res[1]), _resource, (reslen + 1));
assert((reslen + 1) == nulllen(res));
} /* end if (else resource string not starting with '/') */
vrsnlen = nulllen(_http_version) + 1;
vrsn = (char *)malloc(sizeof(char) * vrsnlen);
if (vrsn == NULL)
HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "no space for http-version string");
strncpy(vrsn, _http_version, vrsnlen);
/* place new copies into structure */
request->resource = res;
request->version = vrsn;
done:
/* if there is an error, clean up after ourselves */
if (ret_value != SUCCEED) {
if (request != NULL)
free(request);
if (vrsn != NULL)
free(vrsn);
if (res != NULL)
free(res);
request = NULL;
}
(void)(ret_value);
return request;
} /* end NCH5_s3comms_hrb_init_request() */
#if S3COMMS_DEBUG_HRB
static void
dumphrbnodes(VList* nodes)
{
int i;
if(nodes != NULL) {
fprintf(stderr,"\tnodes={\n");
for(i=0;i<vlistlength(nodes);i++) {
hrb_node_t* node = (hrb_node_t*)vlistget(nodes,i);
fprintf(stderr,"\t\t[%2d] %s=%s\n",i,node->name,node->value);
}
fprintf(stderr,"\t}\n");
}
}
static void
dumphrb(hrb_t* hrb)
{
fprintf(stderr,"hrb={\n");
if(hrb != NULL) {
fprintf(stderr,"\tresource=%s\n",hrb->resource);
fprintf(stderr,"\tversion=%s\n",hrb->version);
fprintf(stderr,"\tbody=|%.*s|\n",(int)ncbyteslength(hrb->body),ncbytescontents(hrb->body));
dumphrbnodes(hrb->headers);
}
fprintf(stderr,"}\n");
}
#endif
static void
hrb_node_free(hrb_node_t *node)
{
if(node != NULL) {
nullfree(node->name);
nullfree(node->value);
nullfree(node->cat);
nullfree(node->lowername);
free(node);
}
}
/****************************************************************************
* S3R FUNCTIONS
****************************************************************************/
/*----------------------------------------------------------------------------
* Function: NCH5_s3comms_s3r_close()
* Purpose:
* Close communications through given S3 Request Handle (`s3r_t`)
* and clean up associated resources.
* Return:
* - SUCCESS: `SUCCEED`
* - FAILURE: `FAIL`
* - fails if handle is null or has invalid magic number
* Programmer: Jacob Smith
* 2017-08-31
*----------------------------------------------------------------------------
*/
int
NCH5_s3comms_s3r_close(s3r_t *handle)
{
int ret_value = SUCCEED;
TRACE(0,"handle=%p",handle);
#if S3COMMS_DEBUG_TRACE
fprintf(stdout, "called NCH5_s3comms_s3r_close.\n");
#endif
if (handle == NULL)
HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle cannot be null.");
if (handle->magic != S3COMMS_S3R_MAGIC)
HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle has invalid magic.");
if(handle->curlheaders != NULL) {
curl_slist_free_all(handle->curlheaders);
handle->curlheaders = NULL;
}
curl_easy_cleanup(handle->curlhandle);
nullfree(handle->rootpath);
nullfree(handle->region);
nullfree(handle->accessid);
nullfree(handle->accesskey);
nullfree(handle->reply);
nullfree(handle->signing_key);
free(handle);
done:
return UNTRACE(ret_value);
} /* NCH5_s3comms_s3r_close */
/*----------------------------------------------------------------------------
* Function: NCH5_s3comms_s3r_getsize()
* Purpose:
* Get the number of bytes of handle's target resource.
* Sets handle and curlhandle with to enact an HTTP HEAD request on file,
* and parses received headers to extract "Content-Length" from response
* headers, storing file size at `handle->filesize`.
* Critical step in opening (initiating) an `s3r_t` handle.
* Wraps `s3r_read()`.
* Sets curlhandle to write headers to a temporary buffer (using extant
* write callback) and provides no buffer for body.
* Upon exit, unsets HTTP HEAD settings from curl handle, returning to
* initial state. In event of error, curl handle state is undefined and is
* not to be trusted.
* Return:
* - SUCCESS: `SUCCEED`
* - FAILURE: `FAIL`
* Programmer: Jacob Smith
* 2017-08-23
*----------------------------------------------------------------------------
*/
int
NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long* sizep)
{
int ret_value = SUCCEED;
char* contentlength = NULL;
char* value = NULL;
long long content_length = -1;
long httpcode = 0;
TRACE(0,"handle=%p url=%s sizep=%p",handle,url,sizep);
#if S3COMMS_DEBUG_TRACE
fprintf(stdout, "called NCH5_s3comms_s3r_getsize.\n");
#endif
if((ret_value = NCH5_s3comms_s3r_head(handle, url, "Content-Length", NULL, &httpcode, &contentlength)))
HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "NCH5_s3comms_s3r_head failed.");
if((ret_value = httptonc(httpcode))) goto done;
/******************
* PARSE RESPONSE *
******************/
value = strchr(contentlength,':');
if(value == NULL)
HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "could not find content length value");
value++;
content_length = strtoumax(value, NULL, 0);
if (UINTMAX_MAX > SIZE_MAX && content_length > SIZE_MAX)
HGOTO_ERROR(H5E_ARGS, NC_ERANGE, FAIL, "content_length overflows size_t");
if (errno == ERANGE) /* errno set by strtoumax*/
HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL,
"could not convert found \"Content-Length\" response (\"%s\")",
contentlength); /* range is null-terminated, remember */
if(sizep) {*sizep = (long long)content_length;}
done:
nullfree(contentlength);
return UNTRACEX(ret_value,"size=%lld",(sizep?-1:*sizep));
} /* NCH5_s3comms_s3r_getsize */
/*----------------------------------------------------------------------------
* Function: NCH5_s3comms_s3r_deletekey()
* Return:
* - SUCCESS: `SUCCEED`
* - FAILURE: `FAIL`
* Programmer: Dennis Heimbigner
*----------------------------------------------------------------------------
*/
int
NCH5_s3comms_s3r_deletekey(s3r_t *handle, const char* url, long* httpcodep)
{
int ret_value = SUCCEED;
VString* data = vsnew();
long httpcode = 0;
TRACE(0,"handle=%p url=%s httpcodep=%p",handle,url,httpcodep);
#if S3COMMS_DEBUG_TRACE
fprintf(stdout, "called NCH5_s3comms_s3r_deletekey.\n");
#endif
/*********************
* Execute *
*********************/
if((ret_value = NCH5_s3comms_s3r_execute(handle, url, HTTPDELETE, NULL, NULL, NULL, &httpcode, data)))
HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed.");
/******************
* RESPONSE *
******************/
if((ret_value = httptonc(httpcode))) goto done;
if(httpcode != 204)
HGOTO_ERROR(H5E_ARGS, NC_ECANTREMOVE, FAIL, "deletekey failed.");
done:
vsfree(data);
if(httpcodep) *httpcodep = httpcode;
return UNTRACEX(ret_value,"httpcode=%d",INULL(httpcodep));
} /* NCH5_s3comms_s3r_getsize */
/*----------------------------------------------------------------------------
* Function: NCH5_s3comms_s3r_head()
* Purpose:
* Generic HEAD request
* @param
* @return NC_NOERR if exists
* @return NC_EINVAL if not exits
* @return error otherwise
*----------------------------------------------------------------------------
*/
int
NCH5_s3comms_s3r_head(s3r_t *handle, const char* url, const char* header, const char* query, long* httpcodep, char** valuep)
{
int ret_value = SUCCEED;
VString* data = vsnew();
long httpcode = 0;
TRACE(0,"handle=%p url=%s header=%s query=%s httpcodep=%p valuep=%p",handle,url,SNULL(header),SNULL(query),httpcodep,valuep);
#if S3COMMS_DEBUG_TRACE
fprintf(stdout, "called NCH5_s3comms_s3r_head.\n");
#endif
if (url == NULL)
HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle has bad (null) url.");
if((ret_value = validate_handle(handle,url)))
HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "invalid handle.");
/*******************
* PERFORM REQUEST *
*******************/
/* only http metadata will be sent by server and recorded by s3comms
*/
if (SUCCEED != NCH5_s3comms_s3r_execute(handle, url, HTTPHEAD, NULL, header, NULL, &httpcode, data))
HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "problem in reading during getsize.");
if((ret_value = httptonc(httpcode))) goto done;
if(header != NULL) {
if(vslength(data) == 0)
HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "HTTP metadata: header=%s; not found",header);
else if (vslength(data) > CURL_MAX_HTTP_HEADER)
HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "HTTP metadata buffer overrun");
#if S3COMMS_DEBUG
else
fprintf(stderr, "HEAD: OK\n");
#endif
}
/******************
* PARSE RESPONSE *
******************/
if(header != NULL) {
char* content;
content = vsextract(data);
if(valuep) {*valuep = content;}
}
/**********************
* UNDO HEAD SETTINGS *
**********************/
if((ret_value = curl_reset(handle)))
HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "error while re-setting CURL options.");
done:
if(httpcodep) *httpcodep = httpcode;
vsfree(data);
return UNTRACEX(ret_value,"httpcodep=%d",INULL(httpcodep));
} /* NCH5_s3comms_s3r_getsize */
/*----------------------------------------------------------------------------
* Function: NCH5_s3comms_s3r_execute()
* Purpose:
* Execute an HTTP verb and optionally return the response.
* Uses configured "curl easy handle" to perform request.
* In event of error, buffer should remain unaltered.
* If handle is set to authorize a request, creates a new (temporary)
* HTTP Request object (hrb_t) for generating requisite headers,
* which is then translated to a `curl slist` and set in the curl handle
* for the request.
* Return:
* - SUCCESS: `SUCCEED`
* - FAILURE: `FAIL`
* Programmer: Dennis Heimbigner
*----------------------------------------------------------------------------
*/
/* TODO: Need to simplify this signature; it is too long */
static int
NCH5_s3comms_s3r_execute(s3r_t *handle, const char* url,
HTTPVerb verb,
const char* range,
const char* searchheader,
const char** otherheaders,
long* httpcodep,
VString* data)
{
int ret_value = SUCCEED;
NCURI* purl= NULL;
struct s3r_cbstruct sds = {S3COMMS_CALLBACK_STRUCT_MAGIC, NULL, NULL, 0};
long httpcode = 0;
#ifdef DEBUG
printf(">>> NCH5_s3comms_s3r_execute(url=%s verb=%s range=%s searchheader=%s)\n",url,verbtext(verb),SNULL(range),SNULL(searchheader));
fflush(stdout);
#endif
#if S3COMMS_DEBUG_TRACE
fprintf(stdout, "called NCH5_s3comms_s3r_execute.\n");
#endif
/**************************************
* ABSOLUTELY NECESSARY SANITY-CHECKS *
**************************************/
if((ret_value = validate_handle(handle, url)))
HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "invalid handle.");
ncuriparse(url,&purl);
if((ret_value = validate_url(purl)))
HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "unparseable url: %s", url);