Skip to content

Commit

Permalink
Convert NCzarr meta-data to use only Zarr attributes
Browse files Browse the repository at this point in the history 
As discussed in a netcdf meeting, convert NCZarr V2 to store all netcdf-4 specific info as attributes. This improves interoperability with other Zarr implementations by no longer using non-standard keys.

## Other Changes
* Remove support for older NCZarr formats.
* Update anonymous dimension naming
* Begin the process of fixing the -Wconversion and -Wsign-compare warnings in libnczarr, nczarr_test, and v3_nczarr_test.
* Update docs/nczarr.md
* Rebuild using the .y and .l files
  • Loading branch information
@DennisHeimbigner
DennisHeimbigner committed Jun 20, 2024
1 parent be009ed commit 076da97
Show file tree
Hide file tree
Showing 50 changed files with 1,249 additions and 933 deletions.
1 change: 1 addition & 0 deletions RELEASE_NOTES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release

## 4.9.3 - TBD

* Convert NCZarr V2 to store all netcdf-4 specific info as attributes. This improves interoperability with other Zarr implementations by no longer using non-standard keys. See [Github #????](https://github.com/Unidata/netcdf-c/issues/????) for more information.
* Cleanup the option code for NETCDF_ENABLE_SET_LOG_LEVEL\[_FUNC\] See [Github #2931](https://github.com/Unidata/netcdf-c/issues/2931) for more information.
* Fix duplicate definition when using aws-sdk-cpp. See [Github #2928](https://github.com/Unidata/netcdf-c/issues/2928) for more information.
* Cleanup various obsolete options and do some code refactoring. See [Github #2926](https://github.com/Unidata/netcdf-c/issues/2926) for more information.
Expand Down
420 changes: 290 additions & 130 deletions docs/nczarr.md
View file

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion include/nc4internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,5 @@ extern void NC_initialize_reserved(void);
#define NC_NCZARR_GROUP "_nczarr_group"
#define NC_NCZARR_ARRAY "_nczarr_array"
#define NC_NCZARR_ATTR "_nczarr_attr"
#define NC_NCZARR_ATTR_UC "_NCZARR_ATTR" /* deprecated */

#endif /* _NC4INTERNAL_ */
13 changes: 11 additions & 2 deletions include/ncjson.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ typedef struct NCjson {
int sort; /* of this object */
char* string; /* sort != DICT|ARRAY */
struct NCjlist {
int len;
size_t len;
struct NCjson** contents;
} list; /* sort == DICT|ARRAY */
} NCjson;
Expand Down Expand Up @@ -108,7 +108,14 @@ OPTEXPORT int NCJaddstring(NCjson* json, int sort, const char* s);
OPTEXPORT int NCJappend(NCjson* object, NCjson* value);

/* Insert key-value pair into a dict object. key will be copied */
OPTEXPORT int NCJinsert(NCjson* object, char* key, NCjson* value);
OPTEXPORT int NCJinsert(NCjson* object, const char* key, NCjson* value);

/* Insert key-value pair as strings into a dict object.
key and value will be copied */
OPTEXPORT int NCJinsertstring(NCjson* object, const char* key, const char* value);

/* Insert key-value pair where value is an int */
OPTEXPORT int NCJinsertint(NCjson* object, const char* key, long long ivalue);

/* Unparser to convert NCjson object to text in buffer */
OPTEXPORT int NCJunparse(const NCjson* json, unsigned flags, char** textp);
Expand All @@ -131,8 +138,10 @@ OPTEXPORT const char* NCJtotext(const NCjson* json);
#define NCJsort(x) ((x)->sort)
#define NCJstring(x) ((x)->string)
#define NCJlength(x) ((x)==NULL ? 0 : (x)->list.len)
#define NCJdictlength(x) ((x)==NULL ? 0 : (x)->list.len/2)
#define NCJcontents(x) ((x)->list.contents)
#define NCJith(x,i) ((x)->list.contents[i])
#define NCJdictith(x,i) ((x)->list.contents[2*i])

/* Setters */
#define NCJsetsort(x,s) (x)->sort=(s)
Expand Down
49 changes: 45 additions & 4 deletions include/netcdf_json.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ typedef struct NCjson {
int sort; /* of this object */
char* string; /* sort != DICT|ARRAY */
struct NCjlist {
int len;
size_t len;
struct NCjson** contents;
} list; /* sort == DICT|ARRAY */
} NCjson;
Expand Down Expand Up @@ -108,7 +108,14 @@ OPTEXPORT int NCJaddstring(NCjson* json, int sort, const char* s);
OPTEXPORT int NCJappend(NCjson* object, NCjson* value);

/* Insert key-value pair into a dict object. key will be copied */
OPTEXPORT int NCJinsert(NCjson* object, char* key, NCjson* value);
OPTEXPORT int NCJinsert(NCjson* object, const char* key, NCjson* value);

/* Insert key-value pair as strings into a dict object.
key and value will be copied */
OPTEXPORT int NCJinsertstring(NCjson* object, const char* key, const char* value);

/* Insert key-value pair where value is an int */
OPTEXPORT int NCJinsertint(NCjson* object, const char* key, long long ivalue);

/* Unparser to convert NCjson object to text in buffer */
OPTEXPORT int NCJunparse(const NCjson* json, unsigned flags, char** textp);
Expand All @@ -131,8 +138,10 @@ OPTEXPORT const char* NCJtotext(const NCjson* json);
#define NCJsort(x) ((x)->sort)
#define NCJstring(x) ((x)->string)
#define NCJlength(x) ((x)==NULL ? 0 : (x)->list.len)
#define NCJdictlength(x) ((x)==NULL ? 0 : (x)->list.len/2)
#define NCJcontents(x) ((x)->list.contents)
#define NCJith(x,i) ((x)->list.contents[i])
#define NCJdictith(x,i) ((x)->list.contents[2*i])

/* Setters */
#define NCJsetsort(x,s) (x)->sort=(s)
Expand Down Expand Up @@ -278,7 +287,9 @@ static int NCJnewstring(int sort, const char* value, NCjson** jsonp);
static int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);
static int NCJclone(const NCjson* json, NCjson** clonep);
static int NCJaddstring(NCjson* json, int sort, const char* s);
static int NCJinsert(NCjson* object, char* key, NCjson* jvalue);
static int NCJinsert(NCjson* object, const char* key, NCjson* jvalue);
static int NCJinsertstring(NCjson* object, const char* key, const char* value);
static int NCJinsertint(NCjson* object, const char* key, long long ivalue);
static int NCJappend(NCjson* object, NCjson* value);
static int NCJunparse(const NCjson* json, unsigned flags, char** textp);
#else /*!NETCDF_JSON_H*/
Expand Down Expand Up @@ -1050,7 +1061,7 @@ NCJaddstring(NCjson* json, int sort, const char* s)

/* Insert key-value pair into a dict object. key will be strdup'd */
OPTSTATIC int
NCJinsert(NCjson* object, char* key, NCjson* jvalue)
NCJinsert(NCjson* object, const char* key, NCjson* jvalue)
{
int stat = NCJ_OK;
NCjson* jkey = NULL;
Expand All @@ -1063,6 +1074,36 @@ NCJinsert(NCjson* object, char* key, NCjson* jvalue)
return NCJTHROW(stat);
}

/* Insert key-value pair as strings into a dict object.
key and value will be strdup'd */
OPTSTATIC int
NCJinsertstring(NCjson* object, const char* key, const char* value)
{
int stat = NCJ_OK;
NCjson* jvalue = NULL;
if(value == NULL)
NCJnew(NCJ_NULL,&jvalue);
else
NCJnewstring(NCJ_STRING,value,&jvalue);
NCJinsert(object,key,jvalue);
done:
return NCJTHROW(stat);
}

/* Insert key-value pair with value being an integer */
OPTSTATIC int
NCJinsertint(NCjson* object, const char* key, long long ivalue)
{
int stat = NCJ_OK;
NCjson* jvalue = NULL;
char digits[128];
snprintf(digits,sizeof(digits),"%lld",ivalue);
NCJnewstring(NCJ_STRING,digits,&jvalue);
NCJinsert(object,key,jvalue);
done:
return NCJTHROW(stat);
}

/* Append value to an array or dict object. */
OPTSTATIC int
NCJappend(NCjson* object, NCjson* value)
Expand Down
36 changes: 34 additions & 2 deletions libdispatch/ncjson.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,9 @@ static int NCJnewstring(int sort, const char* value, NCjson** jsonp);
static int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);
static int NCJclone(const NCjson* json, NCjson** clonep);
static int NCJaddstring(NCjson* json, int sort, const char* s);
static int NCJinsert(NCjson* object, char* key, NCjson* jvalue);
static int NCJinsert(NCjson* object, const char* key, NCjson* jvalue);
static int NCJinsertstring(NCjson* object, const char* key, const char* value);
static int NCJinsertint(NCjson* object, const char* key, long long ivalue);
static int NCJappend(NCjson* object, NCjson* value);
static int NCJunparse(const NCjson* json, unsigned flags, char** textp);
#else /*!NETCDF_JSON_H*/
Expand Down Expand Up @@ -900,7 +902,7 @@ NCJaddstring(NCjson* json, int sort, const char* s)

/* Insert key-value pair into a dict object. key will be strdup'd */
OPTSTATIC int
NCJinsert(NCjson* object, char* key, NCjson* jvalue)
NCJinsert(NCjson* object, const char* key, NCjson* jvalue)
{
int stat = NCJ_OK;
NCjson* jkey = NULL;
Expand All @@ -913,6 +915,36 @@ NCJinsert(NCjson* object, char* key, NCjson* jvalue)
return NCJTHROW(stat);
}

/* Insert key-value pair as strings into a dict object.
key and value will be strdup'd */
OPTSTATIC int
NCJinsertstring(NCjson* object, const char* key, const char* value)
{
int stat = NCJ_OK;
NCjson* jvalue = NULL;
if(value == NULL)
NCJnew(NCJ_NULL,&jvalue);
else
NCJnewstring(NCJ_STRING,value,&jvalue);
NCJinsert(object,key,jvalue);
done:
return NCJTHROW(stat);
}

/* Insert key-value pair with value being an integer */
OPTSTATIC int
NCJinsertint(NCjson* object, const char* key, long long ivalue)
{
int stat = NCJ_OK;
NCjson* jvalue = NULL;
char digits[128];
snprintf(digits,sizeof(digits),"%lld",ivalue);
NCJnewstring(NCJ_STRING,digits,&jvalue);
NCJinsert(object,key,jvalue);
done:
return NCJTHROW(stat);
}

/* Append value to an array or dict object. */
OPTSTATIC int
NCJappend(NCjson* object, NCjson* value)
Expand Down
2 changes: 1 addition & 1 deletion libdispatch/ncs3sdk_h5.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ NC_s3sdkinitialize(void)
}

/* Get environment information */
NC_s3sdkenvironment(void);
NC_s3sdkenvironment();

return NC_NOERR;
}
Expand Down
6 changes: 3 additions & 3 deletions libnczarr/zarr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,8 +269,8 @@ ncz_open_rootgroup(NC_FILE_INFO_T* dataset)

if((stat=nczm_concat(NULL,ZGROUP,&rootpath)))
goto done;
if((stat = NCZ_downloadjson(zfile->map, rootpath, &json)))
goto done;
if((stat = NCZ_downloadjson(zfile->map, rootpath, &json))) goto done;
if(json == NULL) goto done;
/* Process the json */
for(i=0;i<nclistlength(json->contents);i+=2) {
const NCjson* key = nclistget(json->contents,i);
Expand Down Expand Up @@ -315,7 +315,7 @@ applycontrols(NCZ_FILE_INFO_T* zinfo)
int stat = NC_NOERR;
const char* value = NULL;
NClist* modelist = nclistnew();
int noflags = 0; /* track non-default negative flags */
size64_t noflags = 0; /* track non-default negative flags */

if((value = controllookup(zinfo->controllist,"mode")) != NULL) {
if((stat = NCZ_comma_parse(value,modelist))) goto done;
Expand Down
4 changes: 1 addition & 3 deletions libnczarr/zarr.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ EXTERNL int NCZ_stringconvert(nc_type typid, size_t len, void* data0, NCjson** j
/* zsync.c */
EXTERNL int ncz_sync_file(NC_FILE_INFO_T* file, int isclose);
EXTERNL int ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose);
EXTERNL int ncz_sync_atts(NC_FILE_INFO_T*, NC_OBJ* container, NCindex* attlist, int isclose);
EXTERNL int ncz_sync_atts(NC_FILE_INFO_T*, NC_OBJ* container, NCindex* attlist, NCjson* jatts, NCjson* jtypes, int isclose);
EXTERNL int ncz_read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp);
EXTERNL int ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container);
EXTERNL int ncz_read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp);
Expand All @@ -62,8 +62,6 @@ EXTERNL int NCZ_grpkey(const NC_GRP_INFO_T* grp, char** pathp);
EXTERNL int NCZ_varkey(const NC_VAR_INFO_T* var, char** pathp);
EXTERNL int NCZ_dimkey(const NC_DIM_INFO_T* dim, char** pathp);
EXTERNL int ncz_splitkey(const char* path, NClist* segments);
EXTERNL int NCZ_readdict(NCZMAP* zmap, const char* key, NCjson** jsonp);
EXTERNL int NCZ_readarray(NCZMAP* zmap, const char* key, NCjson** jsonp);
EXTERNL int ncz_nctypedecode(const char* snctype, nc_type* nctypep);
EXTERNL int ncz_nctype2dtype(nc_type nctype, int endianness, int purezarr,int len, char** dnamep);
EXTERNL int ncz_dtype2nctype(const char* dtype, nc_type typehint, int purezarr, nc_type* nctypep, int* endianp, int* typelenp);
Expand Down
14 changes: 7 additions & 7 deletions libnczarr/zattr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ ncz_getattlist(NC_GRP_INFO_T *grp, int varid, NC_VAR_INFO_T **varp, NCindex **at
{
NC_VAR_INFO_T *var;

if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, varid)))
if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, (size_t)varid)))
return NC_ENOTVAR;
assert(var->hdr.id == varid);

Expand Down Expand Up @@ -120,7 +120,7 @@ ncz_get_att_special(NC_FILE_INFO_T* h5, NC_VAR_INFO_T* var, const char* name,

/* The global reserved attributes */
if(strcmp(name,NCPROPS)==0) {
int len;
size_t len;
if(h5->provenance.ncproperties == NULL)
{stat = NC_ENOTATT; goto done;}
if(mem_type == NC_NAT) mem_type = NC_CHAR;
Expand All @@ -138,7 +138,7 @@ ncz_get_att_special(NC_FILE_INFO_T* h5, NC_VAR_INFO_T* var, const char* name,
if(strcmp(name,SUPERBLOCKATT)==0)
iv = (unsigned long long)h5->provenance.superblockversion;
else /* strcmp(name,ISNETCDF4ATT)==0 */
iv = NCZ_isnetcdf4(h5);
iv = (unsigned long long)NCZ_isnetcdf4(h5);
if(mem_type == NC_NAT) mem_type = NC_INT;
if(data)
switch (mem_type) {
Expand Down Expand Up @@ -279,8 +279,8 @@ NCZ_del_att(int ncid, int varid, const char *name)
NC_FILE_INFO_T *h5;
NC_ATT_INFO_T *att;
NCindex* attlist = NULL;
int i;
size_t deletedid;
size_t i;
int deletedid;
int retval;

/* Name must be provided. */
Expand Down Expand Up @@ -516,7 +516,7 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
/* For an existing att, if we're not in define mode, the len
must not be greater than the existing len for classic model. */
if (!(h5->flags & NC_INDEF) &&
len * nc4typelen(file_type) > (size_t)att->len * nc4typelen(att->nc_typeid))
len * (size_t)nc4typelen(file_type) > (size_t)att->len * (size_t)nc4typelen(att->nc_typeid))
{
if (h5->cmode & NC_CLASSIC_MODEL)
return NC_ENOTINDEFINE;
Expand Down Expand Up @@ -980,7 +980,7 @@ int
ncz_create_fillvalue(NC_VAR_INFO_T* var)
{
int stat = NC_NOERR;
int i;
size_t i;
NC_ATT_INFO_T* fv = NULL;

/* Have the var's attributes been read? */
Expand Down
2 changes: 1 addition & 1 deletion libnczarr/zchunking.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ NCZ_compute_all_slice_projections(
NCZSliceProjections* results)
{
int stat = NC_NOERR;
size64_t r;
int r;

for(r=0;r<common->rank;r++) {
/* Compute each of the rank SliceProjections instances */
Expand Down
14 changes: 7 additions & 7 deletions libnczarr/zclose.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ zclose_group(NC_GRP_INFO_T *grp)
{
int stat = NC_NOERR;
NCZ_GRP_INFO_T* zgrp;
int i;
size_t i;

assert(grp && grp->format_grp_info != NULL);
LOG((3, "%s: grp->name %s", __func__, grp->hdr.name));
Expand Down Expand Up @@ -123,7 +123,7 @@ zclose_gatts(NC_GRP_INFO_T* grp)
{
int stat = NC_NOERR;
NC_ATT_INFO_T *att;
int a;
size_t a;
for(a = 0; a < ncindexsize(grp->att); a++) {
NCZ_ATT_INFO_T* zatt = NULL;
att = (NC_ATT_INFO_T* )ncindexith(grp->att, a);
Expand All @@ -149,7 +149,7 @@ NCZ_zclose_var1(NC_VAR_INFO_T* var)
int stat = NC_NOERR;
NCZ_VAR_INFO_T* zvar;
NC_ATT_INFO_T* att;
int a;
size_t a;

assert(var && var->format_var_info);
zvar = var->format_var_info;;
Expand Down Expand Up @@ -191,7 +191,7 @@ zclose_vars(NC_GRP_INFO_T* grp)
{
int stat = NC_NOERR;
NC_VAR_INFO_T* var;
int i;
size_t i;

for(i = 0; i < ncindexsize(grp->vars); i++) {
var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
Expand All @@ -215,7 +215,7 @@ zclose_dims(NC_GRP_INFO_T* grp)
{
int stat = NC_NOERR;
NC_DIM_INFO_T* dim;
int i;
size_t i;

for(i = 0; i < ncindexsize(grp->dim); i++) {
NCZ_DIM_INFO_T* zdim;
Expand Down Expand Up @@ -265,7 +265,7 @@ static int
zclose_types(NC_GRP_INFO_T* grp)
{
int stat = NC_NOERR;
int i;
size_t i;
NC_TYPE_INFO_T* type;

for(i = 0; i < ncindexsize(grp->type); i++)
Expand All @@ -289,7 +289,7 @@ static int
zwrite_vars(NC_GRP_INFO_T *grp)
{
int stat = NC_NOERR;
int i;
size_t i;

assert(grp && grp->format_grp_info != NULL);
LOG((3, "%s: grp->name %s", __func__, grp->hdr.name));
Expand Down
Loading

0 comments on commit 076da97

Please sign in to comment.