Skip to content

Commit

Permalink
# Addendum
Browse files Browse the repository at this point in the history 
As suggested by Ward, I ensured that this PR supports
read backward compatibility with old key format.
This addition also adds a test case for this.

## Misc. Other Changes
* Remove some unused code
* Cleanup json error handling
* Fix some more unsigned/signed conversions warning
  • Loading branch information
@DennisHeimbigner
DennisHeimbigner committed Jun 25, 2024
1 parent d177400 commit 06d2fe5
Show file tree
Hide file tree
Showing 29 changed files with 429 additions and 536 deletions.
2 changes: 1 addition & 1 deletion include/ncjson.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ OPTEXPORT int NCJnewstring(int sort, const char* value, NCjson** jsonp);
OPTEXPORT int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);

/* Get dict key value by name */
OPTEXPORT int NCJdictget(const NCjson* dict, const char* key, NCjson** valuep);
OPTEXPORT int NCJdictget(const NCjson* dict, const char* key, const NCjson** valuep);

/* Convert one json sort to value of another type; don't use union so we can know when to reclaim sval */
OPTEXPORT int NCJcvt(const NCjson* value, int outsort, struct NCJconst* output);
Expand Down
4 changes: 2 additions & 2 deletions include/netcdf_json.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ OPTEXPORT int NCJnewstring(int sort, const char* value, NCjson** jsonp);
OPTEXPORT int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);

/* Get dict key value by name */
OPTEXPORT int NCJdictget(const NCjson* dict, const char* key, NCjson** valuep);
OPTEXPORT int NCJdictget(const NCjson* dict, const char* key, const NCjson** valuep);

/* Convert one json sort to value of another type; don't use union so we can know when to reclaim sval */
OPTEXPORT int NCJcvt(const NCjson* value, int outsort, struct NCJconst* output);
Expand Down Expand Up @@ -775,7 +775,7 @@ NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp)
}

OPTSTATIC int
NCJdictget(const NCjson* dict, const char* key, NCjson** valuep)
NCJdictget(const NCjson* dict, const char* key, const NCjson** valuep)
{
int i,stat = NCJ_OK;

Expand Down
2 changes: 1 addition & 1 deletion libdispatch/ncjson.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -616,7 +616,7 @@ NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp)
}

OPTSTATIC int
NCJdictget(const NCjson* dict, const char* key, NCjson** valuep)
NCJdictget(const NCjson* dict, const char* key, const NCjson** valuep)
{
int i,stat = NCJ_OK;

Expand Down
128 changes: 0 additions & 128 deletions libnczarr/zarr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,61 +239,6 @@ NCZ_get_superblock(NC_FILE_INFO_T* file, int* superblockp)
/**************************************************/
/* Utilities */

#if 0
/**
@internal Open the root group object
@param dataset - [in] the root dataset object
@param rootp - [out] created root group
@return NC_NOERR
@author Dennis Heimbigner
*/
static int
ncz_open_rootgroup(NC_FILE_INFO_T* dataset)
{
int stat = NC_NOERR;
int i;
NCZ_FILE_INFO_T* zfile = NULL;
NC_GRP_INFO_T* root = NULL;
void* content = NULL;
char* rootpath = NULL;
NCjson* json = NULL;

ZTRACE(3,"dataset=",dataset->hdr.name);

zfile = dataset->format_file_info;

/* Root should already be defined */
root = dataset->root_grp;

assert(root != NULL);

if((stat=nczm_concat(NULL,ZGROUP,&rootpath)))
goto done;
if((stat = NCZ_downloadjson(zfile->map, rootpath, &json))) goto done;
if(json == NULL) goto done;
/* Process the json */
for(i=0;i<nclistlength(json->contents);i+=2) {
const NCjson* key = nclistget(json->contents,i);
const NCjson* value = nclistget(json->contents,i+1);
if(strcmp(NCJstring(key),"zarr_format")==0) {
int zversion;
if(sscanf(NCJstring(value),"%d",&zversion)!=1)
{stat = NC_ENOTNC; goto done;}
/* Verify against the dataset */
if(zversion != zfile->zarr.zarr_version)
{stat = NC_ENOTNC; goto done;}
}
}

done:
if(json) NCJreclaim(json);
nullfree(rootpath);
nullfree(content);
return ZUNTRACE(stat);
}
#endif


static const char*
controllookup(NClist* controls, const char* key)
{
Expand Down Expand Up @@ -352,76 +297,3 @@ applycontrols(NCZ_FILE_INFO_T* zinfo)
nclistfreeall(modelist);
return stat;
}

#if 0
/**
@internal Rewrite attributes into a group or var
@param map - [in] the map object for storage
@param container - [in] the containing object
@param jattrs - [in] the json for .zattrs
@param jtypes - [in] the json for .ztypes
@return NC_NOERR
@author Dennis Heimbigner
*/
int
ncz_unload_jatts(NCZ_FILE_INFO_T* zinfo, NC_OBJ* container, NCjson* jattrs, NCjson* jtypes)
{
int stat = NC_NOERR;
char* fullpath = NULL;
char* akey = NULL;
char* tkey = NULL;
NCZMAP* map = zinfo->map;

assert((NCJsort(jattrs) == NCJ_DICT));
assert((NCJsort(jtypes) == NCJ_DICT));

if(container->sort == NCGRP) {
NC_GRP_INFO_T* grp = (NC_GRP_INFO_T*)container;
/* Get grp's fullpath name */
if((stat = NCZ_grpkey(grp,&fullpath)))
goto done;
} else {
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)container;
/* Get var's fullpath name */
if((stat = NCZ_varkey(var,&fullpath)))
goto done;
}

/* Construct the path to the .zattrs object */
if((stat = nczm_concat(fullpath,ZATTRS,&akey)))
goto done;

/* Always write as V2 */

{
NCjson* k = NULL;
NCjson* v = NULL;
/* remove any previous version */
if(!NCJremove(jattrs,NCZ_V2_ATTRS,1,&k,&v)) {
NCJreclaim(k); NCJreclaim(v);
}
}

if(!(zinfo->controls.flags & FLAG_PUREZARR)) {
/* Insert the jtypes into the set of attributes */
if((stat = NCJinsert(jattrs,NCZ_V2_ATTRS,jtypes))) goto done;
}

/* Upload the .zattrs object */
if((stat=NCZ_uploadjson(map,tkey,jattrs)))
goto done;

done:
if(stat) {
NCJreclaim(jattrs);
NCJreclaim(jtypes);
}
nullfree(fullpath);
nullfree(akey);
nullfree(tkey);
return stat;
}
#endif



8 changes: 4 additions & 4 deletions libnczarr/zarr.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ EXTERNL int ncz_unload_jatts(NCZ_FILE_INFO_T*, NC_OBJ* container, NCjson* jattrs
EXTERNL int ncz_close_file(NC_FILE_INFO_T* file, int abort);

/* zcvt.c */
EXTERNL int NCZ_json2cvt(NCjson* jsrc, struct ZCVT* zcvt, nc_type* typeidp);
EXTERNL int NCZ_convert1(NCjson* jsrc, nc_type, NCbytes*);
EXTERNL int NCZ_json2cvt(const NCjson* jsrc, struct ZCVT* zcvt, nc_type* typeidp);
EXTERNL int NCZ_convert1(const NCjson* jsrc, nc_type, NCbytes*);
EXTERNL int NCZ_stringconvert1(nc_type typid, char* src, NCjson* jvalue);
EXTERNL int NCZ_stringconvert(nc_type typid, size_t len, void* data0, NCjson** jdatap);

Expand All @@ -65,7 +65,7 @@ EXTERNL int ncz_splitkey(const char* path, NClist* segments);
EXTERNL int ncz_nctypedecode(const char* snctype, nc_type* nctypep);
EXTERNL int ncz_nctype2dtype(nc_type nctype, int endianness, int purezarr,int len, char** dnamep);
EXTERNL int ncz_dtype2nctype(const char* dtype, nc_type typehint, int purezarr, nc_type* nctypep, int* endianp, int* typelenp);
EXTERNL int NCZ_inferattrtype(NCjson* value, nc_type typehint, nc_type* typeidp);
EXTERNL int NCZ_inferattrtype(const NCjson* value, nc_type typehint, nc_type* typeidp);
EXTERNL int NCZ_inferinttype(unsigned long long u64, int negative);
EXTERNL int ncz_fill_value_sort(nc_type nctype, int*);
EXTERNL int NCZ_createobject(NCZMAP* zmap, const char* key, size64_t size);
Expand All @@ -87,7 +87,7 @@ EXTERNL int NCZ_get_maxstrlen(NC_OBJ* obj);
EXTERNL int NCZ_fixed2char(const void* fixed, char** charp, size_t count, int maxstrlen);
EXTERNL int NCZ_char2fixed(const char** charp, void* fixed, size_t count, int maxstrlen);
EXTERNL int NCZ_copy_data(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const void* memory, size_t count, int reading, void* copy);
EXTERNL int NCZ_iscomplexjson(NCjson* value, nc_type typehint);
EXTERNL int NCZ_iscomplexjson(const NCjson* value, nc_type typehint);

/* zwalk.c */
EXTERNL int NCZ_read_chunk(int ncid, int varid, size64_t* zindices, void* chunkdata);
Expand Down
9 changes: 8 additions & 1 deletion libnczarr/zclose.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,9 @@ zclose_group(NC_GRP_INFO_T *grp)
/* Close the zgroup. */
zgrp = grp->format_grp_info;
LOG((4, "%s: closing group %s", __func__, grp->hdr.name));
nullfree(zgrp->zgroup.prefix);
NCJreclaim(zgrp->zgroup.obj);
NCJreclaim(zgrp->zgroup.atts);
nullfree(zgrp);
grp->format_grp_info = NULL; /* avoid memory errors */

Expand Down Expand Up @@ -152,7 +155,6 @@ NCZ_zclose_var1(NC_VAR_INFO_T* var)
size_t a;

assert(var && var->format_var_info);
zvar = var->format_var_info;;
for(a = 0; a < ncindexsize(var->att); a++) {
NCZ_ATT_INFO_T* zatt;
att = (NC_ATT_INFO_T*)ncindexith(var->att, a);
Expand All @@ -170,9 +172,14 @@ NCZ_zclose_var1(NC_VAR_INFO_T* var)
#endif
/* Reclaim the type */
if(var->type_info) (void)zclose_type(var->type_info);
/* reclaim dispatch info */
zvar = var->format_var_info;;
if(zvar->cache) NCZ_free_chunk_cache(zvar->cache);
/* reclaim xarray */
if(zvar->xarray) nclistfreeall(zvar->xarray);
nullfree(zvar->zarray.prefix);
NCJreclaim(zvar->zarray.obj);
NCJreclaim(zvar->zarray.atts);
nullfree(zvar);
var->format_var_info = NULL; /* avoid memory errors */
return stat;
Expand Down
8 changes: 4 additions & 4 deletions libnczarr/zcvt.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
Code taken directly from libdap4/d4cvt.c
*/

static const int ncz_type_size[NC_MAX_ATOMIC_TYPE+1] = {
static const size_t ncz_type_size[NC_MAX_ATOMIC_TYPE+1] = {
0, /*NC_NAT*/
sizeof(char), /*NC_BYTE*/
sizeof(char), /*NC_CHAR*/
Expand Down Expand Up @@ -101,7 +101,7 @@ NCZ_string2cvt(char* src, nc_type srctype, struct ZCVT* zcvt, nc_type* typeidp)

/* Warning: not free returned zcvt.strv; it may point into a string in jsrc */
int
NCZ_json2cvt(NCjson* jsrc, struct ZCVT* zcvt, nc_type* typeidp)
NCZ_json2cvt(const NCjson* jsrc, struct ZCVT* zcvt, nc_type* typeidp)
{
int stat = NC_NOERR;
nc_type srctype = NC_NAT;
Expand Down Expand Up @@ -154,7 +154,7 @@ NCZ_json2cvt(NCjson* jsrc, struct ZCVT* zcvt, nc_type* typeidp)

/* Convert a singleton NCjson value to a memory equivalent value of specified dsttype; */
int
NCZ_convert1(NCjson* jsrc, nc_type dsttype, NCbytes* buf)
NCZ_convert1(const NCjson* jsrc, nc_type dsttype, NCbytes* buf)
{
int stat = NC_NOERR;
nc_type srctype;
Expand Down Expand Up @@ -536,7 +536,7 @@ int
NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap)
{
int stat = NC_NOERR;
int i;
size_t i;
char* src = data0; /* so we can do arithmetic on it */
size_t typelen;
char* str = NULL;
Expand Down
2 changes: 1 addition & 1 deletion libnczarr/zfilter.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -979,7 +979,7 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j
{
int i,stat = NC_NOERR;
NCZ_Filter* filter = NULL;
NCjson* jvalue = NULL;
const NCjson* jvalue = NULL;
NCZ_Plugin* plugin = NULL;
NCZ_Codec codec = codec_empty;
NCZ_HDF5 hdf5 = hdf5_empty;
Expand Down
24 changes: 11 additions & 13 deletions libnczarr/zinternal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ typedef struct NCZ_FILE_INFO {
# define FLAG_SHOWFETCH 2
# define FLAG_LOGGING 4
# define FLAG_XARRAYDIMS 8
# define FLAG_NCZARR_V1 16
# define FLAG_NCZARR_KEY 16 /* _nczarr_xxx keys are stored in object and not in _nczarr_attrs */
NCZM_IMPL mapimpl;
} controls;
int default_maxstrlen; /* default max str size for variables of type string */
Expand All @@ -159,18 +159,13 @@ typedef struct NCZ_ATT_INFO {
/* Struct to hold ZARR-specific info for a group. */
typedef struct NCZ_GRP_INFO {
NCZcommon common;
#if 0
/* The jcontent field stores the following:
1. List of (name,length) for dims in the group
2. List of (name,type) for user-defined types in the group
3. List of var names in the group
4. List of subgroups names in the group
*/
NClist* dims;
NClist* types; /* currently not used */
NClist* vars;
NClist* grps;
#endif
/* Read .zgroup and .zattrs once */
struct ZARROBJ {
char* prefix; /* prefix of .zgroup and .zattrs */
NCjson* obj; /* .zgroup|.zarray */
NCjson* atts;
int nczv1; /* 1 => _nczarr_xxx are in obj and not attributes */
} zgroup;
} NCZ_GRP_INFO_T;

/* Struct to hold ZARR-specific info for a variable. */
Expand All @@ -185,6 +180,9 @@ typedef struct NCZ_VAR_INFO {
char dimension_separator; /* '.' | '/' */
NClist* incompletefilters;
int maxstrlen; /* max length of strings for this variable */
/* Read .zarray and .zattrs once */
struct ZARROBJ zarray;
struct ZARROBJ zattrs;
} NCZ_VAR_INFO_T;

/* Struct to hold ZARR-specific info for a field. */
Expand Down
Loading

0 comments on commit 06d2fe5

Please sign in to comment.