diff --git a/VERSION b/VERSION
index a84947d6f..4404a17ba 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-4.5.0
+4.5.1
diff --git a/docs/source/scripting.rst b/docs/source/scripting.rst
index ed727107e..5d017df24 100644
--- a/docs/source/scripting.rst
+++ b/docs/source/scripting.rst
@@ -77,6 +77,46 @@ the --help command line option
where possible.
+Example: Creating a runfile
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A runfile is a simple text file, where each line in this file can be executed, for example using
+bash in a serial fashion. If each line is independant from all others, they can be executed in parallel using
+``gnu parallel`` or ``sbatch`` (a common HPC batch queue system). For example the following command
+in NEMO produces a runfile with 4 lines:
+
+.. code-block::
+
+ mkrunfile.py progname a=1,2 b=2,4 c=10 > example1.run
+
+ progname a=1 b=2 c=10
+ progname a=2 b=2 c=10
+ progname a=1 b=4 c=10
+ progname a=2 b=4 c=10
+
+where the ``example1.run`` file can be executed with any of the following commands. Depending on your
+resources of course. Memory, number of cores etc.
+
+.. code-block::
+
+ bash example1.run
+ parallel -j 4 < example1.run
+ sbatch_nemo.sh example1.run
+
+in particular the last ``sbatch_nemo.sh`` example will likely need to be tailored for your HPC system.
+
+.. note::
+ Unless the parameters take care of this, you will need to ensure data are written to files that do not
+ collide with each other. For example a directory or file that encodes the values of the parameters,
+ or are numerically sorted (e.g. run010, run011)
+ Currently ``mkrunfile.py`` does not have an automated way for this yet.
+
+And here is an example of creating a runfile from a table with values
+
+.. code-block::
+
+ awk '{printf("progname runfile=run_%s a=%s b=%s\n", $1,$2,$3) }' example1.tab
+
Example: Extracting results from run directories
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/whatsnew.html b/docs/whatsnew.html
index 9872891e7..fa0771bd8 100644
--- a/docs/whatsnew.html
+++ b/docs/whatsnew.html
@@ -22,13 +22,15 @@
What's New in NEMO
-->
- 4.5.0: in git
+ 4.5.0/4.5.1: in git
- - (Jan 2, 2025) fixed very old 64bit itemsize limitation [issue17]
+
- (Jan 3, 2025) added mkrunfile.py to create runfiles looping over parameters
+
- (Jan 3, 2025) fixed very old 64bit itemsize limitation [issue17],
+ this affected at least ccdfits, fitsccd and scanfits.
- 4.4.3/4.4.4/4.4.5 (a.k.a. Malta Release)
+ 4.4.3/4.4.4/4.4.5: 22-dec-2024 (a.k.a. Malta Release)
- (Dec 22, 2024) define TIMEFUZZ in stdinc.h, not per program
- (Dec 12, 2024) drafting two codes: diskspectrum and lineid
diff --git a/inc/fits.h b/inc/fits.h
index 62e0c0a57..c046be540 100644
--- a/inc/fits.h
+++ b/inc/fits.h
@@ -1,8 +1,10 @@
/*
- * definitions and declarations for nemo's FITS.C
+ * definitions and declarations for nemo's src/image/fits/fits.c
+ *
* 22-feb-94 ansi- and C++ safe
* 10-aug-09 size_t instead of int for 2GB+ files
* 25-oct-20 add hdu counter
+ * 3-jan-24 more proper usage of size_t instead of int
*/
#ifndef _fits_h_
@@ -129,7 +131,7 @@ static struct arglist {
typedef struct fits_header {
int hdu; /* keep track with HDU this is (1 being the first) */
/* Optionally we could store the buffer in the first structure too... */
- char *buffer; /* pointer to exact header copy */
+ char *buffer; /* pointer to exact header copy , if present */
size_t buflen; /* current length of buffer */
size_t hlen; /* length of header (bytes, through the END keyword) */
size_t dlen; /* length of data (bytes) */
@@ -200,8 +202,8 @@ typedef struct fits_header {
} fits_header;
-size_t fts_rhead (fits_header *, stream);
-char *fts_shead (fits_header *, string);
+size_t fts_rhead (fits_header *, stream);
+char *fts_shead (fits_header *, string);
int fts_lhead (fits_header *);
int fts_chead (fits_header *, stream);
int fts_thead (fits_header *);
@@ -211,7 +213,7 @@ int fts_ihead (fits_header *, string *);
int fts_fhead (fits_header *, string *);
int fts_phead (fits_header *, string *);
int fts_whead (fits_header *, stream);
-size_t fts_xhead (fits_header *, stream, int, int, int *, int);
+size_t fts_xhead (fits_header *, stream, int, int, int *, int);
int fts_ptable (fits_header *, stream, string *, string, int *, int);
int fts_pgroup (fits_header *, stream, string *, string, int *, int);
@@ -220,8 +222,8 @@ int fts_buf (int);
int fts_sdata (fits_header *, stream);
int fts_cdata (fits_header *, stream, stream, bool, bool);
-int fts_dsize (fits_header *);
-int fts_tsize (fits_header *);
+size_t fts_dsize (fits_header *);
+size_t fts_tsize (fits_header *);
int fts_rdata (fits_header *, stream, int, char *);
int fts_wdata (fits_header *, stream, int, char *);
int fts_rrow (fits_header *, stream, int, char *);
diff --git a/inc/version.h b/inc/version.h
index eb2faa7a0..490785f44 100644
--- a/inc/version.h
+++ b/inc/version.h
@@ -1,5 +1,5 @@
/* do not edit - created by the src/scripts/version script */
#if defined(GETPARAM_VERSION_ID)
-static char *version_h = "git_12231"; // git rev-list --count HEAD
+static char *version_h = "git_12284"; // git rev-list --count HEAD
#endif
-#define NEMO_VERSION "4.5.0"
+#define NEMO_VERSION "4.5.1"
diff --git a/man/man3/filestruct.3 b/man/man3/filestruct.3
index bbe1402b7..4f408f1b6 100644
--- a/man/man3/filestruct.3
+++ b/man/man3/filestruct.3
@@ -1,4 +1,4 @@
-.TH FILESTRUCT 3NEMO "7 April 2012"
+.TH FILESTRUCT 3NEMO "2 January 2024"
.SH "NAME"
filestruct \- primitives for structured binary file I/O
@@ -274,11 +274,12 @@ https://github.com/syrte/snapio - python implementation of snap I/O
.SH "UPDATE HISTORY"
.nf
-.ta +2.0i +2.0i
+.ta +1.25i +4.5i
4-Apr-87 original implementation JEB
30-Aug-87 type coersion, deferred input LPH
16-Apr-88 new types, operators, etc JEB
16-May-92 random access to data PJT
5-mar-94 documented qsf PJT
2-jun-05 added blocked I/O PJT
+2-jan-2024 fix 64bit problem for big items PJT
.fi
diff --git a/man/man5/filestruct.5 b/man/man5/filestruct.5
index fc4d9c20d..f1bb02976 100644
--- a/man/man5/filestruct.5
+++ b/man/man5/filestruct.5
@@ -1,7 +1,9 @@
-.TH FILESTRUCT 5NEMO "16 May 1992"
-.SH NAME
+.TH FILESTRUCT 5NEMO "2 January 2024"
+
+.SH "NAME"
filestruct \- binary structured file format
-.SH SYNOPSIS
+
+.SH "SYNOPSIS"
.nf
\fB#include \fP
\fB#include "filesecret.h"\fP \fI only for local code\fP
@@ -19,11 +21,12 @@ filestruct \- binary structured file format
.PP
\fB struct item {\fP
\fB string itemtyp;\fP
-\fB int itemlen;\fP
+\fB size_t itemlen;\fP
\fB string itemtag;\fP
\fB int *itemdim;\fP
-\fB byte *itemdat;\fP
-\fB long itempos;\fP
+\fB void *itemdat;\fP
+\fB off_t itempos;\fP
+\fB off_t itemoff;\fP
\fB };\fP
.PP
\fB struct strstk {\fP
@@ -31,11 +34,13 @@ filestruct \- binary structured file format
\fB item *ss_stk[SetStkLen];\fP
\fB int ss_stp;\fP
\fB bool ss_seek;\fP
-\fB long ss_pos;\fP \fI/* only if RANDOM access allowed */\fP
+\fB int ss_mode;\fP \fI/* only if RANDOM access allowed */\fP
+\fB off_t ss_pos;\fP \fI/* only if RANDOM access allowed */\fP
\fB itemptr ss_ran;\fP \fI/* only if RANDOM access allowed */\fP
\fB };\fP
.fi
-.SH DESCRIPTION
+
+.SH "DESCRIPTION"
\fIfilestruct\fP is a method for storing data files largely
consisting of character strings, booleans, and possible
multi-dimensional arrays of data. This data may be structured in a
@@ -74,7 +79,8 @@ otherwise. This is essentially the reason why pipes cannot be used in
The actual internal format is governed how the application programmer
uses the \fIget_XXX\fP and \fPput_XXX\fP routines (see
\fIfilestruct(3NEMO)\fP).
-.SH EXPERIMENTAL FEATURES
+
+.SH "EXPERIMENTAL FEATURES"
If compiled with \fB-DRANDOM\fP
some limited random access to data within a data-item is possible.
.PP
@@ -83,22 +89,27 @@ little/big endian machines. Otherwise,
data on disk exist in the host format, and no effort has been made to make
it machine independant (e.g. IEEE floating points and twos-compliment
integers). This is however expected in some future release.
-.SH ZENO FORMAT
+
+.SH "ZENO FORMAT"
The \fIzeno(1NEMO)\fP package also used this format, but there are some
-subtle differences to be described.
-.SH FILES
+subtle differences still to be described here.
+
+.SH "FILES"
.nf
.ta +2.0i
~/src/kernel/io filesecret.[ch] filestruct.h
~/inc filestruct.h
.fi
-.SH SEE ALSO
+
+.SH "SEE ALSO"
tsf(1NEMO), rsf(1NEMO), csf(1NEMO), filestruct(3NEMO), zeno(1NEMO)
-.SH AUTHOR
+
+.SH "AUTHOR"
Joshua Barnes, Lyman Hurd, Peter Teuben
-.SH UPDATE HISTORY
+
+.SH "HISTORY"
.nf
-.ta +1.0i +4.0i
+.ta +1.25i +4.5i
dark-ages V0.0 precurser (filestr) JEB
xx-apr-87 V1.0 basic operators JEB
xx-jul-87 V1.x added f-d coercion, deferred output Lyman Hurd
@@ -108,4 +119,5 @@ xx-xxx-87 V2.0 new types, operators, external fmt JEB
16-may-92 V3.0 finalized random access PJT
6-jul -01 documented the new uNEMO PJT
27-dec-2019 documented ZENO PJT
-.fi
+2-jan-2024 V3.6 fix remaining 64bit issues by using off_t, size_t PJT
+
diff --git a/man/man8/tricks.8 b/man/man8/tricks.8
index 165bb61c7..4c518f6d0 100644
--- a/man/man8/tricks.8
+++ b/man/man8/tricks.8
@@ -93,7 +93,7 @@ The (bash) array ${PIPESTATUS[@]} contains the status ($?) for each program in
a unix pipe. In this example it should return "0 0 0".
.SH "PARALLEL and OpenMP"
-Most programs in NEMO are still single-core, but many current machines have a number of cores. If a program only
+Most programs in NEMO are still single-core, but many current machines have multiple cores. If a program only
uses a fraction of the memory on the machine, the GNU \fIparallel(1)\fP program can be used to spread the load,
e.g.
.nf
@@ -109,6 +109,8 @@ created here, a well crafted Runfile can be created, and with
make -f Runfile -j 2
.fi
should achieve the same result.
+.PP
+The \fBmkrunfile.py\fP script can help making such runfiles by looping over a set of parameters.
.PP
Although NEMO can be configured using \fB--with-openmp\fP to take advantage of multi-cores OpenMP
@@ -129,7 +131,8 @@ should be aware of the user interface implications of controlling how many cores
.SH "SLURM"
slurm is a popular package you will find on large computer clusters. See also
-\fIsbatch_nemo.sh(8NEMO)\fP for a helper script in NEMO to use slurm.
+\fIsbatch_nemo.sh(8NEMO)\fP for a helper script in NEMO to use slurm. It will likely
+have to be tailored for your specific situation.
.SH "TABLES"
diff --git a/src/image/fits/fits.c b/src/image/fits/fits.c
index f016b9b06..cc4f256ae 100644
--- a/src/image/fits/fits.c
+++ b/src/image/fits/fits.c
@@ -151,7 +151,8 @@ size_t fts_rhead(fits_header *fh, stream instr)
{
char buf[FTSLINSIZ+2]; /* buffer to hold one card image */
char a1[9], a2[2], a3[FTSLINSIZ+1],a4[FTSLINSIZ+1],a5[FTSLINSIZ+1]; /* args */
- int k, n, i, icard;
+ int k, i, icard;
+ size_t n;
struct arglist *p;
if (fh->naxis >= 0) { /* we could force a fts_zero here, but ala */
@@ -175,10 +176,9 @@ size_t fts_rhead(fits_header *fh, stream instr)
n = FTSLINSIZ;
} else { /* this is of course the legal way to get cards */
if ((n=fread(buf,1,FTSLINSIZ,instr)) != FTSLINSIZ) { /* get card */
- if (n==0)
- return -1; /* error - nothing read -- never -- */
- else
- return -n-1; /* return some negative number, but < -1 */
+ if (n != 0)
+ warning("Incomplete read %ld bytes in header", n);
+ return 0;
} else {
buf[FTSLINSIZ] = 0; /* be sure it's terminated */
dprintf(5,"RECORD[%d]%65s\n",icard,buf);
@@ -458,7 +458,7 @@ size_t fts_rhead(fits_header *fh, stream instr)
/* process some additional checks */
fts_lhead(fh); /* process a few more sanity checks */
- return fts_dsize(fh); /* return the size of upcoming data */
+ return fts_dsize(fh); /* return the size of upcoming data (could be 0 too) */
}
/*
@@ -1257,7 +1257,7 @@ int fts_phead(fits_header *fh, string *print)
printf("headersize = %d bytes = %d %d-records\n",
(int) fh->hlen, ((int) fh->hlen - 1)/ftsblksiz_i + 1, ftsblksiz_i);
if (dsize > 0)
- printf("datasize = %ld bytes = %ld %d-records\n",
+ printf("datasize = %ld bytes = %ld %ld-records\n",
dsize, (dsize-1)/ftsblksiz_i + 1, ftsblksiz_i);
else
printf("datasize = 0 bytes = 0 %d-records\n", ftsblksiz_i);
@@ -2038,11 +2038,14 @@ int fts_sdata(
if (nskip % ftsblksiz_i) {
ntail = ftsblksiz_i - nskip % ftsblksiz_i;
- dprintf(1,"Skipping %ld bytes for empty tail\n",ntail);
+ dprintf(1,"Skipping %ld bytes for empty tail in data\n",ntail);
nskip += ntail;
}
nskip -= fh->nread;
- if (nskip>0) fseek(instr, nskip, 1);
+ if (nskip>0) {
+ dprintf(1,"fseek %ld from %ld\n", nskip, fh->nread);
+ fseek(instr, nskip, 1); // SEEK_CUR=1
+ }
return 1;
}
@@ -2058,7 +2061,7 @@ int fts_cdata(
bool trailr, /* (i) need to read trailing end too ? */
bool trailw) /* (i) need to write trailing end too ? */
{
- int nread, nwrite, n, ntowrite, ntoread, itemlen, nitems;
+ size_t n, nread, nwrite, ntowrite, ntoread, itemlen, nitems;
char buffer[CONVBUFLEN];
itemlen = ABS(fh->bitpix)/8; /* # bytes in an item */
@@ -2073,7 +2076,7 @@ int fts_cdata(
ntowrite = n;
if (n==0) ntoread=0; /* no data? */
if (fh->flip) dprintf(1,"fts_cdata: Swapping bytes\n");
- dprintf(2,"fts_data: ntoread=%d ntowrite=%d\n",ntoread,ntowrite);
+ dprintf(2,"fts_data: ntoread=%ld ntowrite=%ld\n",ntoread,ntowrite);
while (ntoread > 0) {
if (ntoread > CONVBUFLEN)
n = CONVBUFLEN;
@@ -2081,17 +2084,17 @@ int fts_cdata(
n = ntoread;
nread = fread(buffer,sizeof(char),n,instr);
if (nread != n) /* should be a warning in raw mode */
- error("Tried to read %d, could only read %d",n,nread);
+ error("Tried to read %ld, could only read %ld",n,nread);
fh->nread += nread;
if (fh->flip) {
nitems = nread/itemlen;
- if (nread%itemlen) error("Bad bufsize %d for byteswap",nread);
+ if (nread%itemlen) error("Bad bufsize %ld for byteswap",nread);
bswap(buffer,itemlen, nitems);
}
/****FIX****/
nwrite = fwrite(buffer,sizeof(char),n,outstr);
if (nwrite != n)
- error("Tried to write %d, could only write %d\n",n,nwrite);
+ error("Tried to write %ld, could only write %ld\n",n,nwrite);
ntoread -= nread;
}
return 1;
@@ -2146,9 +2149,10 @@ int fts_cdata816(
* See also eq. (x.y.z) in the NOST manual
*/
-int fts_dsize(fits_header *fh)
+size_t fts_dsize(fits_header *fh)
{
- int i, size;
+ int i;
+ size_t size = 0;
if (fh->naxis > 0) {
if (fh->naxisn[0] == 0) { /* NAXIS1=0 is for random groups */
@@ -2164,6 +2168,7 @@ int fts_dsize(fits_header *fh)
size *= fh->gcount * ABS(fh->bitpix) / 8;
} else
size = 0;
+ dprintf(1,"fts_dsize: %ld\n", size);
return size;
}
@@ -2171,10 +2176,10 @@ int fts_dsize(fits_header *fh)
* fts_tsize: return trailing size of useable data portion
*/
-int fts_tsize(fits_header *fh)
+size_t fts_tsize(fits_header *fh)
{
- int n = fts_dsize(fh);
- int tail = ROUNDUP(n,ftsblksiz_o);
+ size_t n = fts_dsize(fh);
+ size_t tail = ROUNDUP(n,ftsblksiz_o);
return tail - n;
}
diff --git a/src/image/fits/scanfits.c b/src/image/fits/scanfits.c
index 6c9fc8913..a4a886fc8 100644
--- a/src/image/fits/scanfits.c
+++ b/src/image/fits/scanfits.c
@@ -54,7 +54,7 @@ string defv[] = { /* Standard NEMO keyword+help */
"blocking=1,1\n Two blocking factors (blocksize/2880) for i&o",
"select=header,data\n Select header, data, ...?",
"split=f\n Split fits file into HDU pieces '.#'",
- "VERSION=1.8d\n 13-jul-2021 PJT",
+ "VERSION=1.8e\n 3-jan-2024 PJT",
NULL,
};
@@ -67,7 +67,8 @@ extern string *burststring(string, string);
void nemo_main()
{
stream instr, outstr;
- int i, n, nfile, blocking[2];
+ int i,n,nfile, blocking[2];
+ size_t size;
string outfile, hdselect, *insert, *fix, *deletes, *keep, *print;
char basename[128];
struct fits_header fh;
@@ -115,15 +116,17 @@ void nemo_main()
fts_zero(&fh); /* reset header */
fh.hdu = i; /* keep track of HDU (1=first) */
- n = fts_rhead(&fh,instr); /* read header */
- if (n<0) /* if no data (EOF) .. */
+ size = fts_rhead(&fh,instr); /* read header */
+ if (size == 0 ) { /* if no data (EOF) .. */
+ dprintf(1," no more data\n");
break; /* ... quit */
+ }
if (outstr) dprintf(1,"Working on FITS file %d\n",i);
- fts_dhead(&fh,deletes); /* delete= headers */
- fts_khead(&fh,keep); /* keep= headers */
- fts_ihead(&fh,insert); /* insert= headers */
- fts_fhead(&fh,fix); /* fix= headers */
- if (!outstr && !split) fts_phead(&fh,print); /* ? print header */
+ fts_dhead(&fh,deletes); /* delete= headers */
+ fts_khead(&fh,keep); /* keep= headers */
+ fts_ihead(&fh,insert); /* insert= headers */
+ fts_fhead(&fh,fix); /* fix= headers */
+ if (!outstr && !split) fts_phead(&fh,print); /* ? print header */
if ((outstr && (nfile==0 || nfile==i)) || split) {
if (split) {
if (strchr(outfile,'%'))
diff --git a/src/image/io/image.c b/src/image/io/image.c
index a13984b79..6a2259975 100644
--- a/src/image/io/image.c
+++ b/src/image/io/image.c
@@ -33,7 +33,8 @@
* 13-feb-13 V8.1 added region and sub_image() PJT
* 22-may-21 V8.2 deal with Object
* 19-mar-22 V8.3 deprecate Axis=0 images
- * 17-dec-22 deal with Telescope/Object/Unit
+ * 17-dec-22 deal with Telescope/Object/Unit
+ * 2-jan-24 fix alloc large cubes
*
*
* Example of usage: see snapccd.c for writing
@@ -319,7 +320,9 @@ int read_image (stream instr, imageptr *iptr)
get_set (instr,MapTag);
if (Frame(*iptr)==NULL) { /* check if allocated */
- nxyz = Nx(*iptr)*Ny(*iptr)*Nz(*iptr);
+ nxyz = Nx(*iptr);
+ nxyz *= Ny(*iptr);
+ nxyz *= Nz(*iptr);
Frame(*iptr) = (real *) allocate(nxyz * sizeof(real));
dprintf (DLEV,"Frame allocated @ %p ",Frame(*iptr));
} else
diff --git a/src/kernel/io/dprintf.c b/src/kernel/io/dprintf.c
index d955318b4..21b60060d 100644
--- a/src/kernel/io/dprintf.c
+++ b/src/kernel/io/dprintf.c
@@ -33,7 +33,7 @@
int debug_level=0; /* needs to be global; see also getparam.c */
-static char *nemo_file = "dprintf.c: debugging stuff";
+// static char *nemo_file = "dprintf.c: debugging stuff";
bool nemo_debug(int debug)
{
diff --git a/src/kernel/io/filesecret.c b/src/kernel/io/filesecret.c
index 2e23d7eff..616b74314 100644
--- a/src/kernel/io/filesecret.c
+++ b/src/kernel/io/filesecret.c
@@ -42,7 +42,8 @@
* V 3.3 25-may-07 pjt handle > 2GB objects in memory (Pierre Fortin )
* V 3.4 12-dec-09 pjt support the new halfp type for I/O (see also csf)
* 27-Sep-10 jcl MINGW32/WINDOWS support
- * 3.5 8-jun-13 pjt eltcnt type fixed for 64bit so it handles > 2B
+ * 3.5 8-jun-13 pjt eltcnt type fixed for 64bit so it handles > 2GB
+ * 3.6 2-jan-24 pjt subtle fix for items > 64bit; now using off_t and size_t
*
* Although the SWAP test is done on input for every item - for deferred
* input it may fail if in the mean time another file was read which was
@@ -92,6 +93,7 @@ void copy_item(stream ostr, stream istr, string tag)
dims = get_dims(istr, tag); /* find out about shape */
dlen = get_dlen(istr, tag); /* and length in bytes */
if(dlen<0) error("copy_item: %s with dlen=%d",tag,dlen); /* yuck */
+ dprintf(0,"copy_item: %ld\n",dlen);
buf = (byte *) calloc(dlen,1); /* get space for a buffer */
if (buf == NULL) /* and check for error */
error("copy_item: item %s: not enuf memory", tag);
@@ -137,7 +139,9 @@ void copy_item_cvt(stream ostr, stream istr, string tag, string *cvt)
if (! streq(type, SetType)) { /* a basic type or array? */
dims = get_dims(istr, tag); /* find out about shape */
dlen = get_dlen(istr, tag); /* and length in bytes */
- if(dlen<0) error("copy_item_cvt: %s with dlen=%d",tag,dlen);
+ if(dlen<0) error("copy_item_cvt: %s with dlen=%d",tag,dlen);
+ //PJT
+ dprintf(1,"copy_item_cvt: %ld\n",dlen);
bufin = (byte *) calloc(dlen,1); /* get space for a buffer */
if (bufin == NULL) /* and check for error */
error("copy_item_cvt: item %s: not enuf memory", tag);
@@ -727,14 +731,14 @@ string get_string(
dp == NULL || *dp++ == 0 || *dp != 0) /* and shape of data */
error("get_string: item %s: not plural char", tag);
dlen = datlen(ipt,0);
- if(dlen<0) error("get_string: %s with dlen=%d",tag,dlen); /* yuck */
+ if(dlen<0) error("get_string: %s with dlen=%ld",tag,dlen); /* yuck */
dat = (char *) calloc(dlen,1); /* alloc memory for string */
if (dat == NULL) /* did alloc fail? */
error("get_string: item %s: not enuf memory", tag);
copydata(dat, 0, dlen, ipt, str); /* copy string from input */
if (sspt->ss_stp == -1) /* item read at top level? */
freeitem(ipt, TRUE); /* yes, so free it up */
- return (dat); /* return string as value */
+ return dat; /* return string as value */
}
/*
@@ -752,10 +756,10 @@ bool get_tag_ok(
sspt = findstream(str); /* lookup associated entry */
if (sspt->ss_stp == -1) { /* input from top level? */
ipt = nextitem(sspt); /* look at next item */
- return (ipt != NULL && streq(tag, ItemTag(ipt)));
+ return ipt != NULL && streq(tag, ItemTag(ipt));
/* test existance and tag */
} else /* input within a set? */
- return (finditem(sspt, tag) != NULL); /* test success of scan */
+ return finditem(sspt, tag) != NULL; /* test success of scan */
}
/*
@@ -773,7 +777,7 @@ stream str
if (sspt->ss_stp == -1) { /* input from top level? */
ipt = nextitem(sspt); /* get next item read in */
if (ipt == NULL) /* nothing left in input? */
- return (NULL); /* then return nothing */
+ return NULL; /* then return nothing */
*tpt++ = (string) copxstr(ItemTag(ipt), sizeof(char));
/* make copy of item tag */
} else { /* input within a set? */
@@ -784,7 +788,7 @@ stream str
/* make copy of item tag */
}
*tpt = NULL; /* terminate string of tags */
- return ((string *) copxstr(tags, sizeof(string)));
+ return (string *) copxstr(tags, sizeof(string));
/* return copy of copies */
}
@@ -806,7 +810,7 @@ string get_type(
error("get_type: at EOF");
if (sspt->ss_stp == -1) /* was input at top level? */
sspt->ss_stk[0] = ipt; /* put back for next time */
- return ((string) copxstr(ItemTyp(ipt), sizeof(char)));
+ return (string) copxstr(ItemTyp(ipt), sizeof(char));
/* return copy of type */
}
@@ -824,7 +828,7 @@ int *get_dims(
if (sspt->ss_stp == -1) /* was input at top level? */
sspt->ss_stk[0] = ipt; /* put back for next time */
if (ItemDim(ipt) != NULL) /* any dimensions to item? */
- return ((int *) copxstr(ItemDim(ipt), sizeof(int)));
+ return (int *) copxstr(ItemDim(ipt), sizeof(int));
/* return copy of dims */
else
return NULL;
@@ -884,19 +888,19 @@ local bool writeitem(stream str, itemptr ipt)
itemptr *setp, tesp;
if (! streq(ItemTyp(ipt), SetType)) /* just a simple item? */
- return (putitem(str, ipt)); /* then write it out */
+ return putitem(str, ipt); /* then write it out */
else { /* recursively handle set */
if (! putitem(str, ipt)) /* write out set item */
- return (FALSE);
+ return FALSE;
setp = (itemptr*) ItemDat(ipt); /* point to item string */
while (*setp != NULL) /* loop over the items */
writeitem(str, *setp++); /* write each one out */
tesp = makeitem(TesType, NULL, NULL, NULL);
/* create closing item */
if (! putitem(str, tesp)) /* write out tes item */
- return (FALSE);
+ return FALSE;
freeitem(tesp, FALSE); /* and reclaim memory */
- return (TRUE); /* indicate success */
+ return TRUE; /* indicate success */
}
}
@@ -907,12 +911,12 @@ local bool writeitem(stream str, itemptr ipt)
local bool putitem(stream str, itemptr ipt)
{
if (! puthdr(str, ipt)) /* write item header */
- return (FALSE);
+ return FALSE;
if (! streq(ItemTyp(ipt), SetType) && ! streq(ItemTyp(ipt), TesType))
/* an ordinary data item? */
if (! putdat(str, ipt)) /* write item data */
- return (FALSE);
- return (TRUE); /* indicate success */
+ return FALSE;
+ return TRUE ; /* indicate success */
}
/*
@@ -927,22 +931,22 @@ local bool puthdr(stream str, itemptr ipt)
num = (ItemDim(ipt) == NULL) ? SingMagic : PlurMagic;
/* determine magic number */
if (fwrite((char *)&num, sizeof(short), 1, str) != 1)
- return (FALSE); /* return FALSE on failure */
+ return FALSE; /* return FALSE on failure */
if (! putxstr(str, ItemTyp(ipt), sizeof(char)))
- return (FALSE);
+ return FALSE;
if (ItemTag(ipt) != NULL) { /* is item tagged? */
if (xstrlen(ItemTag(ipt), sizeof(char)) > MaxTagLen)
error("puthdr: tag too long");
if (! putxstr(str, ItemTag(ipt), sizeof(char)))
- return (FALSE); /* write item tag */
+ return FALSE; /* write item tag */
}
if (ItemDim(ipt) != NULL) { /* a vectorized item? */
if (xstrlen(ItemDim(ipt), sizeof(int)) > MaxVecDim)
error("puthdr: too many dimensions");
if (! putxstr(str, ItemDim(ipt), sizeof(int)))
- return (FALSE); /* write vect dims */
+ return FALSE; /* write vect dims */
}
- return(TRUE); /* indicate success */
+ return TRUE; /* indicate success */
}
/*
@@ -956,7 +960,7 @@ local bool putdat(stream str, itemptr ipt)
if (ItemDat(ipt) == NULL) /* no data to write? */
error("putdat: item %s has no data", ItemTag(ipt));
len = datlen(ipt, 0); /* count bytes to output */
- return (fwrite((char*)ItemDat(ipt), sizeof(byte), len, str) == len);
+ return fwrite((char*)ItemDat(ipt), sizeof(byte), len, str) == len;
/* write data to stream */
}
@@ -986,7 +990,7 @@ local itemptr scantag(strstkptr sspt, string tag)
error("scantag: item %s not found in set %s",
tag, ItemTag(sspt->ss_stk[sspt->ss_stp]));
}
- return (ipt); /* return item found */
+ return ipt; /* return item found */
}
local itemptr nextitem(strstkptr sspt)
@@ -999,7 +1003,7 @@ local itemptr nextitem(strstkptr sspt)
ipt = readitem(sspt->ss_str, NULL); /* read next item in */
sspt->ss_stk[0] = ipt; /* and save for later */
}
- return (ipt); /* supply item to caller */
+ return ipt; /* supply item to caller */
}
local itemptr finditem(strstkptr sspt, string tag)
@@ -1013,7 +1017,7 @@ local itemptr finditem(strstkptr sspt, string tag)
break; /* done with loop */
ivec++; /* on to next item */
}
- return (*ivec); /* return item or NULL */
+ return *ivec; /* return item or NULL */
}
/*
@@ -1027,7 +1031,7 @@ local itemptr readitem(stream str, itemptr first)
ip = first != NULL ? first : getitem(str); /* use 1st or next item */
if (ip == NULL || /* EOF detected by getitem */
! streq(ItemTyp(ip), SetType)) /* or item not a set? */
- return (ip); /* just return it */
+ return ip; /* just return it */
bufp = &ibuf[0]; /* prepare item buffer */
for ( ; ; ) { /* loop reading items in */
if (bufp >= &ibuf[MaxSetLen]) /* no room for next? */
@@ -1045,7 +1049,7 @@ local itemptr readitem(stream str, itemptr first)
/* construct compound item */
freeitem(ip, TRUE); /* reclaim orig. header */
freeitem(np, TRUE);
- return (res); /* return compound item */
+ return res; /* return compound item */
}
/*
@@ -1058,11 +1062,11 @@ local itemptr getitem(stream str)
ipt = gethdr(str); /* try reading header in */
if (ipt == NULL) /* did gethdr detect EOF? */
- return (NULL); /* return NULL on EOF */
+ return NULL; /* return NULL on EOF */
if (! streq(ItemTyp(ipt), SetType) && /* if item does not start */
! streq(ItemTyp(ipt), TesType)) /* or terminate a set */
getdat(ipt, str); /* try reading data in */
- return (ipt); /* return resulting item */
+ return ipt; /* return resulting item */
}
/*
@@ -1072,7 +1076,7 @@ local itemptr getitem(stream str)
local itemptr gethdr(stream str)
{
short num;
- string typ, tag;
+ string tag, typ = NULL;
int *dim, *ip; /* ISSWAP */
permanent bool firsttime = TRUE;
@@ -1128,7 +1132,7 @@ local itemptr gethdr(stream str)
#endif
} else
dim = NULL;
- return (makeitem(typ, tag, NULL, dim)); /* return item less data */
+ return makeitem(typ, tag, NULL, dim); /* return item less data */
} /* gethdr */
/*
* GETHDR: read a item header from a stream.
@@ -1213,20 +1217,29 @@ local copyproc copyfun(string srctyp, string destyp)
local void copydata(
void *vdat,
- int off,
- int len,
+ off_t off,
+ size_t len,
itemptr ipt,
stream str)
{
char *src, *dat = (char *) vdat;
off_t oldpos;
-
+
off *= ItemLen(ipt); /* offset bytes from start */
if (ItemDat(ipt) != NULL) { /* data already in core? */
src = (char *) ItemDat(ipt) + off; /* get pointer to source */
len *= ItemLen(ipt); /* number of bytes to copy */
- while (--len >= 0) /* loop copying data */
+#if 0
+ // this while loop will hit through 0, and eventually segfault in the copy
+ // compiler bug ?
+ // this pattern happened a few times below, where they've all been converted to the for() loop
+ while (--len >= 0) { /* loop copying data */
*dat++ = *src++; /* byte by byte */
+ }
+#else
+ for (size_t i=0; i BUG */
- while (--len >= 0) /* loop converting data */
+ for (size_t i=0; i= 0) /* loop reading data */
- *dat++ = (double) getflt(str); /* float to double */
+ for (size_t i=0; i= 0) /* loop converting data */
- *dat++ = (float) *src++; /* double to float */
+ for (size_t i=0; i= 0) /* loop reading data */
- *dat++ = (float) getdbl(str); /* double to float */
+ for (size_t i=0; itl_typ != NULL; tp++) /* loop over basic types */
- if (streq(typ, tp->tl_typ)) /* found type we want? */
- return (tp->tl_len); /* return byte length */
+ if (streq(typ, tp->tl_typ)) /* found type we want? */
+ return tp->tl_len; /* return byte length */
error("baselen: type %s unknown", typ); /* bad type string */
- return (0); /* will never get here */
+ return 0; /* will never get here */
}
/************************************************************************/
@@ -1488,13 +1500,13 @@ local strstkptr findstream(stream str)
strstkptr stfree, sspt;
if (last!=NULL && last->ss_str == str)
- return(last);
+ return last;
stfree = NULL; /* remember free slot */
for (sspt = strtable; sspt < strtable+StrTabLen; sspt++) {
/* loop over the table */
if (sspt->ss_str == str) { /* found that stream? */
last = sspt;
- return (sspt); /* then return slot */
+ return sspt; /* then return slot */
}
if (stfree == NULL && sspt->ss_str == NULL)
/* first empty slot? */
@@ -1512,7 +1524,7 @@ local strstkptr findstream(stream str)
stfree->ss_pos = 0L; /* set at start of file */
#endif
last = stfree; /* mark for quick access */
- return (stfree); /* return new slot */
+ return stfree; /* return new slot */
}
local void ss_push(strstkptr sspt, itemptr ipt)
diff --git a/src/kernel/io/filesecret.h b/src/kernel/io/filesecret.h
index 578f9ce71..07b53d108 100644
--- a/src/kernel/io/filesecret.h
+++ b/src/kernel/io/filesecret.h
@@ -32,7 +32,7 @@
typedef struct {
string itemtyp; /* type string, listed in filestruct.h */
- long itemlen; /* length associated with above type */
+ size_t itemlen; /* length associated with above type */
string itemtag; /* name given this item by application */
int *itemdim; /* int-string of dimensions, or NULL */
void *itemdat; /* the real goodies, if any, or NULL */
@@ -74,16 +74,16 @@ typedef struct {
typedef struct {
string tl_typ; /* type string (see filestruct.h) */
- int tl_len; /* length of elements in bytes */
+ size_t tl_len; /* length of elements in bytes */
} typlen, *typlenptr;
/* PROC_COPY's :
* replaces the old "proc" type unsafe stuff (for
* good practice for C, but needed for C++)
*/
-typedef void (*copyproc) (void *, int, int, itemptr, stream);
-typedef void (*copyproc_d)(double *, int, int, itemptr, stream);
-typedef void (*copyproc_f)(float *, int, int, itemptr, stream);
+typedef void (*copyproc) (void *, off_t, size_t, itemptr, stream);
+typedef void (*copyproc_d)(double *, off_t, size_t, itemptr, stream);
+typedef void (*copyproc_f)(float *, off_t, size_t, itemptr, stream);
/*
@@ -104,18 +104,18 @@ local itemptr getitem ( stream str );
local itemptr gethdr ( stream str );
local void getdat ( itemptr ipt, stream str );
local copyproc copyfun ( string srctyp, string destyp );
-local void copydata ( void *dat, int off, int len, itemptr ipt, stream str );
-local void copydata_f2d( double *dat, int off, int len, itemptr ipt, stream str );
-local void copydata_d2f( float *dat, int off, int len, itemptr ipt, stream str );
+local void copydata ( void *dat, off_t off, size_t len, itemptr ipt, stream str );
+local void copydata_f2d( double *dat, off_t off, size_t len, itemptr ipt, stream str );
+local void copydata_d2f( float *dat, off_t off, size_t len, itemptr ipt, stream str );
local float getflt ( stream str );
local double getdbl ( stream str );
-local void saferead ( void *dat, int siz, int cnt, stream str );
+local void saferead ( void *dat, size_t siz, size_t cnt, stream str );
local void safeseek ( stream str, off_t offset, int key );
-local long eltcnt ( itemptr ipt, int skp );
-local size_t datlen ( itemptr ipt, int skp );
+local size_t eltcnt ( itemptr ipt, int dimskp );
+local size_t datlen ( itemptr ipt, int dimskp );
local itemptr makeitem ( string typ, string tag, void *dat, int *dim );
local void freeitem ( itemptr ipt, bool flg);
-local int baselen ( string typ );
+local size_t baselen ( string typ );
local strstkptr findstream ( stream str );
local void ss_push ( strstkptr sspt, itemptr ipt );
local void ss_pop ( strstkptr sspt );
diff --git a/src/nbody/init/mkhomsph.c b/src/nbody/init/mkhomsph.c
index f7b71e4ee..eb308ebc9 100644
--- a/src/nbody/init/mkhomsph.c
+++ b/src/nbody/init/mkhomsph.c
@@ -26,7 +26,7 @@
#include
#include
-string defv[] = { /* DEFAULT INPUT PARAMETERS */
+string defv[] = {
"out=???\n Output file name",
"nbody=2048\n Number of particles",
"rmin=0\n Inner cutoff radius",
@@ -37,14 +37,12 @@ string defv[] = { /* DEFAULT INPUT PARAMETERS */
"seed=0\n Random number seed",
"zerocm=t\n Center c.o.m. ?",
"headline=\n Text headline for output",
- "VERSION=1.4c\n 29-aug-2018 PJT",
+ "VERSION=1.4d\n 2-jan-2024 PJT",
NULL,
};
string usage = "create a uniform sphere of equal massive stars";
-string cvsid="$Id$";
-
local real rmin, rmax;
local real virial, power, vmax;
@@ -76,6 +74,7 @@ void nemo_main(void)
power = getdparam("power");
if (power >= 3.0) error("Illegal power=%g, must be < 3.0",power);
seed = init_xrandom(getparam("seed"));
+ dprintf(0,"seed=%d\n",seed);
zerocm = getbparam("zerocm");
mksphere();
writegalaxy(getparam("out"), getparam("headline"));
@@ -107,7 +106,6 @@ void mksphere(void)
{
Body *bp;
real rmin3, rmax3, r_i, v_i, theta_i, phi_i, mass_i, sigma = 0.0;
- real sinp, cosp, sint, cost;
int i;
btab = (Body *) allocate(nbody * sizeof(Body));
@@ -126,7 +124,8 @@ void mksphere(void)
dprintf(1,"Gaussian isotropic velocities: sigma=%g\n",sigma);
} else if (vmax > 0.0) {
dprintf(1,"Vmax isotropic velocities: vmax=%g\n",vmax);
- }
+ }
+ // @todo use local sin/cos variables to make it more efficient
for (bp=btab, i = 0; i < nbody; bp++, i++) {
Mass(bp) = mass_i;
#ifdef OLD
diff --git a/src/scripts/Makefile b/src/scripts/Makefile
index 0e1ae954d..f18cb9df8 100644
--- a/src/scripts/Makefile
+++ b/src/scripts/Makefile
@@ -7,7 +7,7 @@ SCRIPTS = mknemo mkpdoc manlaser nemoman nemobench ldso nemo.version \
fitsedit fitsmerge nds9 shtool nemo.coverage showvar mir2nemo getline pedit \
wgetc yapp_query nemoplot nemopars nemo \
sbatch_nemo.sh nemo_functions.sh progress_bar.sh \
- python/tabplot.py python/fitsplot.py
+ python/tabplot.py python/fitsplot.py python/mkrunfile.py
MAKES = Makefile.lib
help:
diff --git a/src/scripts/mknemo.d/pgplot b/src/scripts/mknemo.d/pgplot
index fb6e8ac29..96a059284 100755
--- a/src/scripts/mknemo.d/pgplot
+++ b/src/scripts/mknemo.d/pgplot
@@ -22,7 +22,7 @@ $NEMO/src/scripts/pgplot.install png=1 $*
echo "Installed version=$version in $NEMOLIB"
echo pgplot $version `date` >> $NEMO/opt/mknemo.log
-echo "You may need to rerun configure:"
+echo "You may need to rerun NEMO's configure:"
with_yapp="--with-yapp=pgplot --enable-png --with-pgplot-prefix=$NEMOLIB"
echo " ./configure $with_yapp"
echo " source nemo_start.sh"
diff --git a/src/scripts/python/mkrunfile.py b/src/scripts/python/mkrunfile.py
new file mode 100755
index 000000000..1a9a31a1c
--- /dev/null
+++ b/src/scripts/python/mkrunfile.py
@@ -0,0 +1,115 @@
+#! /usr/bin/env python3
+#
+#
+
+import sys
+import numpy as np
+
+# simple usage
+if len(sys.argv) < 3:
+ print("Usage: %s [options[ progname p1=v11,v12... p2=..." % sys.argv[0])
+ print("[options] are not parsed yet")
+ sys.exit(0)
+
+# parse command line args
+progname = sys.argv[1]
+pars = sys.argv[2:]
+
+npar = len(pars)
+par = list(range(npar))
+val = list(range(npar))
+
+# setting
+for i in range(npar):
+ par[i] = 'par%d' % i
+ par[i] = pars[i].split('=')[0]
+ val[i] = pars[i].split('=')[1].split(',')
+
+
+# case 1: -l rundir,run_%04d,1 first case
+rundir = 'rundir'
+run = 'run_%04d'
+i0 = 1
+
+# case 2: -l rundir,run_ second case
+# case 3: -l rundir,run/ this case
+
+
+# loop over pars and vals and create the cmds for the runfile
+# @todo generalize this
+
+if npar == 1:
+ for p0 in range(len(val[0])):
+ _rdc = run % i0
+ _l = '%s=%s' % (rundir, _rdc)
+ i0 = i0 + 1
+ cmd = '%s %s %s=%s' % (progname, _l, par[0], val[0][p0])
+ print(cmd)
+elif npar == 2:
+ for p1 in range(len(val[1])):
+ for p0 in range(len(val[0])):
+ _rdc = run % i0
+ _l = '%s=%s' % (rundir, _rdc)
+ i0 = i0 + 1
+ cmd = '%s %s %s=%s %s=%s' % (progname, _l, par[0], val[0][p0], par[1], val[1][p1])
+ print(cmd)
+elif npar == 3:
+ for p2 in range(len(val[2])):
+ for p1 in range(len(val[1])):
+ for p0 in range(len(val[0])):
+ _rdc = run % i0
+ _l = '%s=%s' % (rundir, _rdc)
+ i0 = i0 + 1
+ cmd = '%s %s %s=%s %s=%s %s=%s' % (progname, _l, par[0], val[0][p0], par[1], val[1][p1], par[2], val[2][p2])
+ print(cmd)
+elif npar == 4:
+ for p3 in range(len(val[3])):
+ for p2 in range(len(val[2])):
+ for p1 in range(len(val[1])):
+ for p0 in range(len(val[0])):
+ _rdc = run % i0
+ _l = '%s=%s' % (rundir, _rdc)
+ i0 = i0 + 1
+ cmd = '%s %s %s=%s %s=%s %s=%s %s=%s' % (progname, _l, par[0], val[0][p0], par[1], val[1][p1], par[2], val[2][p2], par[3], val[3][p3])
+ print(cmd)
+else:
+ print("# too many parameters")
+
+
+
+_future = """
+mkrunfile.py progname a=1,2 b=2,4 c=10
+gives:
+ progname a=1 b=2 c=10
+ progname a=2 b=2 c=10
+ progname a=1 b=4 c=10
+ progname a=2 b=4 c=10
+
+Of course this lacks an important feature: what if each line needs another parameter (e.g. rundir=) in
+which the parameter values, or an ordinal, is encoded. E.g.
+
+ progname a=1 b=2 c=10 rundir=run010
+ progname a=2 b=2 c=10 rundir=run011
+ progname a=1 b=4 c=10 rundir=run012
+ progname a=2 b=4 c=10 rundir=run013
+
+where the base number (run010) can be choosen as well
+
+or
+ progname a=1 b=2 c=10 rundir=run_1_2_10
+ progname a=2 b=2 c=10 rundir=run_2_2_10
+ progname a=1 b=4 c=10 rundir=run_1_4_10
+ progname a=2 b=4 c=10 rundir=run_2_4_10
+
+or even
+ progname a=1 b=2 c=10 rundir=runs/1/2/10
+ progname a=2 b=2 c=10 rundir=runs/2/2/10
+ progname a=1 b=4 c=10 rundir=runs/1/4/10
+ progname a=2 b=4 c=10 rundir=runs/2/4/10
+
+so perhaps:
+
+ -l rundir,run%03d,10 first case
+ -l rundir,run_ second case
+ -l rundir,run/ this case
+"""