Merge branch 'master' of https://github.com/teuben/nemo

VERSION

@@ -1 +1 @@
-4.5.0
+4.5.1

docs/source/scripting.rst

@@ -77,6 +77,46 @@ the --help command line option
 where possible. 
 
 
+Example:  Creating a runfile
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A runfile is a simple text file, where each line in this file can be executed, for example using 
+bash in a serial fashion.  If each line is independant from all others, they can be executed in parallel using
+``gnu parallel`` or ``sbatch`` (a common HPC batch queue system).  For example the following command
+in NEMO produces a runfile with 4 lines:
+
+.. code-block::
+
+     mkrunfile.py progname a=1,2 b=2,4 c=10 > example1.run
+
+     progname a=1 b=2 c=10
+     progname a=2 b=2 c=10
+     progname a=1 b=4 c=10
+     progname a=2 b=4 c=10
+
+where the ``example1.run`` file can be executed with any of the following commands. Depending on your
+resources of course. Memory, number of cores etc.
+
+.. code-block::
+
+     bash example1.run
+     parallel -j 4 < example1.run
+     sbatch_nemo.sh example1.run
+
+in particular the last ``sbatch_nemo.sh`` example will likely need to be tailored for your HPC system.
+
+.. note::
+   Unless the parameters take care of this, you will need to ensure data are written to files that do not
+   collide with each other.  For example a directory or file that encodes the values of the parameters,
+   or are numerically sorted (e.g. run010, run011)
+   Currently ``mkrunfile.py`` does not have an automated way for this yet.
+
+And here is an example of creating a runfile from a table with values
+
+.. code-block::
+
+     awk '{printf("progname runfile=run_%s a=%s b=%s\n", $1,$2,$3) }' example1.tab
+
 
 Example:  Extracting results from run directories
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/whatsnew.html

@@ -22,13 +22,15 @@ <H1> What's New in NEMO </H1>
 -->
 
 <A NAME=4_5_0>
-  <H2> 4.5.0:  in git </H2>
+  <H2> 4.5.0/4.5.1:  in git </H2>
   <UL>  
-    <LI> (Jan 2, 2025)   fixed very old 64bit itemsize limitation [issue17]
+    <LI> (Jan 3, 2025)   added mkrunfile.py to create runfiles looping over parameters
+    <LI> (Jan 3, 2025)   fixed very old 64bit itemsize limitation [issue17],
+ 	   this affected at least ccdfits, fitsccd and scanfits.
   </UL>
 
 <A NAME=4_4_3>
-  <H2> 4.4.3/4.4.4/4.4.5 (a.k.a. Malta Release) </H2>
+  <H2> 4.4.3/4.4.4/4.4.5: 22-dec-2024 (a.k.a. Malta Release) </H2>
   <UL>
     <LI> (Dec 22, 2024) define TIMEFUZZ in stdinc.h, not per program
     <LI> (Dec 12, 2024) drafting two codes: diskspectrum and lineid

inc/fits.h

@@ -1,8 +1,10 @@
 /*
- *  definitions and declarations for nemo's FITS.C
+ *  definitions and declarations for nemo's src/image/fits/fits.c
+ *
  *  22-feb-94  ansi- and C++ safe
  *  10-aug-09  size_t instead of int for 2GB+ files
  *  25-oct-20  add hdu counter
+ *   3-jan-24  more proper usage of size_t instead of int
  */
 
 #ifndef _fits_h_
@@ -129,7 +131,7 @@ static struct arglist {
 typedef struct fits_header {
     int hdu;            /* keep track with HDU this is (1 being the first) */
     /* Optionally we could store the buffer in the first structure too... */
-    char *buffer;       /* pointer to exact header copy */
+    char *buffer;       /* pointer to exact header copy , if present */
     size_t buflen;      /* current length of buffer */
     size_t hlen;        /* length of header (bytes, through the END keyword) */
     size_t dlen;        /* length of data (bytes) */
@@ -200,8 +202,8 @@ typedef struct fits_header {
 } fits_header;
 
 
-size_t fts_rhead    (fits_header *, stream);
-char *fts_shead    (fits_header *, string);
+size_t fts_rhead (fits_header *, stream);
+char *fts_shead  (fits_header *, string);
 int fts_lhead    (fits_header *);
 int fts_chead    (fits_header *, stream);
 int fts_thead    (fits_header *);
@@ -211,7 +213,7 @@ int fts_ihead    (fits_header *, string *);
 int fts_fhead    (fits_header *, string *);
 int fts_phead    (fits_header *, string *);
 int fts_whead    (fits_header *, stream);
-size_t fts_xhead    (fits_header *, stream, int, int, int *, int);
+size_t fts_xhead (fits_header *, stream, int, int, int *, int);
 
 int fts_ptable   (fits_header *, stream, string *, string, int *, int);
 int fts_pgroup   (fits_header *, stream, string *, string, int *, int);
@@ -220,8 +222,8 @@ int fts_buf      (int);
 
 int fts_sdata    (fits_header *, stream);
 int fts_cdata    (fits_header *, stream, stream, bool, bool);
-int fts_dsize    (fits_header *);
-int fts_tsize    (fits_header *);
+size_t fts_dsize (fits_header *);
+size_t fts_tsize (fits_header *);
 int fts_rdata    (fits_header *, stream, int, char *);
 int fts_wdata    (fits_header *, stream, int, char *);
 int fts_rrow     (fits_header *, stream, int, char *);

inc/version.h

@@ -1,5 +1,5 @@
 /* do not edit - created by the src/scripts/version script */
 #if defined(GETPARAM_VERSION_ID)
-static char *version_h = "git_12231"; // git rev-list --count HEAD
+static char *version_h = "git_12284"; // git rev-list --count HEAD
 #endif
-#define NEMO_VERSION "4.5.0"
+#define NEMO_VERSION "4.5.1"

man/man3/filestruct.3

@@ -1,4 +1,4 @@
-.TH FILESTRUCT 3NEMO "7 April 2012"
+.TH FILESTRUCT 3NEMO "2 January 2024"
 
 .SH "NAME"
 filestruct \- primitives for structured binary file I/O
@@ -274,11 +274,12 @@ https://github.com/syrte/snapio - python implementation of snap I/O
 
 .SH "UPDATE HISTORY"
 .nf
-.ta +2.0i +2.0i
+.ta +1.25i +4.5i
 4-Apr-87	original implementation		JEB
 30-Aug-87	type coersion, deferred input	LPH
 16-Apr-88	new types, operators, etc	JEB
 16-May-92	random access to data   	PJT
 5-mar-94	documented qsf          	PJT
 2-jun-05	added blocked I/O		PJT
+2-jan-2024	fix 64bit problem for big items	PJT
 .fi

man/man5/filestruct.5

@@ -1,7 +1,9 @@
-.TH FILESTRUCT 5NEMO "16 May 1992"
-.SH NAME
+.TH FILESTRUCT 5NEMO "2 January 2024"
+
+.SH "NAME"
 filestruct \- binary structured file format 
-.SH SYNOPSIS
+
+.SH "SYNOPSIS"
 .nf
 \fB#include <stdinc.h>\fP
 \fB#include "filesecret.h"\fP        \fI only for local code\fP
@@ -19,23 +21,26 @@ filestruct \- binary structured file format
 .PP
 \fB struct item {\fP
 \fB    string itemtyp;\fP
-\fB    int    itemlen;\fP
+\fB    size_t itemlen;\fP
 \fB    string itemtag;\fP
 \fB    int   *itemdim;\fP
-\fB    byte  *itemdat;\fP
-\fB    long   itempos;\fP
+\fB    void  *itemdat;\fP
+\fB    off_t  itempos;\fP
+\fB    off_t  itemoff;\fP
 \fB };\fP
 .PP
 \fB struct strstk {\fP
 \fB    stream  ss_str;\fP
 \fB    item   *ss_stk[SetStkLen];\fP
 \fB    int     ss_stp;\fP
 \fB    bool    ss_seek;\fP
-\fB    long    ss_pos;\fP            \fI/* only if RANDOM access allowed */\fP
+\fB    int     ss_mode;\fP           \fI/* only if RANDOM access allowed */\fP
+\fB    off_t   ss_pos;\fP            \fI/* only if RANDOM access allowed */\fP
 \fB    itemptr ss_ran;\fP            \fI/* only if RANDOM access allowed */\fP
 \fB };\fP
 .fi
-.SH DESCRIPTION
+
+.SH "DESCRIPTION"
 \fIfilestruct\fP is a method for storing data files largely
 consisting of character strings, booleans, and possible
 multi-dimensional arrays of data.  This data may be structured in a
@@ -74,7 +79,8 @@ otherwise. This is essentially the reason why pipes cannot be used in
 The actual internal format is governed how the application programmer 
 uses the \fIget_XXX\fP and \fPput_XXX\fP routines (see 
 \fIfilestruct(3NEMO)\fP).
-.SH EXPERIMENTAL FEATURES
+
+.SH "EXPERIMENTAL FEATURES"
 If compiled with \fB-DRANDOM\fP 
 some limited random access to data within a data-item is possible.
 .PP
@@ -83,22 +89,27 @@ little/big endian machines. Otherwise,
 data on disk exist in the host format, and no effort has been made to make
 it machine independant (e.g. IEEE floating points and twos-compliment
 integers). This is however expected in some future release.
-.SH ZENO FORMAT
+
+.SH "ZENO FORMAT"
 The \fIzeno(1NEMO)\fP package also used this format, but there are some
-subtle differences to be described.
-.SH FILES
+subtle differences still to be described here.
+
+.SH "FILES"
 .nf
 .ta +2.0i
 ~/src/kernel/io   	filesecret.[ch] filestruct.h
 ~/inc              	filestruct.h
 .fi
-.SH SEE ALSO
+
+.SH "SEE ALSO"
 tsf(1NEMO), rsf(1NEMO), csf(1NEMO), filestruct(3NEMO), zeno(1NEMO)
-.SH AUTHOR
+
+.SH "AUTHOR"
 Joshua Barnes, Lyman Hurd, Peter Teuben
-.SH UPDATE HISTORY
+
+.SH "HISTORY"
 .nf
-.ta +1.0i +4.0i
+.ta +1.25i +4.5i
 dark-ages	V0.0 precurser (filestr)	JEB
 xx-apr-87	V1.0 basic operators     	JEB
 xx-jul-87	V1.x added f-d coercion, deferred output	Lyman Hurd
@@ -108,4 +119,5 @@ xx-xxx-87	V2.0 new types, operators, external fmt  	JEB
 16-may-92	V3.0 finalized random access                        	PJT
 6-jul -01	documented the new uNEMO   	PJT
 27-dec-2019	documented ZENO		PJT
-.fi
+2-jan-2024	V3.6 fix remaining 64bit issues by using off_t, size_t	PJT
+

man/man8/tricks.8

@@ -93,7 +93,7 @@ The (bash) array ${PIPESTATUS[@]} contains the status ($?) for each program in
 a unix pipe. In this example it should return "0 0 0".
 
 .SH "PARALLEL and OpenMP"
-Most programs in NEMO are still single-core, but many current machines have a number of cores. If a program only
+Most programs in NEMO are still single-core, but many current machines have multiple cores. If a program only
 uses a fraction of the memory on the machine, the GNU \fIparallel(1)\fP program can be used to spread the load,
 e.g.
 .nf
@@ -109,6 +109,8 @@ created here, a well crafted Runfile can be created, and with
        make -f Runfile -j 2
 .fi
 should achieve the same result.
+.PP
+The \fBmkrunfile.py\fP script can help making such runfiles by looping over a set of parameters.
 
 .PP
 Although NEMO can be configured using \fB--with-openmp\fP to take advantage of multi-cores OpenMP
@@ -129,7 +131,8 @@ should be aware of the user interface implications of controlling how many cores
 .SH "SLURM"
 
 slurm is a popular package you will find on large computer clusters. See also
-\fIsbatch_nemo.sh(8NEMO)\fP for a helper script in NEMO to use slurm.
+\fIsbatch_nemo.sh(8NEMO)\fP for a helper script in NEMO to use slurm. It will likely
+have to be tailored for your specific situation.
 
 .SH "TABLES"