--- ray/src/util/rcollate.c	2014/05/30 18:10:20	2.12
+++ ray/src/util/rcollate.c	2018/10/18 22:59:48	2.27
@@ -1,5 +1,5 @@
 #ifndef lint
-static const char RCSid[] = "$Id: rcollate.c,v 2.12 2014/05/30 18:10:20 greg Exp $";
+static const char RCSid[] = "$Id: rcollate.c,v 2.27 2018/10/18 22:59:48 greg Exp $";
 #endif
 /*
  * Utility to re-order records in a binary or ASCII data file (matrix)
@@ -11,22 +11,15 @@ static const char RCSid[] = "$Id: rcollate.c,v 2.12 20
 #include "platform.h"
 #include "rtio.h"
 #include "resolu.h"
-#ifdef _WIN32
-#undef ftello
-#define	ftello	ftell
-#undef ssize_t
-#define ssize_t	size_t
+#if defined(_WIN32) || defined(_WIN64)
+  #undef ftello
+  #define	ftello	ftell
+  #undef ssize_t
+  #define ssize_t	size_t
 #else
-#include <sys/mman.h>
+  #include <sys/mman.h>
 #endif
 
-#ifdef getc_unlocked            /* avoid horrendous overhead of flockfile */
-#undef getc
-#undef putc
-#define getc    getc_unlocked
-#define putc    putc_unlocked
-#endif
-
 typedef struct {
 	void	*base;		/* pointer to base memory */
 	size_t	len;		/* allocated memory length */
@@ -57,13 +50,52 @@ free_load(MEMLOAD *mp)
 	mp->len = 0;
 }
 
+/* load memory from an input stream, starting from current position */
+static int
+load_stream(MEMLOAD *mp, FILE *fp)
+{
+	size_t	alloced = 0;
+	char	buf[8192];
+	size_t	nr;
+
+	if (mp == NULL)
+		return(-1);
+	mp->base = NULL;
+	mp->len = 0;
+	mp->mapped = 0;
+	if (fp == NULL)
+		return(-1);
+	while ((nr = fread(buf, 1, sizeof(buf), fp)) > 0) {
+		if (!alloced)
+			mp->base = malloc(alloced = nr);
+		else if (mp->len+nr > alloced)
+			mp->base = realloc(mp->base,
+				alloced = alloced*(2+(nr==sizeof(buf)))/2+nr);
+		if (mp->base == NULL)
+			return(-1);
+		memcpy((char *)mp->base + mp->len, buf, nr);
+		mp->len += nr;
+	}
+	if (ferror(fp)) {
+		free_load(mp);
+		return(-1);
+	}
+	if (alloced > mp->len*5/4)	/* don't waste too much space */
+		mp->base = realloc(mp->base, mp->len);
+	return(mp->len > 0);
+}
+
 /* load a file into memory */
 static int
 load_file(MEMLOAD *mp, FILE *fp)
 {
 	int	fd;
-	off_t	skip, flen;
+	off_t	skip, flen, skipped;
 
+#if defined(_WIN32) || defined(_WIN64)
+				/* too difficult to fix this */
+	return load_stream(mp, fp);
+#endif
 	if (mp == NULL)
 		return(-1);
 	mp->base = NULL;
@@ -79,8 +111,9 @@ load_file(MEMLOAD *mp, FILE *fp)
 	mp->len = (size_t)(flen - skip);
 #ifdef MAP_FILE
 	if (mp->len > 1L<<20) {		/* map file if > 1 MByte */
-		mp->base = mmap(NULL, mp->len, PROT_READ, MAP_PRIVATE, fd, skip);
+		mp->base = mmap(NULL, flen, PROT_READ, MAP_PRIVATE, fd, 0);
 		if (mp->base != MAP_FAILED) {
+			mp->base = (char *)mp->base + skip;
 			mp->mapped = 1;
 			return(1);	/* mmap() success */
 		}
@@ -92,48 +125,19 @@ load_file(MEMLOAD *mp, FILE *fp)
 		mp->len = 0;
 		return(-1);
 	}
-	if (read(fd, (char *)mp->base, mp->len) != mp->len) {
-		free_load(mp);
-		return(-1);
+	skipped = skip;			/* read() fails on really big buffers */
+	while (skipped < flen) {
+		ssize_t	nread = read(fd, (char *)mp->base+(skipped-skip),
+				(flen-skipped <= 1L<<30) ? flen-skipped : 1L<<30);
+		if (nread <= 0) {
+			free_load(mp);
+			return(-1);
+		}
+		skipped += nread;
 	}
 	return(1);
 }
 
-/* load memory from an input stream, starting from current position */
-static int
-load_stream(MEMLOAD *mp, FILE *fp)
-{
-	size_t	alloced = 0;
-	char	buf[8192];
-	size_t	nr;
-
-	if (mp == NULL)
-		return(-1);
-	mp->base = NULL;
-	mp->len = 0;
-	mp->mapped = 0;
-	if (fp == NULL)
-		return(-1);
-	while ((nr = fread(buf, 1, sizeof(buf), fp)) > 0) {
-		if (!alloced)
-			mp->base = malloc(nr);
-		else if (mp->len+nr > alloced)
-			mp->base = realloc(mp->base,
-				alloced = alloced*(2+(nr==sizeof(buf)))/2+nr);
-		if (mp->base == NULL)
-			return(-1);
-		memcpy((char *)mp->base + mp->len, buf, nr);
-		mp->len += nr;
-	}
-	if (ferror(fp)) {
-		free_load(mp);
-		return(-1);
-	}
-	if (alloced > mp->len*5/4)	/* don't waste too much space */
-		mp->base = realloc(mp->base, mp->len);
-	return(mp->len > 0);
-}
-
 /* free a record index */
 #define free_records(rp)	free(rp)
 
@@ -141,6 +145,7 @@ load_stream(MEMLOAD *mp, FILE *fp)
 static RECINDEX *
 index_records(const MEMLOAD *mp, int nw_rec)
 {
+	int		nall = 0;
 	RECINDEX	*rp;
 	char		*cp, *mend;
 	int		n;
@@ -149,7 +154,8 @@ index_records(const MEMLOAD *mp, int nw_rec)
 		return(NULL);
 	if (nw_rec <= 0)
 		return(NULL);
-	rp = (RECINDEX *)malloc(sizeof(RECINDEX) + mp->len/(2*nw_rec)*sizeof(char *));
+	nall = 1000;
+	rp = (RECINDEX *)malloc(sizeof(RECINDEX) + nall*sizeof(char *));
 	if (rp == NULL)
 		return(NULL);
 	rp->nw_rec = nw_rec;
@@ -161,6 +167,13 @@ index_records(const MEMLOAD *mp, int nw_rec)
 			++cp;
 		if (cp >= mend)
 			break;
+		if (rp->nrecs >= nall) {
+			nall += nall>>1;	/* get more record space */
+			rp = (RECINDEX *)realloc(rp,
+					sizeof(RECINDEX) + nall*sizeof(char *));
+			if (rp == NULL)
+				return(NULL);
+		}
 		rp->rec[rp->nrecs++] = cp;	/* point to first non-white */
 		n = rp->nw_rec;
 		while (++cp < mend)		/* find end of record */
@@ -237,11 +250,11 @@ output_stream(FILE *fp)
 
 	if (fp == NULL)
 		return(0);
-	fflush(stdout);			/* assumes nothing in input buffer */
-	while ((n = read(fileno(fp), buf, sizeof(buf))) > 0)
+	fflush(stdout);
+	while ((n = fread(buf, 1, sizeof(buf), fp)) > 0)
 		if (write(fileno(stdout), buf, n) != n)
 			return(0);
-	return(n >= 0);
+	return(!ferror(fp));
 }
 
 /* get next word from stream, leaving stream on EOL or start of next word */
@@ -274,6 +287,9 @@ int		ni_columns = 0;			/* number of input columns */
 int		ni_rows = 0;			/* number of input rows */
 int		no_columns = 0;			/* number of output columns */
 int		no_rows = 0;			/* number of output rows */
+int		transpose = 0;			/* transpose rows & cols? */
+int		i_header = 1;			/* input header? */
+int		o_header = 1;			/* output header? */
 
 /* check settings and assign defaults */
 static int
@@ -288,11 +304,17 @@ check_sizes()
 			comp_size = sizeof(double);
 		else if (!strcmp(fmtid, "byte"))
 			comp_size = 1;
-		else {
+		else if (strcmp(fmtid, "ascii")) {
 			fprintf(stderr, "Unsupported format: %s\n", fmtid);
 			return(0);
 		}
 	}
+	if (transpose && (no_rows <= 0) & (no_columns <= 0)) {
+		if (ni_rows > 0) no_columns = ni_rows;
+		if (ni_columns > 0) no_rows = ni_columns;
+	} else if ((no_rows <= 0) & (no_columns > 0) &&
+			!((ni_rows*ni_columns) % no_columns))
+		no_rows = ni_rows*ni_columns/no_columns;
 	if (n_comp <= 0)
 		n_comp = 3;
 	return(1);
@@ -347,9 +369,9 @@ do_transpose(const MEMLOAD *mp)
 			print_record(rp, j*ni_columns + i);
 			putc(tabEOL[j >= no_columns-1], stdout);
 		} else {			/* binary output */
-			fwrite((char *)mp->base +
-					(n_comp*comp_size)*(j*ni_columns + i),
-					n_comp*comp_size, 1, stdout);
+			putbinary((char *)mp->base +
+			    (unsigned long)(n_comp*comp_size)*(j*ni_columns + i),
+					comp_size, n_comp, stdout);
 		}
 	    if (ferror(stdout)) {
 		fprintf(stderr, "Error writing to stdout\n");
@@ -372,8 +394,8 @@ do_resize(FILE *fp)
 	int	columns2go = no_columns;
 	char	word[256];
 						/* sanity checks */
-	if (comp_size)
-		return(output_stream(fp));	/* binary data -- just copy */
+	if (comp_size || (no_columns == ni_columns) & (no_rows == ni_rows))
+		return(output_stream(fp));	/* no-op -- just copy */
 	if (no_columns <= 0) {
 		fprintf(stderr, "Missing -oc specification\n");
 		return(0);
@@ -426,7 +448,7 @@ done:
 static int
 headline(char *s, void *p)
 {
-	static char	fmt[32];
+	static char	fmt[MAXFMTLEN];
 	int		n;
 
 	if (formatval(fmt, s)) {
@@ -460,13 +482,14 @@ headline(char *s, void *p)
 	if (!strncmp(s, "NCOMP=", 6)) {
 		n = atoi(s+6);
 		if ((n_comp > 0) & (n != n_comp)) {
-			fputs("Incorrect number of components", stderr);
+			fputs("Incorrect number of components\n", stderr);
 			return(-1);
 		}
 		n_comp = n;
 		return(0);
 	}
-	fputs(s, stdout);			/* copy header info. */
+	if (o_header)
+		fputs(s, stdout);		/* copy header info. */
 	return(0);
 }
 
@@ -474,31 +497,28 @@ headline(char *s, void *p)
 int
 main(int argc, char *argv[])
 {
-	int	i_header = 1;			/* input header? */
-	int	o_header = 1;			/* output header? */
-	int	transpose = 0;			/* transpose rows & cols? */
-	int	i;
+	int	a;
 
-	for (i = 1; i < argc && argv[i][0] == '-'; i++)
-		switch (argv[i][1]) {
+	for (a = 1; a < argc && argv[a][0] == '-'; a++)
+		switch (argv[a][1]) {
 		case 'i':			/* input */
-			if (argv[i][2] == 'c')	/* columns */
-				ni_columns = atoi(argv[++i]);
-			else if (argv[i][2] == 'r')
-				ni_rows = atoi(argv[++i]);
+			if (argv[a][2] == 'c')	/* columns */
+				ni_columns = atoi(argv[++a]);
+			else if (argv[a][2] == 'r')
+				ni_rows = atoi(argv[++a]);
 			else
 				goto userr;
 			break;
 		case 'o':			/* output */
-			if (argv[i][2] == 'c')	/* columns */
-				no_columns = atoi(argv[++i]);
-			else if (argv[i][2] == 'r')
-				no_rows = atoi(argv[++i]);
+			if (argv[a][2] == 'c')	/* columns */
+				no_columns = atoi(argv[++a]);
+			else if (argv[a][2] == 'r')
+				no_rows = atoi(argv[++a]);
 			else
 				goto userr;
 			break;
 		case 'h':			/* turn off header */
-			switch (argv[i][2]) {
+			switch (argv[a][2]) {
 			case 'i':
 				i_header = 0;
 				break;
@@ -516,7 +536,7 @@ main(int argc, char *argv[])
 			transpose = !transpose;
 			break;
 		case 'f':			/* format */
-			switch (argv[i][2]) {
+			switch (argv[a][2]) {
 			case 'a':		/* ASCII */
 			case 'A':
 				fmtid = "ascii";
@@ -540,11 +560,12 @@ main(int argc, char *argv[])
 			default:
 				goto userr;
 			}
-			if (argv[i][3]) {
-				if (!isdigit(argv[i][3]))
+			if (argv[a][3]) {
+				if (!isdigit(argv[a][3]))
 					goto userr;
-				n_comp = atoi(argv[i]+3);
-			}
+				n_comp = atoi(argv[a]+3);
+			} else
+				n_comp = 1;
 			break;
 		case 'w':			/* warnings on/off */
 			warnings = !warnings;
@@ -552,11 +573,11 @@ main(int argc, char *argv[])
 		default:
 			goto userr;
 		}
-	if (i < argc-1)				/* arg count OK? */
+	if (a < argc-1)				/* arg count OK? */
 		goto userr;
 						/* open input file? */
-	if (i == argc-1 && freopen(argv[i], "r", stdin) == NULL) {
-		fprintf(stderr, "%s: cannot open for reading\n", argv[i]);
+	if (a == argc-1 && freopen(argv[a], "r", stdin) == NULL) {
+		fprintf(stderr, "%s: cannot open for reading\n", argv[a]);
 		return(1);
 	}
 	if (comp_size) {
@@ -564,8 +585,8 @@ main(int argc, char *argv[])
 		SET_FILE_BINARY(stdout);
 	}
 						/* check for no-op */
-	if (!transpose & (i_header == o_header) && (comp_size ||
-			(no_columns == ni_columns) & (no_rows == ni_rows))) {
+	if (!transpose & (i_header == o_header) &&
+			(no_columns == ni_columns) & (no_rows == ni_rows)) {
 		if (warnings)
 			fprintf(stderr, "%s: no-op -- copying input verbatim\n",
 				argv[0]);
@@ -574,7 +595,7 @@ main(int argc, char *argv[])
 		return(0);
 	}
 	if (i_header) {				/* read header */
-		if (getheader(stdin, &headline, NULL) < 0)
+		if (getheader(stdin, headline, NULL) < 0)
 			return(1);
 		if (!check_sizes())
 			return(1);
@@ -585,11 +606,7 @@ main(int argc, char *argv[])
 	} else if (!check_sizes())
 		return(1);
 	if (o_header) {				/* write header */
-		printargs(argc, argv, stdout);
-		if (transpose && (no_rows <= 0) & (no_columns <= 0)) {
-			if (ni_rows > 0) no_columns = ni_rows;
-			if (ni_columns > 0) no_rows = ni_columns;
-		}
+		printargs(a, argv, stdout);
 		if (no_rows > 0)
 			printf("NROWS=%d\n", no_rows);
 		if (no_columns > 0)
@@ -599,11 +616,11 @@ main(int argc, char *argv[])
 		fputc('\n', stdout);		/* finish new header */
 	}
 	if (transpose) {			/* transposing rows & columns? */
-		MEMLOAD	myMem;			/* need to load into memory */
-		if (i == argc-1) {
+		MEMLOAD	myMem;			/* need to map into memory */
+		if (a == argc-1) {
 			if (load_file(&myMem, stdin) <= 0) {
 				fprintf(stderr, "%s: error loading file into memory\n",
-						argv[i]);
+						argv[a]);
 				return(1);
 			}
 		} else if (load_stream(&myMem, stdin) <= 0) {
@@ -613,8 +630,8 @@ main(int argc, char *argv[])
 		}
 		if (!do_transpose(&myMem))
 			return(1);
-		/* free_load(&myMem); */
-	} else if (!do_resize(stdin))		/* just reshaping input */
+		/* free_load(&myMem);	about to exit, so don't bother */
+	} else if (!do_resize(stdin))		/* reshaping input */
 		return(1);
 	return(0);
 userr: