ViewVC Help
View File | Revision Log | Show Annotations | Download File | Root Listing
root/radiance/ray/src/util/rcollate.c
Revision: 2.32
Committed: Fri Nov 8 05:39:05 2019 UTC (4 years, 4 months ago) by greg
Content type: text/plain
Branch: MAIN
Changes since 2.31: +2 -9 lines
Log Message:
Fixing minor issues with new -o code

File Contents

# Content
1 #ifndef lint
2 static const char RCSid[] = "$Id: rcollate.c,v 2.31 2019/11/08 02:10:38 greg Exp $";
3 #endif
4 /*
5 * Utility to re-order records in a binary or ASCII data file (matrix)
6 */
7
8 #include <stdlib.h>
9 #include <string.h>
10 #include <ctype.h>
11 #include "platform.h"
12 #include "rtio.h"
13 #include "resolu.h"
14 #if defined(_WIN32) || defined(_WIN64)
15 #undef ftello
16 #define ftello ftell
17 #undef ssize_t
18 #define ssize_t size_t
19 #else
20 #include <sys/mman.h>
21 #endif
22
23 #define MAXLEVELS 16 /* max RxC.. block pairs */
24
25 typedef struct {
26 void *mapped; /* memory-mapped pointer */
27 void *base; /* pointer to base memory */
28 size_t len; /* allocated memory length */
29 } MEMLOAD; /* file loaded/mapped into memory */
30
31 typedef struct {
32 int nw_rec; /* number of words per record */
33 long nrecs; /* number of records we found */
34 char *rec[1]; /* record array (extends struct) */
35 } RECINDEX;
36
37 int warnings = 1; /* report warnings? */
38
39 /* free loaded file */
40 static void
41 free_load(MEMLOAD *mp)
42 {
43 if (mp == NULL || (mp->base == NULL) | (mp->len <= 0))
44 return;
45 #ifdef MAP_FILE
46 if (mp->mapped)
47 munmap(mp->mapped, mp->len);
48 else
49 #endif
50 free(mp->base);
51 mp->mapped = NULL;
52 mp->base = NULL;
53 mp->len = 0;
54 }
55
56 /* load memory from an input stream, starting from current position */
57 static int
58 load_stream(MEMLOAD *mp, FILE *fp)
59 {
60 size_t alloced = 0;
61 char buf[8192];
62 size_t nr;
63
64 if (mp == NULL)
65 return(-1);
66 mp->mapped = NULL;
67 mp->base = NULL;
68 mp->len = 0;
69 if (fp == NULL)
70 return(-1);
71 while ((nr = fread(buf, 1, sizeof(buf), fp)) > 0) {
72 if (!alloced)
73 mp->base = malloc(alloced = nr);
74 else if (mp->len+nr > alloced)
75 mp->base = realloc(mp->base,
76 alloced = alloced*(2+(nr==sizeof(buf)))/2+nr);
77 if (mp->base == NULL)
78 return(-1);
79 memcpy((char *)mp->base + mp->len, buf, nr);
80 mp->len += nr;
81 }
82 if (ferror(fp)) {
83 free_load(mp);
84 return(-1);
85 }
86 if (alloced > mp->len*5/4) /* don't waste too much space */
87 mp->base = realloc(mp->base, mp->len);
88 return(mp->len > 0);
89 }
90
91 /* load a file into memory */
92 static int
93 load_file(MEMLOAD *mp, FILE *fp)
94 {
95 int fd;
96 off_t skip, flen, fpos;
97
98 #if defined(_WIN32) || defined(_WIN64)
99 /* too difficult to fix this */
100 return load_stream(mp, fp);
101 #endif
102 if (mp == NULL)
103 return(-1);
104 mp->mapped = NULL;
105 mp->base = NULL;
106 mp->len = 0;
107 if (fp == NULL)
108 return(-1);
109 fd = fileno(fp);
110 skip = ftello(fp);
111 flen = lseek(fd, 0, SEEK_END);
112 if (flen <= skip)
113 return((int)(flen - skip));
114 mp->len = (size_t)(flen - skip);
115 #ifdef MAP_FILE
116 if (mp->len > 1L<<20) { /* map file if > 1 MByte */
117 mp->mapped = mmap(NULL, flen, PROT_READ, MAP_PRIVATE, fd, 0);
118 if (mp->mapped != MAP_FAILED) {
119 mp->base = (char *)mp->mapped + skip;
120 return(1); /* mmap() success */
121 }
122 mp->mapped = NULL; /* else fall back to reading it in... */
123 }
124 #endif
125 if (lseek(fd, skip, SEEK_SET) != skip ||
126 (mp->base = malloc(mp->len)) == NULL) {
127 mp->len = 0;
128 return(-1);
129 }
130 fpos = skip;
131 while (fpos < flen) { /* read() fails if n > 2 GBytes */
132 ssize_t nread = read(fd, (char *)mp->base+(fpos-skip),
133 (flen-fpos < 1L<<24) ? flen-fpos : 1L<<24);
134 if (nread <= 0) {
135 free_load(mp);
136 return(-1);
137 }
138 fpos += nread;
139 }
140 return(1);
141 }
142
143 /* free a record index */
144 #define free_records(rp) free(rp)
145
146 /* compute record index */
147 static RECINDEX *
148 index_records(const MEMLOAD *mp, int nw_rec)
149 {
150 int nall = 0;
151 RECINDEX *rp;
152 char *cp, *mend;
153 int n;
154
155 if (mp == NULL || (mp->base == NULL) | (mp->len <= 0))
156 return(NULL);
157 if (nw_rec <= 0)
158 return(NULL);
159 nall = 1000;
160 rp = (RECINDEX *)malloc(sizeof(RECINDEX) + nall*sizeof(char *));
161 if (rp == NULL)
162 return(NULL);
163 rp->nw_rec = nw_rec;
164 rp->nrecs = 0;
165 cp = (char *)mp->base;
166 mend = cp + mp->len;
167 for ( ; ; ) { /* whitespace-separated words */
168 while (cp < mend && !*cp | isspace(*cp))
169 ++cp;
170 if (cp >= mend)
171 break;
172 if (rp->nrecs >= nall) {
173 nall += nall>>1; /* get more record space */
174 rp = (RECINDEX *)realloc(rp,
175 sizeof(RECINDEX) + nall*sizeof(char *));
176 if (rp == NULL)
177 return(NULL);
178 }
179 rp->rec[rp->nrecs++] = cp; /* point to first non-white */
180 n = rp->nw_rec;
181 while (++cp < mend) /* find end of record */
182 if (!*cp | isspace(*cp)) {
183 if (--n <= 0)
184 break; /* got requisite # words */
185 do { /* else find next word */
186 if (*cp == '\n') {
187 fprintf(stderr,
188 "Unexpected EOL in record!\n");
189 free_records(rp);
190 return(NULL);
191 }
192 if (++cp >= mend)
193 break;
194 } while (!*cp | isspace(*cp));
195 }
196 }
197 rp->rec[rp->nrecs] = mend; /* reallocate to save space */
198 rp = (RECINDEX *)realloc(rp,
199 sizeof(RECINDEX) + rp->nrecs*sizeof(char *));
200 return(rp);
201 }
202
203 /* count number of columns based on first EOL */
204 static int
205 count_columns(const RECINDEX *rp)
206 {
207 char *cp = rp->rec[0];
208 char *mend = rp->rec[rp->nrecs];
209 int i;
210
211 while (*cp != '\n')
212 if (++cp >= mend)
213 return(0);
214 for (i = 0; i < rp->nrecs; i++)
215 if (rp->rec[i] >= cp)
216 break;
217 return(i);
218 }
219
220 /* copy nth record from index to stdout */
221 static int
222 print_record(const RECINDEX *rp, long n)
223 {
224 int words2go = rp->nw_rec;
225 char *scp;
226
227 if ((n < 0) | (n >= rp->nrecs))
228 return(0);
229 scp = rp->rec[n];
230 do {
231 putc(*scp++, stdout);
232 if (!*scp | isspace(*scp)) {
233 if (--words2go <= 0)
234 break;
235 putc(' ', stdout); /* single space btwn. words */
236 do
237 if (++scp >= rp->rec[n+1])
238 break;
239 while (!*scp | isspace(*scp));
240 }
241 } while (scp < rp->rec[n+1]);
242 /* caller adds record sep. */
243 return(1);
244 }
245
246 /* copy a stream to stdout */
247 static int
248 output_stream(FILE *fp)
249 {
250 char buf[8192];
251 ssize_t n;
252
253 if (fp == NULL)
254 return(0);
255 fflush(stdout);
256 while ((n = fread(buf, 1, sizeof(buf), fp)) > 0)
257 if (write(fileno(stdout), buf, n) != n)
258 return(0);
259 return(!ferror(fp));
260 }
261
262 /* get next word from stream, leaving stream on EOL or start of next word */
263 static char *
264 fget_word(char buf[256], FILE *fp)
265 {
266 int c;
267 char *cp;
268 /* skip nul's and white space */
269 while (!(c = getc(fp)) || isspace(c))
270 ;
271 if (c == EOF)
272 return(NULL);
273 cp = buf;
274 do
275 *cp++ = c;
276 while ((c = getc(fp)) != EOF && !isspace(c) && cp < buf+255);
277 *cp = '\0';
278 while (isspace(c) & (c != '\n'))
279 c = getc(fp);
280 if (c != EOF)
281 ungetc(c, fp);
282 return(buf);
283 }
284
285 char *fmtid = NULL; /* format id */
286 int comp_size = 0; /* binary bytes/channel */
287 int n_comp = 0; /* components/record */
288 int ni_columns = 0; /* number of input columns */
289 int ni_rows = 0; /* number of input rows */
290 int no_columns = 0; /* number of output columns */
291 int no_rows = 0; /* number of output rows */
292 int transpose = 0; /* transpose rows & cols? */
293 int i_header = 1; /* input header? */
294 int o_header = 1; /* output header? */
295 int outArray[MAXLEVELS][2]; /* output block nesting */
296 int outLevels = 0; /* number of blocking levels */
297
298 /* parse RxCx... string */
299 static int
300 get_array(const char *spec, int blklvl[][2], int nlvls)
301 {
302 int n;
303
304 if (nlvls <= 0) {
305 fputs("Too many block levels!\n", stderr);
306 exit(1);
307 }
308 if (sscanf(spec, "%dx%d", &blklvl[0][0], &blklvl[0][1]) != 2) {
309 fputs("Bad block specification!\n", stderr);
310 exit(1);
311 }
312 while (isdigit(*spec))
313 spec++;
314 spec++; /* 'x' */
315 while (isdigit(*spec))
316 spec++;
317 if ((*spec != 'x') & (*spec != 'X')) {
318 if (*spec) {
319 fputs("Blocks must be separated by 'x' or 'X'\n", stderr);
320 exit(1);
321 }
322 return(1);
323 }
324 spec++;
325 n = get_array(spec, blklvl+1, nlvls-1);
326 if (!n)
327 return(0);
328 blklvl[0][0] *= blklvl[1][0];
329 blklvl[0][1] *= blklvl[1][1];
330 return(n+1);
331 }
332
333 /* check settings and assign defaults */
334 static int
335 check_sizes()
336 {
337 if (fmtid == NULL) {
338 fmtid = "ascii";
339 } else if (!comp_size) {
340 if (!strcmp(fmtid, "float"))
341 comp_size = sizeof(float);
342 else if (!strcmp(fmtid, "double"))
343 comp_size = sizeof(double);
344 else if (!strcmp(fmtid, "byte"))
345 comp_size = 1;
346 else if (strcmp(fmtid, "ascii")) {
347 fprintf(stderr, "Unsupported format: %s\n", fmtid);
348 return(0);
349 }
350 }
351 if (transpose && (no_rows <= 0) & (no_columns <= 0)) {
352 if (ni_rows > 0) no_columns = ni_rows;
353 if (ni_columns > 0) no_rows = ni_columns;
354 } else if ((no_rows <= 0) & (no_columns > 0) &&
355 !((ni_rows*ni_columns) % no_columns))
356 no_rows = ni_rows*ni_columns/no_columns;
357 if (n_comp <= 0)
358 n_comp = 3;
359 return(1);
360 }
361
362 /* call to compute block input position */
363 static long
364 get_block_pos(int r, int c, int blklvl[][2], int nlvls)
365 {
366 long n = 0;
367
368 while (nlvls > 1) {
369 int sr = r/blklvl[1][0];
370 int sc = c/blklvl[1][1];
371 r -= sr*blklvl[1][0];
372 c -= sc*blklvl[1][1];
373 n += sr*blklvl[1][0]*blklvl[0][1] + sc*blklvl[1][0]*blklvl[1][1];
374 blklvl++;
375 nlvls--;
376 }
377 n += r*blklvl[0][1] + c;
378 return(n);
379 }
380
381 /* return input offset based on array ordering and transpose option */
382 static long
383 get_input_pos(int r, int c)
384 {
385 long n;
386
387 if (outLevels > 1) { /* block reordering */
388 n = get_block_pos(r, c, outArray, outLevels);
389 if (transpose) {
390 r = n/no_columns;
391 c = n - r*no_columns;
392 n = (long)r*ni_columns + c;
393 }
394 } else if (transpose) /* transpose only */
395 n = (long)c*ni_columns + r;
396 else /* XXX should never happen! */
397 n = (long)r*no_columns + c;
398 return(n);
399 }
400
401 /* output reordered ASCII or binary data from memory */
402 static int
403 do_reorder(const MEMLOAD *mp)
404 {
405 static const char tabEOL[2] = {'\t','\n'};
406 RECINDEX *rp = NULL;
407 long nrecords;
408 int i, j;
409 /* propogate sizes */
410 if (ni_rows <= 0)
411 ni_rows = no_columns;
412 if (ni_columns <= 0)
413 ni_columns = no_rows;
414 /* get # records (& index) */
415 if (!comp_size) {
416 if ((rp = index_records(mp, n_comp)) == NULL)
417 return(0);
418 if (ni_columns <= 0)
419 ni_columns = count_columns(rp);
420 nrecords = rp->nrecs;
421 } else if ((ni_rows > 0) & (ni_columns > 0)) {
422 nrecords = ni_rows*ni_columns;
423 if (nrecords > mp->len/(n_comp*comp_size)) {
424 fprintf(stderr,
425 "Input too small for specified size and type\n");
426 return(0);
427 }
428 } else
429 nrecords = mp->len/(n_comp*comp_size);
430 /* check sizes */
431 if ((ni_rows <= 0) & (ni_columns > 0))
432 ni_rows = nrecords/ni_columns;
433 if ((ni_columns <= 0) & (ni_rows > 0))
434 ni_columns = nrecords/ni_rows;
435 if (nrecords != ni_rows*ni_columns)
436 goto badspec;
437 if (transpose) {
438 if (no_columns <= 0)
439 no_columns = ni_rows;
440 if (no_rows <= 0)
441 no_rows = ni_columns;
442 if ((no_rows != ni_columns) | (no_columns != ni_rows))
443 goto badspec;
444 }
445 /* reorder records */
446 for (i = 0; i < no_rows; i++) {
447 for (j = 0; j < no_columns; j++) {
448 long n = get_input_pos(i, j);
449 if (rp != NULL) { /* ASCII output */
450 print_record(rp, n);
451 putc(tabEOL[j >= no_columns-1], stdout);
452 } else { /* binary output */
453 putbinary((char *)mp->base + (n_comp*comp_size)*n,
454 comp_size, n_comp, stdout);
455 }
456 }
457 if (ferror(stdout)) {
458 fprintf(stderr, "Error writing to stdout\n");
459 return(0);
460 }
461 }
462 if (rp != NULL)
463 free_records(rp);
464 return(1);
465 badspec:
466 fprintf(stderr, "Bad dimension(s)\n");
467 return(0);
468 }
469
470 /* resize ASCII stream input by ignoring EOLs between records */
471 static int
472 do_resize(FILE *fp)
473 {
474 long records2go = ni_rows*ni_columns;
475 int columns2go = no_columns;
476 char word[256];
477 /* sanity checks */
478 if (comp_size || (no_columns == ni_columns) & (no_rows == ni_rows))
479 return(output_stream(fp)); /* no-op -- just copy */
480 if (no_columns <= 0) {
481 fprintf(stderr, "Missing -oc specification\n");
482 return(0);
483 }
484 if ((records2go <= 0) & (no_rows > 0))
485 records2go = no_rows*no_columns;
486 else if (no_rows*no_columns != records2go) {
487 fprintf(stderr,
488 "Input and output data sizes disagree (%dx%d != %dx%d)\n",
489 ni_rows, ni_columns, no_rows, no_columns);
490 return(0);
491 }
492 do { /* reshape records */
493 int n;
494
495 for (n = n_comp; n--; ) {
496 if (fget_word(word, fp) == NULL) {
497 if (records2go > 0 || n < n_comp-1)
498 break;
499 goto done; /* normal EOD */
500 }
501 fputs(word, stdout);
502 if (n) { /* mid-record? */
503 int c = getc(fp);
504 if ((c == '\n') | (c == EOF))
505 break;
506 ungetc(c, fp);
507 putc(' ', stdout);
508 }
509 }
510 if (n >= 0) {
511 fprintf(stderr, "Incomplete record / unexpected EOF\n");
512 return(0);
513 }
514 if (--columns2go <= 0) { /* time to end output row? */
515 putc('\n', stdout);
516 columns2go = no_columns;
517 } else /* else separate records */
518 putc('\t', stdout);
519 } while (--records2go); /* expected EOD? */
520 done:
521 if (warnings && columns2go != no_columns)
522 fprintf(stderr, "Warning -- incomplete final row\n");
523 if (warnings && fget_word(word, fp) != NULL)
524 fprintf(stderr, "Warning -- characters beyond expected EOD\n");
525 return(1);
526 }
527
528 /* process a header line and copy to stdout */
529 static int
530 headline(char *s, void *p)
531 {
532 static char fmt[MAXFMTLEN];
533 int n;
534
535 if (formatval(fmt, s)) {
536 if (fmtid == NULL) {
537 fmtid = fmt;
538 return(0);
539 }
540 if (!strcmp(fmt, fmtid))
541 return(0);
542 fprintf(stderr, "Input format '%s' != '%s'\n", fmt, fmtid);
543 return(-1);
544 }
545 if (!strncmp(s, "NROWS=", 6)) {
546 n = atoi(s+6);
547 if ((ni_rows > 0) & (n != ni_rows)) {
548 fputs("Incorrect input row count\n", stderr);
549 return(-1);
550 }
551 ni_rows = n;
552 return(0);
553 }
554 if (!strncmp(s, "NCOLS=", 6)) {
555 n = atoi(s+6);
556 if ((ni_columns > 0) & (n != ni_columns)) {
557 fputs("Incorrect input column count\n", stderr);
558 return(-1);
559 }
560 ni_columns = n;
561 return(0);
562 }
563 if (!strncmp(s, "NCOMP=", 6)) {
564 n = atoi(s+6);
565 if ((n_comp > 0) & (n != n_comp)) {
566 fputs("Incorrect number of components\n", stderr);
567 return(-1);
568 }
569 n_comp = n;
570 return(0);
571 }
572 if (o_header)
573 fputs(s, stdout); /* copy header info. */
574 return(0);
575 }
576
577 /* main routine for converting rows/columns in data file */
578 int
579 main(int argc, char *argv[])
580 {
581 int a;
582
583 for (a = 1; a < argc && argv[a][0] == '-'; a++)
584 switch (argv[a][1]) {
585 case 'i': /* input */
586 if (argv[a][2] == 'c') /* columns */
587 ni_columns = atoi(argv[++a]);
588 else if (argv[a][2] == 'r')
589 ni_rows = atoi(argv[++a]);
590 else
591 goto userr;
592 break;
593 case 'o': /* output */
594 if (argv[a][2] == 'c') /* columns */
595 no_columns = atoi(argv[++a]);
596 else if (argv[a][2] == 'r')
597 no_rows = atoi(argv[++a]);
598 else if (argv[a][2] ||
599 !(outLevels=get_array(argv[++a], outArray, MAXLEVELS)))
600 goto userr;
601 break;
602 case 'h': /* turn off header */
603 switch (argv[a][2]) {
604 case 'i':
605 i_header = 0;
606 break;
607 case 'o':
608 o_header = 0;
609 break;
610 case '\0':
611 i_header = o_header = 0;
612 break;
613 default:
614 goto userr;
615 }
616 break;
617 case 't': /* transpose on/off */
618 transpose = !transpose;
619 break;
620 case 'f': /* format */
621 switch (argv[a][2]) {
622 case 'a': /* ASCII */
623 case 'A':
624 fmtid = "ascii";
625 comp_size = 0;
626 break;
627 case 'f': /* float */
628 case 'F':
629 fmtid = "float";
630 comp_size = sizeof(float);
631 break;
632 case 'd': /* double */
633 case 'D':
634 fmtid = "double";
635 comp_size = sizeof(double);
636 break;
637 case 'b': /* binary (bytes) */
638 case 'B':
639 fmtid = "byte";
640 comp_size = 1;
641 break;
642 default:
643 goto userr;
644 }
645 if (argv[a][3]) {
646 if (!isdigit(argv[a][3]))
647 goto userr;
648 n_comp = atoi(argv[a]+3);
649 } else
650 n_comp = 1;
651 break;
652 case 'w': /* warnings on/off */
653 warnings = !warnings;
654 break;
655 default:
656 goto userr;
657 }
658 if (a < argc-1) /* arg count OK? */
659 goto userr;
660 if (outLevels) { /* should check consistency? */
661 no_rows = outArray[0][0];
662 no_columns = outArray[0][1];
663 }
664 /* open input file? */
665 if (a == argc-1 && freopen(argv[a], "r", stdin) == NULL) {
666 fprintf(stderr, "%s: cannot open for reading\n", argv[a]);
667 return(1);
668 }
669 if (comp_size) {
670 SET_FILE_BINARY(stdin);
671 SET_FILE_BINARY(stdout);
672 }
673 /* check for no-op */
674 if (!transpose & (outLevels <= 1) & (i_header == o_header) &&
675 (no_columns == ni_columns) & (no_rows == ni_rows)) {
676 if (warnings)
677 fprintf(stderr, "%s: no-op -- copying input verbatim\n",
678 argv[0]);
679 if (!output_stream(stdin))
680 return(1);
681 return(0);
682 }
683 if (i_header) { /* read header */
684 if (getheader(stdin, headline, NULL) < 0)
685 return(1);
686 if (!check_sizes())
687 return(1);
688 if (comp_size) { /* a little late... */
689 SET_FILE_BINARY(stdin);
690 SET_FILE_BINARY(stdout);
691 }
692 } else if (!check_sizes())
693 return(1);
694 if (o_header) { /* write/add to header */
695 if (!i_header)
696 newheader("RADIANCE", stdout);
697 printargs(a, argv, stdout);
698 if (no_rows > 0)
699 printf("NROWS=%d\n", no_rows);
700 if (no_columns > 0)
701 printf("NCOLS=%d\n", no_columns);
702 printf("NCOMP=%d\n", n_comp);
703 fputformat(fmtid, stdout);
704 fputc('\n', stdout); /* finish new header */
705 }
706 if (transpose | (outLevels > 1)) { /* moving stuff around? */
707 MEMLOAD myMem; /* need to map into memory */
708 if (a == argc-1) {
709 if (load_file(&myMem, stdin) <= 0) {
710 fprintf(stderr, "%s: error loading file into memory\n",
711 argv[a]);
712 return(1);
713 }
714 } else if (load_stream(&myMem, stdin) <= 0) {
715 fprintf(stderr, "%s: error loading stdin into memory\n",
716 argv[0]);
717 return(1);
718 }
719 if (!do_reorder(&myMem))
720 return(1);
721 /* free_load(&myMem); about to exit, so don't bother */
722 } else if (!do_resize(stdin)) /* reshaping input */
723 return(1);
724 return(0);
725 userr:
726 fprintf(stderr,
727 "Usage: %s [-h[io]][-w][-f[afdb][N]][-t][-ic in_col][-ir in_row][-oc out_col][-or out_row][-o RxC[xR1xC1..]] [input.dat]\n",
728 argv[0]);
729 return(1);
730 }