ViewVC Help
View File | Revision Log | Show Annotations | Download File | Root Listing
root/radiance/ray/src/util/rcollate.c
Revision: 2.35
Committed: Sat Dec 28 18:05:14 2019 UTC (4 years, 3 months ago) by greg
Content type: text/plain
Branch: MAIN
CVS Tags: rad5R3
Changes since 2.34: +1 -2 lines
Log Message:
Removed redundant include files

File Contents

# Content
1 #ifndef lint
2 static const char RCSid[] = "$Id: rcollate.c,v 2.34 2019/11/08 22:12:33 greg Exp $";
3 #endif
4 /*
5 * Utility to re-order records in a binary or ASCII data file (matrix)
6 */
7
8 #include <stdlib.h>
9 #include <ctype.h>
10 #include "platform.h"
11 #include "rtio.h"
12 #include "resolu.h"
13 #if defined(_WIN32) || defined(_WIN64)
14 #undef ftello
15 #define ftello ftell
16 #undef ssize_t
17 #define ssize_t size_t
18 #else
19 #include <sys/mman.h>
20 #endif
21
22 #define MAXLEVELS 16 /* max RxC.. block pairs */
23
24 typedef struct {
25 void *mapped; /* memory-mapped pointer */
26 void *base; /* pointer to base memory */
27 size_t len; /* allocated memory length */
28 } MEMLOAD; /* file loaded/mapped into memory */
29
30 typedef struct {
31 int nw_rec; /* number of words per record */
32 long nrecs; /* number of records we found */
33 char *rec[1]; /* record array (extends struct) */
34 } RECINDEX;
35
36 int warnings = 1; /* report warnings? */
37
38 /* free loaded file */
39 static void
40 free_load(MEMLOAD *mp)
41 {
42 if (mp == NULL || (mp->base == NULL) | (mp->len <= 0))
43 return;
44 #ifdef MAP_FILE
45 if (mp->mapped)
46 munmap(mp->mapped, mp->len);
47 else
48 #endif
49 free(mp->base);
50 mp->mapped = NULL;
51 mp->base = NULL;
52 mp->len = 0;
53 }
54
55 /* load memory from an input stream, starting from current position */
56 static int
57 load_stream(MEMLOAD *mp, FILE *fp)
58 {
59 size_t alloced = 0;
60 char buf[8192];
61 size_t nr;
62
63 if (mp == NULL)
64 return(-1);
65 mp->mapped = NULL;
66 mp->base = NULL;
67 mp->len = 0;
68 if (fp == NULL)
69 return(-1);
70 while ((nr = fread(buf, 1, sizeof(buf), fp)) > 0) {
71 if (!alloced)
72 mp->base = malloc(alloced = nr);
73 else if (mp->len+nr > alloced)
74 mp->base = realloc(mp->base,
75 alloced = alloced*(2+(nr==sizeof(buf)))/2+nr);
76 if (mp->base == NULL)
77 return(-1);
78 memcpy((char *)mp->base + mp->len, buf, nr);
79 mp->len += nr;
80 }
81 if (ferror(fp)) {
82 free_load(mp);
83 return(-1);
84 }
85 if (alloced > mp->len*5/4) /* don't waste too much space */
86 mp->base = realloc(mp->base, mp->len);
87 return(mp->len > 0);
88 }
89
90 /* load a file into memory */
91 static int
92 load_file(MEMLOAD *mp, FILE *fp)
93 {
94 int fd;
95 off_t skip, flen, fpos;
96
97 #if defined(_WIN32) || defined(_WIN64)
98 /* too difficult to fix this */
99 return load_stream(mp, fp);
100 #endif
101 if (mp == NULL)
102 return(-1);
103 mp->mapped = NULL;
104 mp->base = NULL;
105 mp->len = 0;
106 if (fp == NULL)
107 return(-1);
108 fd = fileno(fp);
109 skip = ftello(fp);
110 flen = lseek(fd, 0, SEEK_END);
111 if (flen <= skip)
112 return((int)(flen - skip));
113 mp->len = (size_t)(flen - skip);
114 #ifdef MAP_FILE
115 if (mp->len > 1L<<20) { /* map file if > 1 MByte */
116 mp->mapped = mmap(NULL, flen, PROT_READ, MAP_PRIVATE, fd, 0);
117 if (mp->mapped != MAP_FAILED) {
118 mp->base = (char *)mp->mapped + skip;
119 return(1); /* mmap() success */
120 }
121 mp->mapped = NULL; /* else fall back to reading it in... */
122 }
123 #endif
124 if (lseek(fd, skip, SEEK_SET) != skip ||
125 (mp->base = malloc(mp->len)) == NULL) {
126 mp->len = 0;
127 return(-1);
128 }
129 fpos = skip;
130 while (fpos < flen) { /* read() fails if n > 2 GBytes */
131 ssize_t nread = read(fd, (char *)mp->base+(fpos-skip),
132 (flen-fpos < 1L<<24) ? flen-fpos : 1L<<24);
133 if (nread <= 0) {
134 free_load(mp);
135 return(-1);
136 }
137 fpos += nread;
138 }
139 return(1);
140 }
141
142 /* free a record index */
143 #define free_records(rp) free(rp)
144
145 /* compute record index */
146 static RECINDEX *
147 index_records(const MEMLOAD *mp, int nw_rec)
148 {
149 int nall = 0;
150 RECINDEX *rp;
151 char *cp, *mend;
152 int n;
153
154 if (mp == NULL || (mp->base == NULL) | (mp->len <= 0))
155 return(NULL);
156 if (nw_rec <= 0)
157 return(NULL);
158 nall = 1000;
159 rp = (RECINDEX *)malloc(sizeof(RECINDEX) + nall*sizeof(char *));
160 if (rp == NULL)
161 return(NULL);
162 rp->nw_rec = nw_rec;
163 rp->nrecs = 0;
164 cp = (char *)mp->base;
165 mend = cp + mp->len;
166 for ( ; ; ) { /* whitespace-separated words */
167 while (cp < mend && !*cp | isspace(*cp))
168 ++cp;
169 if (cp >= mend)
170 break;
171 if (rp->nrecs >= nall) {
172 nall += nall>>1; /* get more record space */
173 rp = (RECINDEX *)realloc(rp,
174 sizeof(RECINDEX) + nall*sizeof(char *));
175 if (rp == NULL)
176 return(NULL);
177 }
178 rp->rec[rp->nrecs++] = cp; /* point to first non-white */
179 n = rp->nw_rec;
180 while (++cp < mend) /* find end of record */
181 if (!*cp | isspace(*cp)) {
182 if (--n <= 0)
183 break; /* got requisite # words */
184 do { /* else find next word */
185 if (*cp == '\n') {
186 fprintf(stderr,
187 "Unexpected EOL in record!\n");
188 free_records(rp);
189 return(NULL);
190 }
191 if (++cp >= mend)
192 break;
193 } while (!*cp | isspace(*cp));
194 }
195 }
196 rp->rec[rp->nrecs] = mend; /* reallocate to save space */
197 rp = (RECINDEX *)realloc(rp,
198 sizeof(RECINDEX) + rp->nrecs*sizeof(char *));
199 return(rp);
200 }
201
202 /* count number of columns based on first EOL */
203 static int
204 count_columns(const RECINDEX *rp)
205 {
206 char *cp = rp->rec[0];
207 char *mend = rp->rec[rp->nrecs];
208 int i;
209
210 while (*cp != '\n')
211 if (++cp >= mend)
212 return(0);
213 for (i = 0; i < rp->nrecs; i++)
214 if (rp->rec[i] >= cp)
215 break;
216 return(i);
217 }
218
219 /* copy nth record from index to stdout */
220 static int
221 print_record(const RECINDEX *rp, long n)
222 {
223 int words2go = rp->nw_rec;
224 char *scp;
225
226 if ((n < 0) | (n >= rp->nrecs))
227 return(0);
228 scp = rp->rec[n];
229 do {
230 putc(*scp++, stdout);
231 if (!*scp | isspace(*scp)) {
232 if (--words2go <= 0)
233 break;
234 putc(' ', stdout); /* single space btwn. words */
235 do
236 if (++scp >= rp->rec[n+1])
237 break;
238 while (!*scp | isspace(*scp));
239 }
240 } while (scp < rp->rec[n+1]);
241 /* caller adds record sep. */
242 return(1);
243 }
244
245 /* copy a stream to stdout */
246 static int
247 output_stream(FILE *fp)
248 {
249 char buf[8192];
250 ssize_t n;
251
252 if (fp == NULL)
253 return(0);
254 fflush(stdout);
255 while ((n = fread(buf, 1, sizeof(buf), fp)) > 0)
256 if (write(fileno(stdout), buf, n) != n)
257 return(0);
258 return(!ferror(fp));
259 }
260
261 /* get next word from stream, leaving stream on EOL or start of next word */
262 static char *
263 fget_word(char buf[256], FILE *fp)
264 {
265 int c;
266 char *cp;
267 /* skip nul's and white space */
268 while (!(c = getc(fp)) || isspace(c))
269 ;
270 if (c == EOF)
271 return(NULL);
272 cp = buf;
273 do
274 *cp++ = c;
275 while ((c = getc(fp)) != EOF && !isspace(c) && cp < buf+255);
276 *cp = '\0';
277 while (isspace(c) & (c != '\n'))
278 c = getc(fp);
279 if (c != EOF)
280 ungetc(c, fp);
281 return(buf);
282 }
283
284 char *fmtid = NULL; /* format id */
285 int comp_size = 0; /* binary bytes/channel */
286 int n_comp = 0; /* components/record */
287 int ni_columns = 0; /* number of input columns */
288 int ni_rows = 0; /* number of input rows */
289 int no_columns = 0; /* number of output columns */
290 int no_rows = 0; /* number of output rows */
291 int transpose = 0; /* transpose rows & cols? */
292 int i_header = 1; /* input header? */
293 int o_header = 1; /* output header? */
294 int outArray[MAXLEVELS][2]; /* output block nesting */
295 int outLevels = 0; /* number of blocking levels */
296
297 /* parse RxCx... string */
298 static int
299 get_array(const char *spec, int blklvl[][2], int nlvls)
300 {
301 int n;
302
303 if (nlvls <= 0) {
304 fputs("Too many block levels!\n", stderr);
305 exit(1);
306 }
307 if (sscanf(spec, "%dx%d", &blklvl[0][0], &blklvl[0][1]) != 2) {
308 fputs("Bad block specification!\n", stderr);
309 exit(1);
310 }
311 while (isdigit(*spec))
312 spec++;
313 spec++; /* 'x' */
314 while (isdigit(*spec))
315 spec++;
316 if ((*spec != 'x') & (*spec != 'X')) {
317 if (*spec) {
318 fputs("Blocks must be separated by 'x' or 'X'\n", stderr);
319 exit(1);
320 }
321 return(1);
322 }
323 spec++;
324 n = get_array(spec, blklvl+1, nlvls-1);
325 if (!n)
326 return(0);
327 blklvl[0][0] *= blklvl[1][0];
328 blklvl[0][1] *= blklvl[1][1];
329 return(n+1);
330 }
331
332 /* check settings and assign defaults */
333 static int
334 check_sizes()
335 {
336 if (fmtid == NULL) {
337 fmtid = "ascii";
338 } else if (!comp_size) {
339 if (!strcmp(fmtid, "float"))
340 comp_size = sizeof(float);
341 else if (!strcmp(fmtid, "double"))
342 comp_size = sizeof(double);
343 else if (!strcmp(fmtid, "byte"))
344 comp_size = 1;
345 else if (strcmp(fmtid, "ascii")) {
346 fprintf(stderr, "Unsupported format: %s\n", fmtid);
347 return(0);
348 }
349 }
350 if (transpose && (no_rows <= 0) & (no_columns <= 0)) {
351 if (ni_rows > 0) no_columns = ni_rows;
352 if (ni_columns > 0) no_rows = ni_columns;
353 } else if ((no_rows <= 0) & (no_columns > 0) &&
354 !((ni_rows*ni_columns) % no_columns))
355 no_rows = ni_rows*ni_columns/no_columns;
356 if (n_comp <= 0)
357 n_comp = 3;
358 return(1);
359 }
360
361 /* call to compute block input position */
362 static long
363 get_block_pos(int r, int c, int blklvl[][2], int nlvls)
364 {
365 long n = 0;
366
367 while (nlvls > 1) {
368 int sr = r/blklvl[1][0];
369 int sc = c/blklvl[1][1];
370 r -= sr*blklvl[1][0];
371 c -= sc*blklvl[1][1];
372 n += sr*blklvl[1][0]*blklvl[0][1] + sc*blklvl[1][0]*blklvl[1][1];
373 blklvl++;
374 nlvls--;
375 }
376 n += r*blklvl[0][1] + c;
377 return(n);
378 }
379
380 /* return input offset based on array ordering and transpose option */
381 static long
382 get_input_pos(int r, int c)
383 {
384 long n;
385
386 if (outLevels > 1) { /* block reordering */
387 n = get_block_pos(r, c, outArray, outLevels);
388 if (transpose) {
389 r = n/ni_rows;
390 c = n - r*ni_rows;
391 n = (long)c*ni_columns + r;
392 }
393 } else if (transpose) /* transpose only */
394 n = (long)c*ni_columns + r;
395 else /* XXX should never happen! */
396 n = (long)r*no_columns + c;
397 return(n);
398 }
399
400 /* output reordered ASCII or binary data from memory */
401 static int
402 do_reorder(const MEMLOAD *mp)
403 {
404 static const char tabEOL[2] = {'\t','\n'};
405 RECINDEX *rp = NULL;
406 long nrecords;
407 int i, j;
408 /* propogate sizes */
409 if (ni_rows <= 0)
410 ni_rows = no_columns;
411 if (ni_columns <= 0)
412 ni_columns = no_rows;
413 /* get # records (& index) */
414 if (!comp_size) {
415 if ((rp = index_records(mp, n_comp)) == NULL)
416 return(0);
417 if (ni_columns <= 0)
418 ni_columns = count_columns(rp);
419 nrecords = rp->nrecs;
420 } else if ((ni_rows > 0) & (ni_columns > 0)) {
421 nrecords = ni_rows*ni_columns;
422 if (nrecords > mp->len/(n_comp*comp_size)) {
423 fputs("Input too small for specified size and type\n",
424 stderr);
425 return(0);
426 }
427 } else
428 nrecords = mp->len/(n_comp*comp_size);
429 /* check sizes */
430 if ((ni_rows <= 0) & (ni_columns > 0))
431 ni_rows = nrecords/ni_columns;
432 if ((ni_columns <= 0) & (ni_rows > 0))
433 ni_columns = nrecords/ni_rows;
434 if (nrecords != ni_rows*ni_columns)
435 goto badspec;
436 if (transpose) {
437 if (no_columns <= 0)
438 no_columns = ni_rows;
439 if (no_rows <= 0)
440 no_rows = ni_columns;
441 if (outLevels <= 1 &&
442 (no_rows != ni_columns) | (no_columns != ni_rows))
443 goto badspec;
444 } else {
445 if (no_columns <= 0)
446 no_columns = ni_columns;
447 if (no_rows <= 0)
448 no_rows = ni_rows;
449 }
450 if (ni_rows*ni_columns != no_rows*no_columns) {
451 fputs("Number of input and output records do not match\n",
452 stderr);
453 return(0);
454 }
455 /* reorder records */
456 for (i = 0; i < no_rows; i++) {
457 for (j = 0; j < no_columns; j++) {
458 long n = get_input_pos(i, j);
459 if (n >= nrecords) {
460 fputs("Index past end-of-file\n", stderr);
461 return(0);
462 }
463 if (rp != NULL) { /* ASCII output */
464 print_record(rp, n);
465 putc(tabEOL[j >= no_columns-1], stdout);
466 } else { /* binary output */
467 putbinary((char *)mp->base + (n_comp*comp_size)*n,
468 comp_size, n_comp, stdout);
469 }
470 }
471 if (ferror(stdout)) {
472 fprintf(stderr, "Error writing to stdout\n");
473 return(0);
474 }
475 }
476 if (rp != NULL)
477 free_records(rp);
478 return(1);
479 badspec:
480 fprintf(stderr, "Bad dimension(s)\n");
481 return(0);
482 }
483
484 /* resize ASCII stream input by ignoring EOLs between records */
485 static int
486 do_resize(FILE *fp)
487 {
488 long records2go = ni_rows*ni_columns;
489 int columns2go = no_columns;
490 char word[256];
491 /* sanity checks */
492 if (comp_size || (no_columns == ni_columns) & (no_rows == ni_rows))
493 return(output_stream(fp)); /* no-op -- just copy */
494 if (no_columns <= 0) {
495 fprintf(stderr, "Missing -oc specification\n");
496 return(0);
497 }
498 if ((records2go <= 0) & (no_rows > 0))
499 records2go = no_rows*no_columns;
500 else if (no_rows*no_columns != records2go) {
501 fprintf(stderr,
502 "Input and output data sizes disagree (%dx%d != %dx%d)\n",
503 ni_rows, ni_columns, no_rows, no_columns);
504 return(0);
505 }
506 do { /* reshape records */
507 int n;
508
509 for (n = n_comp; n--; ) {
510 if (fget_word(word, fp) == NULL) {
511 if (records2go > 0 || n < n_comp-1)
512 break;
513 goto done; /* normal EOD */
514 }
515 fputs(word, stdout);
516 if (n) { /* mid-record? */
517 int c = getc(fp);
518 if ((c == '\n') | (c == EOF))
519 break;
520 ungetc(c, fp);
521 putc(' ', stdout);
522 }
523 }
524 if (n >= 0) {
525 fprintf(stderr, "Incomplete record / unexpected EOF\n");
526 return(0);
527 }
528 if (--columns2go <= 0) { /* time to end output row? */
529 putc('\n', stdout);
530 columns2go = no_columns;
531 } else /* else separate records */
532 putc('\t', stdout);
533 } while (--records2go); /* expected EOD? */
534 done:
535 if (warnings && columns2go != no_columns)
536 fprintf(stderr, "Warning -- incomplete final row\n");
537 if (warnings && fget_word(word, fp) != NULL)
538 fprintf(stderr, "Warning -- characters beyond expected EOD\n");
539 return(1);
540 }
541
542 /* process a header line and copy to stdout */
543 static int
544 headline(char *s, void *p)
545 {
546 static char fmt[MAXFMTLEN];
547 int n;
548
549 if (formatval(fmt, s)) {
550 if (fmtid == NULL) {
551 fmtid = fmt;
552 return(0);
553 }
554 if (!strcmp(fmt, fmtid))
555 return(0);
556 fprintf(stderr, "Input format '%s' != '%s'\n", fmt, fmtid);
557 return(-1);
558 }
559 if (!strncmp(s, "NROWS=", 6)) {
560 n = atoi(s+6);
561 if ((ni_rows > 0) & (n != ni_rows)) {
562 fputs("Incorrect input row count\n", stderr);
563 return(-1);
564 }
565 ni_rows = n;
566 return(0);
567 }
568 if (!strncmp(s, "NCOLS=", 6)) {
569 n = atoi(s+6);
570 if ((ni_columns > 0) & (n != ni_columns)) {
571 fputs("Incorrect input column count\n", stderr);
572 return(-1);
573 }
574 ni_columns = n;
575 return(0);
576 }
577 if (!strncmp(s, "NCOMP=", 6)) {
578 n = atoi(s+6);
579 if ((n_comp > 0) & (n != n_comp)) {
580 fputs("Incorrect number of components\n", stderr);
581 return(-1);
582 }
583 n_comp = n;
584 return(0);
585 }
586 if (o_header)
587 fputs(s, stdout); /* copy header info. */
588 return(0);
589 }
590
591 /* main routine for converting rows/columns in data file */
592 int
593 main(int argc, char *argv[])
594 {
595 int a;
596
597 for (a = 1; a < argc && argv[a][0] == '-'; a++)
598 switch (argv[a][1]) {
599 case 'i': /* input */
600 if (argv[a][2] == 'c') /* columns */
601 ni_columns = atoi(argv[++a]);
602 else if (argv[a][2] == 'r')
603 ni_rows = atoi(argv[++a]);
604 else
605 goto userr;
606 break;
607 case 'o': /* output */
608 if (argv[a][2] == 'c') /* columns */
609 no_columns = atoi(argv[++a]);
610 else if (argv[a][2] == 'r')
611 no_rows = atoi(argv[++a]);
612 else if (argv[a][2] ||
613 !(outLevels=get_array(argv[++a], outArray, MAXLEVELS)))
614 goto userr;
615 break;
616 case 'h': /* turn off header */
617 switch (argv[a][2]) {
618 case 'i':
619 i_header = 0;
620 break;
621 case 'o':
622 o_header = 0;
623 break;
624 case '\0':
625 i_header = o_header = 0;
626 break;
627 default:
628 goto userr;
629 }
630 break;
631 case 't': /* transpose on/off */
632 transpose = !transpose;
633 break;
634 case 'f': /* format */
635 switch (argv[a][2]) {
636 case 'a': /* ASCII */
637 case 'A':
638 fmtid = "ascii";
639 comp_size = 0;
640 break;
641 case 'f': /* float */
642 case 'F':
643 fmtid = "float";
644 comp_size = sizeof(float);
645 break;
646 case 'd': /* double */
647 case 'D':
648 fmtid = "double";
649 comp_size = sizeof(double);
650 break;
651 case 'b': /* binary (bytes) */
652 case 'B':
653 fmtid = "byte";
654 comp_size = 1;
655 break;
656 default:
657 goto userr;
658 }
659 if (argv[a][3]) {
660 if (!isdigit(argv[a][3]))
661 goto userr;
662 n_comp = atoi(argv[a]+3);
663 } else
664 n_comp = 1;
665 break;
666 case 'w': /* warnings on/off */
667 warnings = !warnings;
668 break;
669 default:
670 goto userr;
671 }
672 if (a < argc-1) /* arg count OK? */
673 goto userr;
674 if (outLevels) { /* should check consistency? */
675 no_rows = outArray[0][0];
676 no_columns = outArray[0][1];
677 }
678 /* open input file? */
679 if (a == argc-1 && freopen(argv[a], "r", stdin) == NULL) {
680 fprintf(stderr, "%s: cannot open for reading\n", argv[a]);
681 return(1);
682 }
683 if (comp_size) {
684 SET_FILE_BINARY(stdin);
685 SET_FILE_BINARY(stdout);
686 }
687 /* check for no-op */
688 if (!transpose & (outLevels <= 1) & (i_header == o_header) &&
689 (no_columns == ni_columns) & (no_rows == ni_rows)) {
690 if (warnings)
691 fprintf(stderr, "%s: no-op -- copying input verbatim\n",
692 argv[0]);
693 if (!output_stream(stdin))
694 return(1);
695 return(0);
696 }
697 if (i_header) { /* read header */
698 if (getheader(stdin, headline, NULL) < 0)
699 return(1);
700 if (!check_sizes())
701 return(1);
702 if (comp_size) { /* a little late... */
703 SET_FILE_BINARY(stdin);
704 SET_FILE_BINARY(stdout);
705 }
706 } else if (!check_sizes())
707 return(1);
708 if (o_header) { /* write/add to header */
709 if (!i_header)
710 newheader("RADIANCE", stdout);
711 printargs(a, argv, stdout);
712 if (no_rows > 0)
713 printf("NROWS=%d\n", no_rows);
714 if (no_columns > 0)
715 printf("NCOLS=%d\n", no_columns);
716 printf("NCOMP=%d\n", n_comp);
717 fputformat(fmtid, stdout);
718 fputc('\n', stdout); /* finish new header */
719 }
720 if (transpose | (outLevels > 1)) { /* moving stuff around? */
721 MEMLOAD myMem; /* need to map into memory */
722 if (a == argc-1) {
723 if (load_file(&myMem, stdin) <= 0) {
724 fprintf(stderr, "%s: error loading file into memory\n",
725 argv[a]);
726 return(1);
727 }
728 } else if (load_stream(&myMem, stdin) <= 0) {
729 fprintf(stderr, "%s: error loading stdin into memory\n",
730 argv[0]);
731 return(1);
732 }
733 if (!do_reorder(&myMem))
734 return(1);
735 /* free_load(&myMem); about to exit, so don't bother */
736 } else if (!do_resize(stdin)) /* reshaping input */
737 return(1);
738 return(0);
739 userr:
740 fprintf(stderr,
741 "Usage: %s [-h[io]][-w][-f[afdb][N]][-t][-ic in_col][-ir in_row][-oc out_col][-or out_row][-o RxC[xR1xC1..]] [input.dat]\n",
742 argv[0]);
743 return(1);
744 }