ViewVC Help
View File | Revision Log | Show Annotations | Download File | Root Listing
root/radiance/ray/src/util/rcollate.c
Revision: 2.19
Committed: Thu Sep 18 00:12:42 2014 UTC (9 years, 7 months ago) by greg
Content type: text/plain
Branch: MAIN
CVS Tags: rad4R2P2
Changes since 2.18: +6 -9 lines
Log Message:
Simplified code a bit -- no significant change in behavior expected

File Contents

# Content
1 #ifndef lint
2 static const char RCSid[] = "$Id: rcollate.c,v 2.18 2014/08/01 18:21:04 greg Exp $";
3 #endif
4 /*
5 * Utility to re-order records in a binary or ASCII data file (matrix)
6 */
7
8 #include <stdlib.h>
9 #include <string.h>
10 #include <ctype.h>
11 #include "platform.h"
12 #include "rtio.h"
13 #include "resolu.h"
14 #ifdef _WIN32
15 #undef ftello
16 #define ftello ftell
17 #undef ssize_t
18 #define ssize_t size_t
19 #else
20 #include <sys/mman.h>
21 #endif
22
23 #ifdef getc_unlocked /* avoid horrendous overhead of flockfile */
24 #undef getc
25 #undef putc
26 #define getc getc_unlocked
27 #define putc putc_unlocked
28 #endif
29
30 typedef struct {
31 void *base; /* pointer to base memory */
32 size_t len; /* allocated memory length */
33 int mapped; /* memory-mapped file? */
34 } MEMLOAD; /* file loaded/mapped into memory */
35
36 typedef struct {
37 int nw_rec; /* number of words per record */
38 int nrecs; /* number of records we found */
39 char *rec[1]; /* record array (extends struct) */
40 } RECINDEX;
41
42 int warnings = 1; /* report warnings? */
43
44 /* free loaded file */
45 static void
46 free_load(MEMLOAD *mp)
47 {
48 if (mp == NULL || (mp->base == NULL) | (mp->len <= 0))
49 return;
50 #ifdef MAP_FILE
51 if (mp->mapped)
52 munmap(mp->base, mp->len);
53 else
54 #endif
55 free(mp->base);
56 mp->base = NULL;
57 mp->len = 0;
58 }
59
60 /* load a file into memory */
61 static int
62 load_file(MEMLOAD *mp, FILE *fp)
63 {
64 int fd;
65 off_t skip, flen;
66
67 if (mp == NULL)
68 return(-1);
69 mp->base = NULL;
70 mp->len = 0;
71 mp->mapped = 0;
72 if (fp == NULL)
73 return(-1);
74 fd = fileno(fp);
75 skip = ftello(fp);
76 flen = lseek(fd, 0, SEEK_END);
77 if (flen <= skip)
78 return((int)(flen - skip));
79 mp->len = (size_t)(flen - skip);
80 #ifdef MAP_FILE
81 if (mp->len > 1L<<20) { /* map file if > 1 MByte */
82 mp->base = mmap(NULL, mp->len, PROT_READ, MAP_PRIVATE, fd, skip);
83 if (mp->base != MAP_FAILED) {
84 mp->mapped = 1;
85 return(1); /* mmap() success */
86 }
87 mp->base = NULL; /* fall back to reading it in... */
88 }
89 #endif
90 if (lseek(fd, skip, SEEK_SET) != skip ||
91 (mp->base = malloc(mp->len)) == NULL) {
92 mp->len = 0;
93 return(-1);
94 }
95 if (read(fd, (char *)mp->base, mp->len) != mp->len) {
96 free_load(mp);
97 return(-1);
98 }
99 return(1);
100 }
101
102 /* load memory from an input stream, starting from current position */
103 static int
104 load_stream(MEMLOAD *mp, FILE *fp)
105 {
106 size_t alloced = 0;
107 char buf[8192];
108 size_t nr;
109
110 if (mp == NULL)
111 return(-1);
112 mp->base = NULL;
113 mp->len = 0;
114 mp->mapped = 0;
115 if (fp == NULL)
116 return(-1);
117 while ((nr = fread(buf, 1, sizeof(buf), fp)) > 0) {
118 if (!alloced)
119 mp->base = malloc(nr);
120 else if (mp->len+nr > alloced)
121 mp->base = realloc(mp->base,
122 alloced = alloced*(2+(nr==sizeof(buf)))/2+nr);
123 if (mp->base == NULL)
124 return(-1);
125 memcpy((char *)mp->base + mp->len, buf, nr);
126 mp->len += nr;
127 }
128 if (ferror(fp)) {
129 free_load(mp);
130 return(-1);
131 }
132 if (alloced > mp->len*5/4) /* don't waste too much space */
133 mp->base = realloc(mp->base, mp->len);
134 return(mp->len > 0);
135 }
136
137 /* free a record index */
138 #define free_records(rp) free(rp)
139
140 /* compute record index */
141 static RECINDEX *
142 index_records(const MEMLOAD *mp, int nw_rec)
143 {
144 RECINDEX *rp;
145 char *cp, *mend;
146 int n;
147
148 if (mp == NULL || (mp->base == NULL) | (mp->len <= 0))
149 return(NULL);
150 if (nw_rec <= 0)
151 return(NULL);
152 rp = (RECINDEX *)malloc(sizeof(RECINDEX) + mp->len/(2*nw_rec)*sizeof(char *));
153 if (rp == NULL)
154 return(NULL);
155 rp->nw_rec = nw_rec;
156 rp->nrecs = 0;
157 cp = (char *)mp->base;
158 mend = cp + mp->len;
159 for ( ; ; ) { /* whitespace-separated words */
160 while (cp < mend && !*cp | isspace(*cp))
161 ++cp;
162 if (cp >= mend)
163 break;
164 rp->rec[rp->nrecs++] = cp; /* point to first non-white */
165 n = rp->nw_rec;
166 while (++cp < mend) /* find end of record */
167 if (!*cp | isspace(*cp)) {
168 if (--n <= 0)
169 break; /* got requisite # words */
170 do { /* else find next word */
171 if (*cp == '\n') {
172 fprintf(stderr,
173 "Unexpected EOL in record!\n");
174 free_records(rp);
175 return(NULL);
176 }
177 if (++cp >= mend)
178 break;
179 } while (!*cp | isspace(*cp));
180 }
181 }
182 rp->rec[rp->nrecs] = mend; /* reallocate to save space */
183 rp = (RECINDEX *)realloc(rp,
184 sizeof(RECINDEX) + rp->nrecs*sizeof(char *));
185 return(rp);
186 }
187
188 /* count number of columns based on first EOL */
189 static int
190 count_columns(const RECINDEX *rp)
191 {
192 char *cp = rp->rec[0];
193 char *mend = rp->rec[rp->nrecs];
194 int i;
195
196 while (*cp != '\n')
197 if (++cp >= mend)
198 return(0);
199 for (i = 0; i < rp->nrecs; i++)
200 if (rp->rec[i] >= cp)
201 break;
202 return(i);
203 }
204
205 /* copy nth record from index to stdout */
206 static int
207 print_record(const RECINDEX *rp, int n)
208 {
209 int words2go = rp->nw_rec;
210 char *scp;
211
212 if ((n < 0) | (n >= rp->nrecs))
213 return(0);
214 scp = rp->rec[n];
215 do {
216 putc(*scp++, stdout);
217 if (!*scp | isspace(*scp)) {
218 if (--words2go <= 0)
219 break;
220 putc(' ', stdout); /* single space btwn. words */
221 do
222 if (++scp >= rp->rec[n+1])
223 break;
224 while (!*scp | isspace(*scp));
225 }
226 } while (scp < rp->rec[n+1]);
227 /* caller adds record sep. */
228 return(1);
229 }
230
231 /* copy a stream to stdout */
232 static int
233 output_stream(FILE *fp)
234 {
235 char buf[8192];
236 ssize_t n;
237
238 if (fp == NULL)
239 return(0);
240 fflush(stdout);
241 while ((n = fread(buf, 1, sizeof(buf), fp)) > 0)
242 if (write(fileno(stdout), buf, n) != n)
243 return(0);
244 return(!ferror(fp));
245 }
246
247 /* get next word from stream, leaving stream on EOL or start of next word */
248 static char *
249 fget_word(char buf[256], FILE *fp)
250 {
251 int c;
252 char *cp;
253 /* skip nul's and white space */
254 while (!(c = getc(fp)) || isspace(c))
255 ;
256 if (c == EOF)
257 return(NULL);
258 cp = buf;
259 do
260 *cp++ = c;
261 while ((c = getc(fp)) != EOF && !isspace(c) && cp < buf+255);
262 *cp = '\0';
263 while (isspace(c) & (c != '\n'))
264 c = getc(fp);
265 if (c != EOF)
266 ungetc(c, fp);
267 return(buf);
268 }
269
270 char *fmtid = NULL; /* format id */
271 int comp_size = 0; /* binary bytes/channel */
272 int n_comp = 0; /* components/record */
273 int ni_columns = 0; /* number of input columns */
274 int ni_rows = 0; /* number of input rows */
275 int no_columns = 0; /* number of output columns */
276 int no_rows = 0; /* number of output rows */
277 int transpose = 0; /* transpose rows & cols? */
278 int i_header = 1; /* input header? */
279 int o_header = 1; /* output header? */
280
281 /* check settings and assign defaults */
282 static int
283 check_sizes()
284 {
285 if (fmtid == NULL) {
286 fmtid = "ascii";
287 } else if (!comp_size) {
288 if (!strcmp(fmtid, "float"))
289 comp_size = sizeof(float);
290 else if (!strcmp(fmtid, "double"))
291 comp_size = sizeof(double);
292 else if (!strcmp(fmtid, "byte"))
293 comp_size = 1;
294 else if (strcmp(fmtid, "ascii")) {
295 fprintf(stderr, "Unsupported format: %s\n", fmtid);
296 return(0);
297 }
298 }
299 if (transpose && (no_rows <= 0) & (no_columns <= 0)) {
300 if (ni_rows > 0) no_columns = ni_rows;
301 if (ni_columns > 0) no_rows = ni_columns;
302 } else if ((no_rows <= 0) & (no_columns > 0) &&
303 !((ni_rows*ni_columns) % no_columns))
304 no_rows = ni_rows*ni_columns/no_columns;
305 if (n_comp <= 0)
306 n_comp = 3;
307 return(1);
308 }
309
310 /* output transposed ASCII or binary data from memory */
311 static int
312 do_transpose(const MEMLOAD *mp)
313 {
314 static const char tabEOL[2] = {'\t','\n'};
315 RECINDEX *rp = NULL;
316 long nrecords;
317 int i, j;
318 /* propogate sizes */
319 if (ni_rows <= 0)
320 ni_rows = no_columns;
321 if (ni_columns <= 0)
322 ni_columns = no_rows;
323 /* get # records (& index) */
324 if (!comp_size) {
325 if ((rp = index_records(mp, n_comp)) == NULL)
326 return(0);
327 if (ni_columns <= 0)
328 ni_columns = count_columns(rp);
329 nrecords = rp->nrecs;
330 } else if ((ni_rows > 0) & (ni_columns > 0)) {
331 nrecords = ni_rows*ni_columns;
332 if (nrecords > mp->len/(n_comp*comp_size)) {
333 fprintf(stderr,
334 "Input too small for specified size and type\n");
335 return(0);
336 }
337 } else
338 nrecords = mp->len/(n_comp*comp_size);
339 /* check sizes */
340 if ((ni_rows <= 0) & (ni_columns > 0))
341 ni_rows = nrecords/ni_columns;
342 if ((ni_columns <= 0) & (ni_rows > 0))
343 ni_columns = nrecords/ni_rows;
344 if (nrecords != ni_rows*ni_columns)
345 goto badspec;
346 if (no_columns <= 0)
347 no_columns = ni_rows;
348 if (no_rows <= 0)
349 no_rows = ni_columns;
350 if ((no_rows != ni_columns) | (no_columns != ni_rows))
351 goto badspec;
352 /* transpose records */
353 for (i = 0; i < no_rows; i++) {
354 for (j = 0; j < no_columns; j++)
355 if (rp != NULL) { /* ASCII output */
356 print_record(rp, j*ni_columns + i);
357 putc(tabEOL[j >= no_columns-1], stdout);
358 } else { /* binary output */
359 fwrite((char *)mp->base +
360 (n_comp*comp_size)*(j*ni_columns + i),
361 n_comp*comp_size, 1, stdout);
362 }
363 if (ferror(stdout)) {
364 fprintf(stderr, "Error writing to stdout\n");
365 return(0);
366 }
367 }
368 if (rp != NULL)
369 free_records(rp);
370 return(1);
371 badspec:
372 fprintf(stderr, "Bad transpose specification -- check dimension(s)\n");
373 return(0);
374 }
375
376 /* resize ASCII stream input by ignoring EOLs between records */
377 static int
378 do_resize(FILE *fp)
379 {
380 long records2go = ni_rows*ni_columns;
381 int columns2go = no_columns;
382 char word[256];
383 /* sanity checks */
384 if (comp_size || (no_columns == ni_columns) & (no_rows == ni_rows))
385 return(output_stream(fp)); /* no-op -- just copy */
386 if (no_columns <= 0) {
387 fprintf(stderr, "Missing -oc specification\n");
388 return(0);
389 }
390 if ((records2go <= 0) & (no_rows > 0))
391 records2go = no_rows*no_columns;
392 else if (no_rows*no_columns != records2go) {
393 fprintf(stderr,
394 "Input and output data sizes disagree (%dx%d != %dx%d)\n",
395 ni_rows, ni_columns, no_rows, no_columns);
396 return(0);
397 }
398 do { /* reshape records */
399 int n;
400
401 for (n = n_comp; n--; ) {
402 if (fget_word(word, fp) == NULL) {
403 if (records2go > 0 || n < n_comp-1)
404 break;
405 goto done; /* normal EOD */
406 }
407 fputs(word, stdout);
408 if (n) { /* mid-record? */
409 int c = getc(fp);
410 if ((c == '\n') | (c == EOF))
411 break;
412 ungetc(c, fp);
413 putc(' ', stdout);
414 }
415 }
416 if (n >= 0) {
417 fprintf(stderr, "Incomplete record / unexpected EOF\n");
418 return(0);
419 }
420 if (--columns2go <= 0) { /* time to end output row? */
421 putc('\n', stdout);
422 columns2go = no_columns;
423 } else /* else separate records */
424 putc('\t', stdout);
425 } while (--records2go); /* expected EOD? */
426 done:
427 if (warnings && columns2go != no_columns)
428 fprintf(stderr, "Warning -- incomplete final row\n");
429 if (warnings && fget_word(word, fp) != NULL)
430 fprintf(stderr, "Warning -- characters beyond expected EOD\n");
431 return(1);
432 }
433
434 /* process a header line and copy to stdout */
435 static int
436 headline(char *s, void *p)
437 {
438 static char fmt[32];
439 int n;
440
441 if (formatval(fmt, s)) {
442 if (fmtid == NULL) {
443 fmtid = fmt;
444 return(0);
445 }
446 if (!strcmp(fmt, fmtid))
447 return(0);
448 fprintf(stderr, "Input format '%s' != '%s'\n", fmt, fmtid);
449 return(-1);
450 }
451 if (!strncmp(s, "NROWS=", 6)) {
452 n = atoi(s+6);
453 if ((ni_rows > 0) & (n != ni_rows)) {
454 fputs("Incorrect input row count\n", stderr);
455 return(-1);
456 }
457 ni_rows = n;
458 return(0);
459 }
460 if (!strncmp(s, "NCOLS=", 6)) {
461 n = atoi(s+6);
462 if ((ni_columns > 0) & (n != ni_columns)) {
463 fputs("Incorrect input column count\n", stderr);
464 return(-1);
465 }
466 ni_columns = n;
467 return(0);
468 }
469 if (!strncmp(s, "NCOMP=", 6)) {
470 n = atoi(s+6);
471 if ((n_comp > 0) & (n != n_comp)) {
472 fputs("Incorrect number of components\n", stderr);
473 return(-1);
474 }
475 n_comp = n;
476 return(0);
477 }
478 if (o_header)
479 fputs(s, stdout); /* copy header info. */
480 return(0);
481 }
482
483 /* main routine for converting rows/columns in data file */
484 int
485 main(int argc, char *argv[])
486 {
487 int a;
488
489 for (a = 1; a < argc && argv[a][0] == '-'; a++)
490 switch (argv[a][1]) {
491 case 'i': /* input */
492 if (argv[a][2] == 'c') /* columns */
493 ni_columns = atoi(argv[++a]);
494 else if (argv[a][2] == 'r')
495 ni_rows = atoi(argv[++a]);
496 else
497 goto userr;
498 break;
499 case 'o': /* output */
500 if (argv[a][2] == 'c') /* columns */
501 no_columns = atoi(argv[++a]);
502 else if (argv[a][2] == 'r')
503 no_rows = atoi(argv[++a]);
504 else
505 goto userr;
506 break;
507 case 'h': /* turn off header */
508 switch (argv[a][2]) {
509 case 'i':
510 i_header = 0;
511 break;
512 case 'o':
513 o_header = 0;
514 break;
515 case '\0':
516 i_header = o_header = 0;
517 break;
518 default:
519 goto userr;
520 }
521 break;
522 case 't': /* transpose on/off */
523 transpose = !transpose;
524 break;
525 case 'f': /* format */
526 switch (argv[a][2]) {
527 case 'a': /* ASCII */
528 case 'A':
529 fmtid = "ascii";
530 comp_size = 0;
531 break;
532 case 'f': /* float */
533 case 'F':
534 fmtid = "float";
535 comp_size = sizeof(float);
536 break;
537 case 'd': /* double */
538 case 'D':
539 fmtid = "double";
540 comp_size = sizeof(double);
541 break;
542 case 'b': /* binary (bytes) */
543 case 'B':
544 fmtid = "byte";
545 comp_size = 1;
546 break;
547 default:
548 goto userr;
549 }
550 if (argv[a][3]) {
551 if (!isdigit(argv[a][3]))
552 goto userr;
553 n_comp = atoi(argv[a]+3);
554 } else
555 n_comp = 1;
556 break;
557 case 'w': /* warnings on/off */
558 warnings = !warnings;
559 break;
560 default:
561 goto userr;
562 }
563 if (a < argc-1) /* arg count OK? */
564 goto userr;
565 /* open input file? */
566 if (a == argc-1 && freopen(argv[a], "r", stdin) == NULL) {
567 fprintf(stderr, "%s: cannot open for reading\n", argv[a]);
568 return(1);
569 }
570 if (comp_size) {
571 SET_FILE_BINARY(stdin);
572 SET_FILE_BINARY(stdout);
573 }
574 /* check for no-op */
575 if (!transpose & (i_header == o_header) &&
576 (no_columns == ni_columns) & (no_rows == ni_rows)) {
577 if (warnings)
578 fprintf(stderr, "%s: no-op -- copying input verbatim\n",
579 argv[0]);
580 if (!output_stream(stdin))
581 return(1);
582 return(0);
583 }
584 if (i_header) { /* read header */
585 if (getheader(stdin, headline, NULL) < 0)
586 return(1);
587 if (!check_sizes())
588 return(1);
589 if (comp_size) { /* a little late... */
590 SET_FILE_BINARY(stdin);
591 SET_FILE_BINARY(stdout);
592 }
593 } else if (!check_sizes())
594 return(1);
595 if (o_header) { /* write header */
596 printargs(a, argv, stdout);
597 if (no_rows > 0)
598 printf("NROWS=%d\n", no_rows);
599 if (no_columns > 0)
600 printf("NCOLS=%d\n", no_columns);
601 printf("NCOMP=%d\n", n_comp);
602 fputformat(fmtid, stdout);
603 fputc('\n', stdout); /* finish new header */
604 }
605 if (transpose) { /* transposing rows & columns? */
606 MEMLOAD myMem; /* need to map into memory */
607 if (a == argc-1) {
608 if (load_file(&myMem, stdin) <= 0) {
609 fprintf(stderr, "%s: error loading file into memory\n",
610 argv[a]);
611 return(1);
612 }
613 } else if (load_stream(&myMem, stdin) <= 0) {
614 fprintf(stderr, "%s: error loading stdin into memory\n",
615 argv[0]);
616 return(1);
617 }
618 if (!do_transpose(&myMem))
619 return(1);
620 /* free_load(&myMem); about to exit, so don't bother */
621 } else if (!do_resize(stdin)) /* reshaping input */
622 return(1);
623 return(0);
624 userr:
625 fprintf(stderr,
626 "Usage: %s [-h[io]][-w][-f[afdb][N]][-t][-ic in_col][-ir in_row][-oc out_col][-or out_row] [input.dat]\n",
627 argv[0]);
628 return(1);
629 }