ViewVC Help
View File | Revision Log | Show Annotations | Download File | Root Listing
root/radiance/ray/src/util/rcollate.c
Revision: 2.21
Committed: Tue Jun 16 20:35:56 2015 UTC (8 years, 10 months ago) by greg
Content type: text/plain
Branch: MAIN
CVS Tags: rad5R0
Changes since 2.20: +2 -2 lines
Log Message:
Fixed bug in allocation for reading from stream

File Contents

# Content
1 #ifndef lint
2 static const char RCSid[] = "$Id: rcollate.c,v 2.20 2015/06/16 19:06:29 greg Exp $";
3 #endif
4 /*
5 * Utility to re-order records in a binary or ASCII data file (matrix)
6 */
7
8 #include <stdlib.h>
9 #include <string.h>
10 #include <ctype.h>
11 #include "platform.h"
12 #include "rtio.h"
13 #include "resolu.h"
14 #ifdef _WIN32
15 #undef ftello
16 #define ftello ftell
17 #undef ssize_t
18 #define ssize_t size_t
19 #else
20 #include <sys/mman.h>
21 #endif
22
23 #ifdef getc_unlocked /* avoid horrendous overhead of flockfile */
24 #undef getc
25 #undef putc
26 #define getc getc_unlocked
27 #define putc putc_unlocked
28 #endif
29
30 typedef struct {
31 void *base; /* pointer to base memory */
32 size_t len; /* allocated memory length */
33 int mapped; /* memory-mapped file? */
34 } MEMLOAD; /* file loaded/mapped into memory */
35
36 typedef struct {
37 int nw_rec; /* number of words per record */
38 int nrecs; /* number of records we found */
39 char *rec[1]; /* record array (extends struct) */
40 } RECINDEX;
41
42 int warnings = 1; /* report warnings? */
43
44 /* free loaded file */
45 static void
46 free_load(MEMLOAD *mp)
47 {
48 if (mp == NULL || (mp->base == NULL) | (mp->len <= 0))
49 return;
50 #ifdef MAP_FILE
51 if (mp->mapped)
52 munmap(mp->base, mp->len);
53 else
54 #endif
55 free(mp->base);
56 mp->base = NULL;
57 mp->len = 0;
58 }
59
60 /* load memory from an input stream, starting from current position */
61 static int
62 load_stream(MEMLOAD *mp, FILE *fp)
63 {
64 size_t alloced = 0;
65 char buf[8192];
66 size_t nr;
67
68 if (mp == NULL)
69 return(-1);
70 mp->base = NULL;
71 mp->len = 0;
72 mp->mapped = 0;
73 if (fp == NULL)
74 return(-1);
75 while ((nr = fread(buf, 1, sizeof(buf), fp)) > 0) {
76 if (!alloced)
77 mp->base = malloc(alloced = nr);
78 else if (mp->len+nr > alloced)
79 mp->base = realloc(mp->base,
80 alloced = alloced*(2+(nr==sizeof(buf)))/2+nr);
81 if (mp->base == NULL)
82 return(-1);
83 memcpy((char *)mp->base + mp->len, buf, nr);
84 mp->len += nr;
85 }
86 if (ferror(fp)) {
87 free_load(mp);
88 return(-1);
89 }
90 if (alloced > mp->len*5/4) /* don't waste too much space */
91 mp->base = realloc(mp->base, mp->len);
92 return(mp->len > 0);
93 }
94
95 /* load a file into memory */
96 static int
97 load_file(MEMLOAD *mp, FILE *fp)
98 {
99 int fd;
100 off_t skip, flen;
101
102 #ifdef _WIN32 /* too difficult to fix this */
103 return load_stream(mp, fp);
104 #endif
105 if (mp == NULL)
106 return(-1);
107 mp->base = NULL;
108 mp->len = 0;
109 mp->mapped = 0;
110 if (fp == NULL)
111 return(-1);
112 fd = fileno(fp);
113 skip = ftello(fp);
114 flen = lseek(fd, 0, SEEK_END);
115 if (flen <= skip)
116 return((int)(flen - skip));
117 mp->len = (size_t)(flen - skip);
118 #ifdef MAP_FILE
119 if (mp->len > 1L<<20) { /* map file if > 1 MByte */
120 mp->base = mmap(NULL, mp->len, PROT_READ, MAP_PRIVATE, fd, skip);
121 if (mp->base != MAP_FAILED) {
122 mp->mapped = 1;
123 return(1); /* mmap() success */
124 }
125 mp->base = NULL; /* fall back to reading it in... */
126 }
127 #endif
128 if (lseek(fd, skip, SEEK_SET) != skip ||
129 (mp->base = malloc(mp->len)) == NULL) {
130 mp->len = 0;
131 return(-1);
132 }
133 if (read(fd, (char *)mp->base, mp->len) != mp->len) {
134 free_load(mp);
135 return(-1);
136 }
137 return(1);
138 }
139
140 /* free a record index */
141 #define free_records(rp) free(rp)
142
143 /* compute record index */
144 static RECINDEX *
145 index_records(const MEMLOAD *mp, int nw_rec)
146 {
147 RECINDEX *rp;
148 char *cp, *mend;
149 int n;
150
151 if (mp == NULL || (mp->base == NULL) | (mp->len <= 0))
152 return(NULL);
153 if (nw_rec <= 0)
154 return(NULL);
155 rp = (RECINDEX *)malloc(sizeof(RECINDEX) + mp->len/(2*nw_rec)*sizeof(char *));
156 if (rp == NULL)
157 return(NULL);
158 rp->nw_rec = nw_rec;
159 rp->nrecs = 0;
160 cp = (char *)mp->base;
161 mend = cp + mp->len;
162 for ( ; ; ) { /* whitespace-separated words */
163 while (cp < mend && !*cp | isspace(*cp))
164 ++cp;
165 if (cp >= mend)
166 break;
167 rp->rec[rp->nrecs++] = cp; /* point to first non-white */
168 n = rp->nw_rec;
169 while (++cp < mend) /* find end of record */
170 if (!*cp | isspace(*cp)) {
171 if (--n <= 0)
172 break; /* got requisite # words */
173 do { /* else find next word */
174 if (*cp == '\n') {
175 fprintf(stderr,
176 "Unexpected EOL in record!\n");
177 free_records(rp);
178 return(NULL);
179 }
180 if (++cp >= mend)
181 break;
182 } while (!*cp | isspace(*cp));
183 }
184 }
185 rp->rec[rp->nrecs] = mend; /* reallocate to save space */
186 rp = (RECINDEX *)realloc(rp,
187 sizeof(RECINDEX) + rp->nrecs*sizeof(char *));
188 return(rp);
189 }
190
191 /* count number of columns based on first EOL */
192 static int
193 count_columns(const RECINDEX *rp)
194 {
195 char *cp = rp->rec[0];
196 char *mend = rp->rec[rp->nrecs];
197 int i;
198
199 while (*cp != '\n')
200 if (++cp >= mend)
201 return(0);
202 for (i = 0; i < rp->nrecs; i++)
203 if (rp->rec[i] >= cp)
204 break;
205 return(i);
206 }
207
208 /* copy nth record from index to stdout */
209 static int
210 print_record(const RECINDEX *rp, int n)
211 {
212 int words2go = rp->nw_rec;
213 char *scp;
214
215 if ((n < 0) | (n >= rp->nrecs))
216 return(0);
217 scp = rp->rec[n];
218 do {
219 putc(*scp++, stdout);
220 if (!*scp | isspace(*scp)) {
221 if (--words2go <= 0)
222 break;
223 putc(' ', stdout); /* single space btwn. words */
224 do
225 if (++scp >= rp->rec[n+1])
226 break;
227 while (!*scp | isspace(*scp));
228 }
229 } while (scp < rp->rec[n+1]);
230 /* caller adds record sep. */
231 return(1);
232 }
233
234 /* copy a stream to stdout */
235 static int
236 output_stream(FILE *fp)
237 {
238 char buf[8192];
239 ssize_t n;
240
241 if (fp == NULL)
242 return(0);
243 fflush(stdout);
244 while ((n = fread(buf, 1, sizeof(buf), fp)) > 0)
245 if (write(fileno(stdout), buf, n) != n)
246 return(0);
247 return(!ferror(fp));
248 }
249
250 /* get next word from stream, leaving stream on EOL or start of next word */
251 static char *
252 fget_word(char buf[256], FILE *fp)
253 {
254 int c;
255 char *cp;
256 /* skip nul's and white space */
257 while (!(c = getc(fp)) || isspace(c))
258 ;
259 if (c == EOF)
260 return(NULL);
261 cp = buf;
262 do
263 *cp++ = c;
264 while ((c = getc(fp)) != EOF && !isspace(c) && cp < buf+255);
265 *cp = '\0';
266 while (isspace(c) & (c != '\n'))
267 c = getc(fp);
268 if (c != EOF)
269 ungetc(c, fp);
270 return(buf);
271 }
272
273 char *fmtid = NULL; /* format id */
274 int comp_size = 0; /* binary bytes/channel */
275 int n_comp = 0; /* components/record */
276 int ni_columns = 0; /* number of input columns */
277 int ni_rows = 0; /* number of input rows */
278 int no_columns = 0; /* number of output columns */
279 int no_rows = 0; /* number of output rows */
280 int transpose = 0; /* transpose rows & cols? */
281 int i_header = 1; /* input header? */
282 int o_header = 1; /* output header? */
283
284 /* check settings and assign defaults */
285 static int
286 check_sizes()
287 {
288 if (fmtid == NULL) {
289 fmtid = "ascii";
290 } else if (!comp_size) {
291 if (!strcmp(fmtid, "float"))
292 comp_size = sizeof(float);
293 else if (!strcmp(fmtid, "double"))
294 comp_size = sizeof(double);
295 else if (!strcmp(fmtid, "byte"))
296 comp_size = 1;
297 else if (strcmp(fmtid, "ascii")) {
298 fprintf(stderr, "Unsupported format: %s\n", fmtid);
299 return(0);
300 }
301 }
302 if (transpose && (no_rows <= 0) & (no_columns <= 0)) {
303 if (ni_rows > 0) no_columns = ni_rows;
304 if (ni_columns > 0) no_rows = ni_columns;
305 } else if ((no_rows <= 0) & (no_columns > 0) &&
306 !((ni_rows*ni_columns) % no_columns))
307 no_rows = ni_rows*ni_columns/no_columns;
308 if (n_comp <= 0)
309 n_comp = 3;
310 return(1);
311 }
312
313 /* output transposed ASCII or binary data from memory */
314 static int
315 do_transpose(const MEMLOAD *mp)
316 {
317 static const char tabEOL[2] = {'\t','\n'};
318 RECINDEX *rp = NULL;
319 long nrecords;
320 int i, j;
321 /* propogate sizes */
322 if (ni_rows <= 0)
323 ni_rows = no_columns;
324 if (ni_columns <= 0)
325 ni_columns = no_rows;
326 /* get # records (& index) */
327 if (!comp_size) {
328 if ((rp = index_records(mp, n_comp)) == NULL)
329 return(0);
330 if (ni_columns <= 0)
331 ni_columns = count_columns(rp);
332 nrecords = rp->nrecs;
333 } else if ((ni_rows > 0) & (ni_columns > 0)) {
334 nrecords = ni_rows*ni_columns;
335 if (nrecords > mp->len/(n_comp*comp_size)) {
336 fprintf(stderr,
337 "Input too small for specified size and type\n");
338 return(0);
339 }
340 } else
341 nrecords = mp->len/(n_comp*comp_size);
342 /* check sizes */
343 if ((ni_rows <= 0) & (ni_columns > 0))
344 ni_rows = nrecords/ni_columns;
345 if ((ni_columns <= 0) & (ni_rows > 0))
346 ni_columns = nrecords/ni_rows;
347 if (nrecords != ni_rows*ni_columns)
348 goto badspec;
349 if (no_columns <= 0)
350 no_columns = ni_rows;
351 if (no_rows <= 0)
352 no_rows = ni_columns;
353 if ((no_rows != ni_columns) | (no_columns != ni_rows))
354 goto badspec;
355 /* transpose records */
356 for (i = 0; i < no_rows; i++) {
357 for (j = 0; j < no_columns; j++)
358 if (rp != NULL) { /* ASCII output */
359 print_record(rp, j*ni_columns + i);
360 putc(tabEOL[j >= no_columns-1], stdout);
361 } else { /* binary output */
362 fwrite((char *)mp->base +
363 (n_comp*comp_size)*(j*ni_columns + i),
364 n_comp*comp_size, 1, stdout);
365 }
366 if (ferror(stdout)) {
367 fprintf(stderr, "Error writing to stdout\n");
368 return(0);
369 }
370 }
371 if (rp != NULL)
372 free_records(rp);
373 return(1);
374 badspec:
375 fprintf(stderr, "Bad transpose specification -- check dimension(s)\n");
376 return(0);
377 }
378
379 /* resize ASCII stream input by ignoring EOLs between records */
380 static int
381 do_resize(FILE *fp)
382 {
383 long records2go = ni_rows*ni_columns;
384 int columns2go = no_columns;
385 char word[256];
386 /* sanity checks */
387 if (comp_size || (no_columns == ni_columns) & (no_rows == ni_rows))
388 return(output_stream(fp)); /* no-op -- just copy */
389 if (no_columns <= 0) {
390 fprintf(stderr, "Missing -oc specification\n");
391 return(0);
392 }
393 if ((records2go <= 0) & (no_rows > 0))
394 records2go = no_rows*no_columns;
395 else if (no_rows*no_columns != records2go) {
396 fprintf(stderr,
397 "Input and output data sizes disagree (%dx%d != %dx%d)\n",
398 ni_rows, ni_columns, no_rows, no_columns);
399 return(0);
400 }
401 do { /* reshape records */
402 int n;
403
404 for (n = n_comp; n--; ) {
405 if (fget_word(word, fp) == NULL) {
406 if (records2go > 0 || n < n_comp-1)
407 break;
408 goto done; /* normal EOD */
409 }
410 fputs(word, stdout);
411 if (n) { /* mid-record? */
412 int c = getc(fp);
413 if ((c == '\n') | (c == EOF))
414 break;
415 ungetc(c, fp);
416 putc(' ', stdout);
417 }
418 }
419 if (n >= 0) {
420 fprintf(stderr, "Incomplete record / unexpected EOF\n");
421 return(0);
422 }
423 if (--columns2go <= 0) { /* time to end output row? */
424 putc('\n', stdout);
425 columns2go = no_columns;
426 } else /* else separate records */
427 putc('\t', stdout);
428 } while (--records2go); /* expected EOD? */
429 done:
430 if (warnings && columns2go != no_columns)
431 fprintf(stderr, "Warning -- incomplete final row\n");
432 if (warnings && fget_word(word, fp) != NULL)
433 fprintf(stderr, "Warning -- characters beyond expected EOD\n");
434 return(1);
435 }
436
437 /* process a header line and copy to stdout */
438 static int
439 headline(char *s, void *p)
440 {
441 static char fmt[32];
442 int n;
443
444 if (formatval(fmt, s)) {
445 if (fmtid == NULL) {
446 fmtid = fmt;
447 return(0);
448 }
449 if (!strcmp(fmt, fmtid))
450 return(0);
451 fprintf(stderr, "Input format '%s' != '%s'\n", fmt, fmtid);
452 return(-1);
453 }
454 if (!strncmp(s, "NROWS=", 6)) {
455 n = atoi(s+6);
456 if ((ni_rows > 0) & (n != ni_rows)) {
457 fputs("Incorrect input row count\n", stderr);
458 return(-1);
459 }
460 ni_rows = n;
461 return(0);
462 }
463 if (!strncmp(s, "NCOLS=", 6)) {
464 n = atoi(s+6);
465 if ((ni_columns > 0) & (n != ni_columns)) {
466 fputs("Incorrect input column count\n", stderr);
467 return(-1);
468 }
469 ni_columns = n;
470 return(0);
471 }
472 if (!strncmp(s, "NCOMP=", 6)) {
473 n = atoi(s+6);
474 if ((n_comp > 0) & (n != n_comp)) {
475 fputs("Incorrect number of components\n", stderr);
476 return(-1);
477 }
478 n_comp = n;
479 return(0);
480 }
481 if (o_header)
482 fputs(s, stdout); /* copy header info. */
483 return(0);
484 }
485
486 /* main routine for converting rows/columns in data file */
487 int
488 main(int argc, char *argv[])
489 {
490 int a;
491
492 for (a = 1; a < argc && argv[a][0] == '-'; a++)
493 switch (argv[a][1]) {
494 case 'i': /* input */
495 if (argv[a][2] == 'c') /* columns */
496 ni_columns = atoi(argv[++a]);
497 else if (argv[a][2] == 'r')
498 ni_rows = atoi(argv[++a]);
499 else
500 goto userr;
501 break;
502 case 'o': /* output */
503 if (argv[a][2] == 'c') /* columns */
504 no_columns = atoi(argv[++a]);
505 else if (argv[a][2] == 'r')
506 no_rows = atoi(argv[++a]);
507 else
508 goto userr;
509 break;
510 case 'h': /* turn off header */
511 switch (argv[a][2]) {
512 case 'i':
513 i_header = 0;
514 break;
515 case 'o':
516 o_header = 0;
517 break;
518 case '\0':
519 i_header = o_header = 0;
520 break;
521 default:
522 goto userr;
523 }
524 break;
525 case 't': /* transpose on/off */
526 transpose = !transpose;
527 break;
528 case 'f': /* format */
529 switch (argv[a][2]) {
530 case 'a': /* ASCII */
531 case 'A':
532 fmtid = "ascii";
533 comp_size = 0;
534 break;
535 case 'f': /* float */
536 case 'F':
537 fmtid = "float";
538 comp_size = sizeof(float);
539 break;
540 case 'd': /* double */
541 case 'D':
542 fmtid = "double";
543 comp_size = sizeof(double);
544 break;
545 case 'b': /* binary (bytes) */
546 case 'B':
547 fmtid = "byte";
548 comp_size = 1;
549 break;
550 default:
551 goto userr;
552 }
553 if (argv[a][3]) {
554 if (!isdigit(argv[a][3]))
555 goto userr;
556 n_comp = atoi(argv[a]+3);
557 } else
558 n_comp = 1;
559 break;
560 case 'w': /* warnings on/off */
561 warnings = !warnings;
562 break;
563 default:
564 goto userr;
565 }
566 if (a < argc-1) /* arg count OK? */
567 goto userr;
568 /* open input file? */
569 if (a == argc-1 && freopen(argv[a], "r", stdin) == NULL) {
570 fprintf(stderr, "%s: cannot open for reading\n", argv[a]);
571 return(1);
572 }
573 if (comp_size) {
574 SET_FILE_BINARY(stdin);
575 SET_FILE_BINARY(stdout);
576 }
577 /* check for no-op */
578 if (!transpose & (i_header == o_header) &&
579 (no_columns == ni_columns) & (no_rows == ni_rows)) {
580 if (warnings)
581 fprintf(stderr, "%s: no-op -- copying input verbatim\n",
582 argv[0]);
583 if (!output_stream(stdin))
584 return(1);
585 return(0);
586 }
587 if (i_header) { /* read header */
588 if (getheader(stdin, headline, NULL) < 0)
589 return(1);
590 if (!check_sizes())
591 return(1);
592 if (comp_size) { /* a little late... */
593 SET_FILE_BINARY(stdin);
594 SET_FILE_BINARY(stdout);
595 }
596 } else if (!check_sizes())
597 return(1);
598 if (o_header) { /* write header */
599 printargs(a, argv, stdout);
600 if (no_rows > 0)
601 printf("NROWS=%d\n", no_rows);
602 if (no_columns > 0)
603 printf("NCOLS=%d\n", no_columns);
604 printf("NCOMP=%d\n", n_comp);
605 fputformat(fmtid, stdout);
606 fputc('\n', stdout); /* finish new header */
607 }
608 if (transpose) { /* transposing rows & columns? */
609 MEMLOAD myMem; /* need to map into memory */
610 if (a == argc-1) {
611 if (load_file(&myMem, stdin) <= 0) {
612 fprintf(stderr, "%s: error loading file into memory\n",
613 argv[a]);
614 return(1);
615 }
616 } else if (load_stream(&myMem, stdin) <= 0) {
617 fprintf(stderr, "%s: error loading stdin into memory\n",
618 argv[0]);
619 return(1);
620 }
621 if (!do_transpose(&myMem))
622 return(1);
623 /* free_load(&myMem); about to exit, so don't bother */
624 } else if (!do_resize(stdin)) /* reshaping input */
625 return(1);
626 return(0);
627 userr:
628 fprintf(stderr,
629 "Usage: %s [-h[io]][-w][-f[afdb][N]][-t][-ic in_col][-ir in_row][-oc out_col][-or out_row] [input.dat]\n",
630 argv[0]);
631 return(1);
632 }