ViewVC Help
View File | Revision Log | Show Annotations | Download File | Root Listing
root/radiance/ray/src/util/rcollate.c
Revision: 2.20
Committed: Tue Jun 16 19:06:29 2015 UTC (8 years, 9 months ago) by greg
Content type: text/plain
Branch: MAIN
Changes since 2.19: +39 -36 lines
Log Message:
Work-around fix for transposing ASCII input files under Windows

File Contents

# User Rev Content
1 greg 2.1 #ifndef lint
2 greg 2.20 static const char RCSid[] = "$Id: rcollate.c,v 2.19 2014/09/18 00:12:42 greg Exp $";
3 greg 2.1 #endif
4     /*
5     * Utility to re-order records in a binary or ASCII data file (matrix)
6     */
7    
8     #include <stdlib.h>
9     #include <string.h>
10     #include <ctype.h>
11     #include "platform.h"
12     #include "rtio.h"
13     #include "resolu.h"
14 greg 2.7 #ifdef _WIN32
15     #undef ftello
16     #define ftello ftell
17     #undef ssize_t
18     #define ssize_t size_t
19     #else
20 greg 2.1 #include <sys/mman.h>
21     #endif
22    
23     #ifdef getc_unlocked /* avoid horrendous overhead of flockfile */
24     #undef getc
25     #undef putc
26     #define getc getc_unlocked
27     #define putc putc_unlocked
28     #endif
29    
30     typedef struct {
31     void *base; /* pointer to base memory */
32     size_t len; /* allocated memory length */
33     int mapped; /* memory-mapped file? */
34     } MEMLOAD; /* file loaded/mapped into memory */
35    
36     typedef struct {
37     int nw_rec; /* number of words per record */
38     int nrecs; /* number of records we found */
39     char *rec[1]; /* record array (extends struct) */
40     } RECINDEX;
41    
42 greg 2.4 int warnings = 1; /* report warnings? */
43    
44 greg 2.1 /* free loaded file */
45     static void
46     free_load(MEMLOAD *mp)
47     {
48     if (mp == NULL || (mp->base == NULL) | (mp->len <= 0))
49     return;
50     #ifdef MAP_FILE
51     if (mp->mapped)
52     munmap(mp->base, mp->len);
53     else
54     #endif
55     free(mp->base);
56     mp->base = NULL;
57     mp->len = 0;
58     }
59    
60 greg 2.20 /* load memory from an input stream, starting from current position */
61     static int
62     load_stream(MEMLOAD *mp, FILE *fp)
63     {
64     size_t alloced = 0;
65     char buf[8192];
66     size_t nr;
67    
68     if (mp == NULL)
69     return(-1);
70     mp->base = NULL;
71     mp->len = 0;
72     mp->mapped = 0;
73     if (fp == NULL)
74     return(-1);
75     while ((nr = fread(buf, 1, sizeof(buf), fp)) > 0) {
76     if (!alloced)
77     mp->base = malloc(nr);
78     else if (mp->len+nr > alloced)
79     mp->base = realloc(mp->base,
80     alloced = alloced*(2+(nr==sizeof(buf)))/2+nr);
81     if (mp->base == NULL)
82     return(-1);
83     memcpy((char *)mp->base + mp->len, buf, nr);
84     mp->len += nr;
85     }
86     if (ferror(fp)) {
87     free_load(mp);
88     return(-1);
89     }
90     if (alloced > mp->len*5/4) /* don't waste too much space */
91     mp->base = realloc(mp->base, mp->len);
92     return(mp->len > 0);
93     }
94    
95 greg 2.1 /* load a file into memory */
96     static int
97     load_file(MEMLOAD *mp, FILE *fp)
98     {
99     int fd;
100     off_t skip, flen;
101    
102 greg 2.20 #ifdef _WIN32 /* too difficult to fix this */
103     return load_stream(mp, fp);
104     #endif
105 greg 2.1 if (mp == NULL)
106     return(-1);
107     mp->base = NULL;
108     mp->len = 0;
109     mp->mapped = 0;
110     if (fp == NULL)
111     return(-1);
112     fd = fileno(fp);
113     skip = ftello(fp);
114     flen = lseek(fd, 0, SEEK_END);
115     if (flen <= skip)
116     return((int)(flen - skip));
117     mp->len = (size_t)(flen - skip);
118     #ifdef MAP_FILE
119     if (mp->len > 1L<<20) { /* map file if > 1 MByte */
120 greg 2.2 mp->base = mmap(NULL, mp->len, PROT_READ, MAP_PRIVATE, fd, skip);
121 greg 2.1 if (mp->base != MAP_FAILED) {
122     mp->mapped = 1;
123     return(1); /* mmap() success */
124     }
125     mp->base = NULL; /* fall back to reading it in... */
126     }
127     #endif
128     if (lseek(fd, skip, SEEK_SET) != skip ||
129     (mp->base = malloc(mp->len)) == NULL) {
130     mp->len = 0;
131     return(-1);
132     }
133     if (read(fd, (char *)mp->base, mp->len) != mp->len) {
134     free_load(mp);
135     return(-1);
136     }
137     return(1);
138     }
139    
140     /* free a record index */
141     #define free_records(rp) free(rp)
142    
143     /* compute record index */
144     static RECINDEX *
145     index_records(const MEMLOAD *mp, int nw_rec)
146     {
147     RECINDEX *rp;
148     char *cp, *mend;
149     int n;
150    
151     if (mp == NULL || (mp->base == NULL) | (mp->len <= 0))
152     return(NULL);
153     if (nw_rec <= 0)
154     return(NULL);
155     rp = (RECINDEX *)malloc(sizeof(RECINDEX) + mp->len/(2*nw_rec)*sizeof(char *));
156     if (rp == NULL)
157     return(NULL);
158     rp->nw_rec = nw_rec;
159     rp->nrecs = 0;
160     cp = (char *)mp->base;
161     mend = cp + mp->len;
162     for ( ; ; ) { /* whitespace-separated words */
163     while (cp < mend && !*cp | isspace(*cp))
164     ++cp;
165     if (cp >= mend)
166     break;
167     rp->rec[rp->nrecs++] = cp; /* point to first non-white */
168     n = rp->nw_rec;
169     while (++cp < mend) /* find end of record */
170     if (!*cp | isspace(*cp)) {
171     if (--n <= 0)
172     break; /* got requisite # words */
173     do { /* else find next word */
174     if (*cp == '\n') {
175     fprintf(stderr,
176     "Unexpected EOL in record!\n");
177     free_records(rp);
178     return(NULL);
179     }
180     if (++cp >= mend)
181     break;
182     } while (!*cp | isspace(*cp));
183     }
184     }
185     rp->rec[rp->nrecs] = mend; /* reallocate to save space */
186     rp = (RECINDEX *)realloc(rp,
187     sizeof(RECINDEX) + rp->nrecs*sizeof(char *));
188     return(rp);
189     }
190    
191     /* count number of columns based on first EOL */
192     static int
193     count_columns(const RECINDEX *rp)
194     {
195     char *cp = rp->rec[0];
196     char *mend = rp->rec[rp->nrecs];
197     int i;
198    
199     while (*cp != '\n')
200     if (++cp >= mend)
201     return(0);
202     for (i = 0; i < rp->nrecs; i++)
203     if (rp->rec[i] >= cp)
204     break;
205     return(i);
206     }
207    
208     /* copy nth record from index to stdout */
209     static int
210     print_record(const RECINDEX *rp, int n)
211     {
212     int words2go = rp->nw_rec;
213     char *scp;
214    
215     if ((n < 0) | (n >= rp->nrecs))
216     return(0);
217     scp = rp->rec[n];
218     do {
219     putc(*scp++, stdout);
220     if (!*scp | isspace(*scp)) {
221     if (--words2go <= 0)
222     break;
223     putc(' ', stdout); /* single space btwn. words */
224     do
225     if (++scp >= rp->rec[n+1])
226     break;
227     while (!*scp | isspace(*scp));
228     }
229     } while (scp < rp->rec[n+1]);
230     /* caller adds record sep. */
231     return(1);
232     }
233    
234     /* copy a stream to stdout */
235     static int
236     output_stream(FILE *fp)
237     {
238     char buf[8192];
239     ssize_t n;
240    
241     if (fp == NULL)
242     return(0);
243 greg 2.13 fflush(stdout);
244     while ((n = fread(buf, 1, sizeof(buf), fp)) > 0)
245 greg 2.1 if (write(fileno(stdout), buf, n) != n)
246     return(0);
247 greg 2.13 return(!ferror(fp));
248 greg 2.1 }
249    
250     /* get next word from stream, leaving stream on EOL or start of next word */
251     static char *
252     fget_word(char buf[256], FILE *fp)
253     {
254     int c;
255     char *cp;
256     /* skip nul's and white space */
257     while (!(c = getc(fp)) || isspace(c))
258     ;
259     if (c == EOF)
260     return(NULL);
261     cp = buf;
262     do
263     *cp++ = c;
264     while ((c = getc(fp)) != EOF && !isspace(c) && cp < buf+255);
265     *cp = '\0';
266     while (isspace(c) & (c != '\n'))
267     c = getc(fp);
268     if (c != EOF)
269     ungetc(c, fp);
270     return(buf);
271     }
272    
273 greg 2.9 char *fmtid = NULL; /* format id */
274     int comp_size = 0; /* binary bytes/channel */
275     int n_comp = 0; /* components/record */
276 greg 2.1 int ni_columns = 0; /* number of input columns */
277     int ni_rows = 0; /* number of input rows */
278     int no_columns = 0; /* number of output columns */
279     int no_rows = 0; /* number of output rows */
280 greg 2.16 int transpose = 0; /* transpose rows & cols? */
281     int i_header = 1; /* input header? */
282     int o_header = 1; /* output header? */
283 greg 2.1
284 greg 2.9 /* check settings and assign defaults */
285 greg 2.11 static int
286 greg 2.9 check_sizes()
287     {
288     if (fmtid == NULL) {
289     fmtid = "ascii";
290     } else if (!comp_size) {
291     if (!strcmp(fmtid, "float"))
292     comp_size = sizeof(float);
293     else if (!strcmp(fmtid, "double"))
294     comp_size = sizeof(double);
295     else if (!strcmp(fmtid, "byte"))
296     comp_size = 1;
297 greg 2.14 else if (strcmp(fmtid, "ascii")) {
298 greg 2.11 fprintf(stderr, "Unsupported format: %s\n", fmtid);
299     return(0);
300     }
301 greg 2.9 }
302 greg 2.16 if (transpose && (no_rows <= 0) & (no_columns <= 0)) {
303     if (ni_rows > 0) no_columns = ni_rows;
304     if (ni_columns > 0) no_rows = ni_columns;
305     } else if ((no_rows <= 0) & (no_columns > 0) &&
306     !((ni_rows*ni_columns) % no_columns))
307     no_rows = ni_rows*ni_columns/no_columns;
308 greg 2.9 if (n_comp <= 0)
309     n_comp = 3;
310 greg 2.11 return(1);
311 greg 2.9 }
312    
313 greg 2.1 /* output transposed ASCII or binary data from memory */
314     static int
315     do_transpose(const MEMLOAD *mp)
316     {
317     static const char tabEOL[2] = {'\t','\n'};
318     RECINDEX *rp = NULL;
319     long nrecords;
320     int i, j;
321 greg 2.5 /* propogate sizes */
322     if (ni_rows <= 0)
323     ni_rows = no_columns;
324     if (ni_columns <= 0)
325     ni_columns = no_rows;
326 greg 2.1 /* get # records (& index) */
327 greg 2.9 if (!comp_size) {
328     if ((rp = index_records(mp, n_comp)) == NULL)
329 greg 2.1 return(0);
330     if (ni_columns <= 0)
331     ni_columns = count_columns(rp);
332     nrecords = rp->nrecs;
333 greg 2.3 } else if ((ni_rows > 0) & (ni_columns > 0)) {
334 greg 2.1 nrecords = ni_rows*ni_columns;
335 greg 2.9 if (nrecords > mp->len/(n_comp*comp_size)) {
336 greg 2.3 fprintf(stderr,
337     "Input too small for specified size and type\n");
338     return(0);
339     }
340     } else
341 greg 2.9 nrecords = mp->len/(n_comp*comp_size);
342 greg 2.1 /* check sizes */
343     if ((ni_rows <= 0) & (ni_columns > 0))
344     ni_rows = nrecords/ni_columns;
345     if ((ni_columns <= 0) & (ni_rows > 0))
346     ni_columns = nrecords/ni_rows;
347     if (nrecords != ni_rows*ni_columns)
348     goto badspec;
349     if (no_columns <= 0)
350     no_columns = ni_rows;
351     if (no_rows <= 0)
352     no_rows = ni_columns;
353     if ((no_rows != ni_columns) | (no_columns != ni_rows))
354     goto badspec;
355     /* transpose records */
356     for (i = 0; i < no_rows; i++) {
357     for (j = 0; j < no_columns; j++)
358     if (rp != NULL) { /* ASCII output */
359     print_record(rp, j*ni_columns + i);
360     putc(tabEOL[j >= no_columns-1], stdout);
361     } else { /* binary output */
362     fwrite((char *)mp->base +
363 greg 2.9 (n_comp*comp_size)*(j*ni_columns + i),
364     n_comp*comp_size, 1, stdout);
365 greg 2.1 }
366     if (ferror(stdout)) {
367     fprintf(stderr, "Error writing to stdout\n");
368     return(0);
369     }
370     }
371     if (rp != NULL)
372     free_records(rp);
373     return(1);
374     badspec:
375     fprintf(stderr, "Bad transpose specification -- check dimension(s)\n");
376     return(0);
377     }
378    
379     /* resize ASCII stream input by ignoring EOLs between records */
380     static int
381     do_resize(FILE *fp)
382     {
383     long records2go = ni_rows*ni_columns;
384     int columns2go = no_columns;
385     char word[256];
386     /* sanity checks */
387 greg 2.19 if (comp_size || (no_columns == ni_columns) & (no_rows == ni_rows))
388     return(output_stream(fp)); /* no-op -- just copy */
389 greg 2.1 if (no_columns <= 0) {
390     fprintf(stderr, "Missing -oc specification\n");
391     return(0);
392     }
393     if ((records2go <= 0) & (no_rows > 0))
394     records2go = no_rows*no_columns;
395     else if (no_rows*no_columns != records2go) {
396     fprintf(stderr,
397     "Input and output data sizes disagree (%dx%d != %dx%d)\n",
398     ni_rows, ni_columns, no_rows, no_columns);
399     return(0);
400     }
401     do { /* reshape records */
402     int n;
403    
404 greg 2.9 for (n = n_comp; n--; ) {
405 greg 2.1 if (fget_word(word, fp) == NULL) {
406 greg 2.9 if (records2go > 0 || n < n_comp-1)
407 greg 2.1 break;
408     goto done; /* normal EOD */
409     }
410     fputs(word, stdout);
411     if (n) { /* mid-record? */
412     int c = getc(fp);
413     if ((c == '\n') | (c == EOF))
414     break;
415     ungetc(c, fp);
416     putc(' ', stdout);
417     }
418     }
419     if (n >= 0) {
420     fprintf(stderr, "Incomplete record / unexpected EOF\n");
421     return(0);
422     }
423     if (--columns2go <= 0) { /* time to end output row? */
424     putc('\n', stdout);
425     columns2go = no_columns;
426     } else /* else separate records */
427     putc('\t', stdout);
428     } while (--records2go); /* expected EOD? */
429     done:
430 greg 2.4 if (warnings && columns2go != no_columns)
431 greg 2.1 fprintf(stderr, "Warning -- incomplete final row\n");
432 greg 2.4 if (warnings && fget_word(word, fp) != NULL)
433     fprintf(stderr, "Warning -- characters beyond expected EOD\n");
434 greg 2.1 return(1);
435     }
436    
437     /* process a header line and copy to stdout */
438     static int
439     headline(char *s, void *p)
440     {
441 greg 2.9 static char fmt[32];
442     int n;
443 greg 2.1
444     if (formatval(fmt, s)) {
445 greg 2.9 if (fmtid == NULL) {
446     fmtid = fmt;
447     return(0);
448     }
449 greg 2.1 if (!strcmp(fmt, fmtid))
450     return(0);
451     fprintf(stderr, "Input format '%s' != '%s'\n", fmt, fmtid);
452     return(-1);
453     }
454 greg 2.9 if (!strncmp(s, "NROWS=", 6)) {
455     n = atoi(s+6);
456     if ((ni_rows > 0) & (n != ni_rows)) {
457     fputs("Incorrect input row count\n", stderr);
458     return(-1);
459     }
460     ni_rows = n;
461     return(0);
462     }
463     if (!strncmp(s, "NCOLS=", 6)) {
464     n = atoi(s+6);
465     if ((ni_columns > 0) & (n != ni_columns)) {
466     fputs("Incorrect input column count\n", stderr);
467     return(-1);
468     }
469     ni_columns = n;
470     return(0);
471     }
472     if (!strncmp(s, "NCOMP=", 6)) {
473     n = atoi(s+6);
474     if ((n_comp > 0) & (n != n_comp)) {
475 greg 2.14 fputs("Incorrect number of components\n", stderr);
476 greg 2.9 return(-1);
477     }
478     n_comp = n;
479     return(0);
480     }
481 greg 2.16 if (o_header)
482     fputs(s, stdout); /* copy header info. */
483 greg 2.1 return(0);
484     }
485    
486     /* main routine for converting rows/columns in data file */
487     int
488     main(int argc, char *argv[])
489     {
490 greg 2.14 int a;
491 greg 2.1
492 greg 2.14 for (a = 1; a < argc && argv[a][0] == '-'; a++)
493     switch (argv[a][1]) {
494 greg 2.1 case 'i': /* input */
495 greg 2.14 if (argv[a][2] == 'c') /* columns */
496     ni_columns = atoi(argv[++a]);
497     else if (argv[a][2] == 'r')
498     ni_rows = atoi(argv[++a]);
499 greg 2.1 else
500     goto userr;
501     break;
502     case 'o': /* output */
503 greg 2.14 if (argv[a][2] == 'c') /* columns */
504     no_columns = atoi(argv[++a]);
505     else if (argv[a][2] == 'r')
506     no_rows = atoi(argv[++a]);
507 greg 2.1 else
508     goto userr;
509     break;
510 greg 2.10 case 'h': /* turn off header */
511 greg 2.14 switch (argv[a][2]) {
512 greg 2.10 case 'i':
513     i_header = 0;
514     break;
515     case 'o':
516     o_header = 0;
517     break;
518     case '\0':
519     i_header = o_header = 0;
520     break;
521     default:
522     goto userr;
523     }
524 greg 2.1 break;
525     case 't': /* transpose on/off */
526     transpose = !transpose;
527     break;
528     case 'f': /* format */
529 greg 2.14 switch (argv[a][2]) {
530 greg 2.1 case 'a': /* ASCII */
531     case 'A':
532     fmtid = "ascii";
533 greg 2.9 comp_size = 0;
534 greg 2.1 break;
535     case 'f': /* float */
536     case 'F':
537     fmtid = "float";
538 greg 2.9 comp_size = sizeof(float);
539 greg 2.1 break;
540     case 'd': /* double */
541     case 'D':
542     fmtid = "double";
543 greg 2.9 comp_size = sizeof(double);
544 greg 2.1 break;
545     case 'b': /* binary (bytes) */
546     case 'B':
547     fmtid = "byte";
548 greg 2.9 comp_size = 1;
549 greg 2.1 break;
550     default:
551     goto userr;
552     }
553 greg 2.14 if (argv[a][3]) {
554     if (!isdigit(argv[a][3]))
555 greg 2.1 goto userr;
556 greg 2.14 n_comp = atoi(argv[a]+3);
557     } else
558     n_comp = 1;
559 greg 2.1 break;
560 greg 2.4 case 'w': /* warnings on/off */
561     warnings = !warnings;
562     break;
563 greg 2.1 default:
564     goto userr;
565     }
566 greg 2.14 if (a < argc-1) /* arg count OK? */
567 greg 2.1 goto userr;
568     /* open input file? */
569 greg 2.14 if (a == argc-1 && freopen(argv[a], "r", stdin) == NULL) {
570     fprintf(stderr, "%s: cannot open for reading\n", argv[a]);
571 greg 2.1 return(1);
572     }
573 greg 2.9 if (comp_size) {
574 greg 2.1 SET_FILE_BINARY(stdin);
575     SET_FILE_BINARY(stdout);
576     }
577     /* check for no-op */
578 greg 2.19 if (!transpose & (i_header == o_header) &&
579     (no_columns == ni_columns) & (no_rows == ni_rows)) {
580 greg 2.4 if (warnings)
581     fprintf(stderr, "%s: no-op -- copying input verbatim\n",
582 greg 2.1 argv[0]);
583     if (!output_stream(stdin))
584     return(1);
585     return(0);
586     }
587 greg 2.10 if (i_header) { /* read header */
588 greg 2.15 if (getheader(stdin, headline, NULL) < 0)
589 greg 2.1 return(1);
590 greg 2.11 if (!check_sizes())
591     return(1);
592 greg 2.9 if (comp_size) { /* a little late... */
593     SET_FILE_BINARY(stdin);
594     SET_FILE_BINARY(stdout);
595     }
596 greg 2.11 } else if (!check_sizes())
597     return(1);
598 greg 2.10 if (o_header) { /* write header */
599 greg 2.14 printargs(a, argv, stdout);
600 greg 2.9 if (no_rows > 0)
601     printf("NROWS=%d\n", no_rows);
602     if (no_columns > 0)
603     printf("NCOLS=%d\n", no_columns);
604     printf("NCOMP=%d\n", n_comp);
605 greg 2.1 fputformat(fmtid, stdout);
606     fputc('\n', stdout); /* finish new header */
607 greg 2.10 }
608 greg 2.1 if (transpose) { /* transposing rows & columns? */
609 greg 2.19 MEMLOAD myMem; /* need to map into memory */
610 greg 2.14 if (a == argc-1) {
611 greg 2.1 if (load_file(&myMem, stdin) <= 0) {
612     fprintf(stderr, "%s: error loading file into memory\n",
613 greg 2.14 argv[a]);
614 greg 2.1 return(1);
615     }
616     } else if (load_stream(&myMem, stdin) <= 0) {
617     fprintf(stderr, "%s: error loading stdin into memory\n",
618     argv[0]);
619     return(1);
620     }
621     if (!do_transpose(&myMem))
622     return(1);
623 greg 2.17 /* free_load(&myMem); about to exit, so don't bother */
624     } else if (!do_resize(stdin)) /* reshaping input */
625 greg 2.1 return(1);
626     return(0);
627     userr:
628     fprintf(stderr,
629 greg 2.10 "Usage: %s [-h[io]][-w][-f[afdb][N]][-t][-ic in_col][-ir in_row][-oc out_col][-or out_row] [input.dat]\n",
630 greg 2.1 argv[0]);
631     return(1);
632     }