src/common/color.c

#ifndef lint
static const char       RCSid[] = "$Id: color.c,v 2.25 2023/06/30 15:00:32 greg Exp $";
#endif
/*
 *  color.c - routines for color calculations.
 *
 *  Externals declared in color.h
 */

#include "copyright.h"

#include  <stdio.h>
#include  <stdlib.h>
#include  <math.h>
#include  "color.h"

#ifdef getc_unlocked            /* avoid horrendous overhead of flockfile */
#undef getc
#undef putc
#undef ferror
#define getc    getc_unlocked
#define putc    putc_unlocked
#define ferror  ferror_unlocked
#endif

#define  MINELEN        8       /* minimum scanline length for encoding */
#define  MAXELEN        0x7fff  /* maximum scanline length for encoding */
#define  MINRUN         4       /* minimum run length */


void *
tempbuffer(                     /* get a temporary buffer */
        unsigned int  len
)
{
        static void             *tempbuf = NULL;
        static unsigned int     tempbuflen = 0;

        if (!len) {             /* call to free */
                if (tempbuflen) {
                        free(tempbuf);
                        tempbuf = NULL;
                        tempbuflen = 0;
                }
                return(NULL);
        }
        if (len <= tempbuflen)  /* big enough already? */
                return(tempbuf);
                                /* else free & reallocate */
        if (tempbuflen)
                free(tempbuf);
        tempbuf = malloc(len);
        tempbuflen = len*(tempbuf != NULL);
        return(tempbuf);
}


int
fwritecolrs(                    /* write out a colr scanline */
        COLR  *scanline,
        int  len,
        FILE  *fp
)
{
        int  i, j, beg, cnt = 1;
        int  c2;
        
        if ((len < MINELEN) | (len > MAXELEN))  /* OOBs, write out flat */
                return(fwrite((char *)scanline,sizeof(COLR),len,fp) - len);
                                        /* put magic header */
        putc(2, fp);
        putc(2, fp);
        putc(len>>8, fp);
        putc(len&0xff, fp);
                                        /* put components seperately */
        for (i = 0; i < 4; i++) {
            for (j = 0; j < len; j += cnt) {    /* find next run */
                for (beg = j; beg < len; beg += cnt) {
                    for (cnt = 1; (cnt < 127) & (beg+cnt < len) &&
                            scanline[beg+cnt][i] == scanline[beg][i]; cnt++)
                        ;
                    if (cnt >= MINRUN)
                        break;                  /* long enough */
                }
                if ((beg-j > 1) & (beg-j < MINRUN)) {
                    c2 = j+1;
                    while (scanline[c2++][i] == scanline[j][i])
                        if (c2 == beg) {        /* short run */
                            putc(128+beg-j, fp);
                            putc(scanline[j][i], fp);
                            j = beg;
                            break;
                        }
                }
                while (j < beg) {               /* write out non-run */
                    if ((c2 = beg-j) > 128) c2 = 128;
                    putc(c2, fp);
                    while (c2--)
                        putc(scanline[j++][i], fp);
                }
                if (cnt >= MINRUN) {            /* write out run */
                    putc(128+cnt, fp);
                    putc(scanline[beg][i], fp);
                } else
                    cnt = 0;
            }
        }
        return(ferror(fp) ? -1 : 0);
}

/*
 * An old-format scanline is either a stream of valid RGBE or XYZE real
 * pixels or at least one real pixel followed by some number of
 * invalid real pixels of the form (1,1,1,n), where n is a count.
 * These can themselves be repeated to create a multibyte repeat
 * count, with the least significant byte first (little-endian order.)
 * Repeat counts are limited by the size of an int; if a repetition
 * leads to an overrun, the rest of the the repetition will be
 * silently ignored.
 */
static int
oldreadcolrs(                   /* read in an old-style colr scanline */
        COLR  *scanline,
        int  len,
        FILE  *fp
)
{
        int  rshift = 0;
        int  i;
        
        while (len > 0) {
                scanline[0][RED] = getc(fp);
                scanline[0][GRN] = getc(fp);
                scanline[0][BLU] = getc(fp);
                scanline[0][EXP] = i = getc(fp);
                if (i == EOF)
                        return(-1);
                if (scanline[0][GRN] == 1 &&
                                (scanline[0][RED] == 1) &
                                (scanline[0][BLU] == 1)) {
                        i = scanline[0][EXP] << rshift;
                        while (i--) {
                                copycolr(scanline[0], scanline[-1]);
                                if (--len <= 0)
                                        return(0);
                                scanline++;
                        }
                        rshift += 8;
                } else {
                        scanline++;
                        len--;
                        rshift = 0;
                }
        }
        return(0);
}

/* 
 * There are two scanline formats: old and new.  The old format
 * compresses runs of RGBE or XYZE four-byte real pixels; the new
 * format breaks the pixels into R, G, B, and E lines (or XYZE lines)
 * which are individually run-length encoded.
 *
 * An old-format scanline always begins with a valid real pixel; at
 * least one of the RGB (or XYZ) values will have its high-order bit
 * set.  A new-format scanline begins with four bytes which are not a
 * valid real pixel: (2, 2, lenhigh, lenlow) where lenhigh is always
 * less than 128 and hence never has a high-order bit set.
 *
 * A new-format scanline is broken into its RGBE or XYZE components.
 * Each is output and run-length encoded separately so that a scanline
 * is broken into four records.  In turn, each record is organized
 * into chunks of up to 128 characters, which begin with a count byte.
 * If the count byte is greater than 128, the following data byte is
 * repeated (count-128) times.  If not, the count byte is followed by
 * that many data bytes.
 */
int
freadcolrs(                     /* read in an encoded colr scanline */
        COLR  *scanline,
        int  len,
        FILE  *fp
)
{
        int  i, j;
        int  code, val;
                                        /* determine scanline type */
        if (len <= 0)
                return(0);
        if ((i = getc(fp)) == EOF)
                return(-1);
        scanline[0][RED] = i;
        scanline[0][GRN] = getc(fp);
        scanline[0][BLU] = getc(fp);
        if ((i = getc(fp)) == EOF)
                return(-1);
        if ((scanline[0][RED] != 2) | (scanline[0][GRN] != 2) |
                        (scanline[0][BLU] & 0x80)) {
                scanline[0][EXP] = i;
                return(oldreadcolrs(scanline+1, len-1, fp));
        }
        if ((scanline[0][BLU]<<8 | i) != len)
                return(-1);             /* length mismatch! */
                                        /* read each component */
        for (i = 0; i < 4; i++)
            for (j = 0; j < len; ) {
                if ((code = getc(fp)) == EOF)
                    return(-1);
                if (code > 128) {       /* run */
                    code &= 127;
                    if ((val = getc(fp)) == EOF)
                        return -1;
                    if (j + code > len)
                        return -1;      /* overrun */
                    while (code--)
                        scanline[j++][i] = val;
                } else {                /* non-run */
                    if (j + code > len)
                        return -1;      /* overrun */
                    while (code--) {
                        if ((val = getc(fp)) == EOF)
                            return -1;
                        scanline[j++][i] = val;
                    }
                }
            }
        return(0);
}


int
fwritescan(                     /* write out a scanline */
        COLOR  *scanline,
        int  len,
        FILE  *fp
)
{
        COLR  *clrscan;
        int  n;
        COLR  *sp;
                                        /* get scanline buffer */
        if ((sp = (COLR *)tempbuffer(len*sizeof(COLR))) == NULL)
                return(-1);
        clrscan = sp;
                                        /* convert scanline */
        n = len;
        while (n-- > 0) {
                setcolr(sp[0], scanline[0][RED],
                                  scanline[0][GRN],
                                  scanline[0][BLU]);
                scanline++;
                sp++;
        }
        return(fwritecolrs(clrscan, len, fp));
}


int
freadscan(                      /* read in a scanline */
        COLOR  *scanline,
        int  len,
        FILE  *fp
)
{
        COLR  *clrscan;

        if ((clrscan = (COLR *)tempbuffer(len*sizeof(COLR))) == NULL)
                return(-1);
        if (freadcolrs(clrscan, len, fp) < 0)
                return(-1);
                                        /* convert scanline */
        colr_color(scanline[0], clrscan[0]);
        while (--len > 0) {
                scanline++; clrscan++;
                if (clrscan[0][GRN] == clrscan[-1][GRN] &&
                            (clrscan[0][RED] == clrscan[-1][RED]) &
                            (clrscan[0][BLU] == clrscan[-1][BLU]) &
                            (clrscan[0][EXP] == clrscan[-1][EXP]))
                        copycolor(scanline[0], scanline[-1]);
                else
                        colr_color(scanline[0], clrscan[0]);
        }
        return(0);
}


void
setcolr(                        /* assign a short color value */
        COLR  clr,
        double  r,
        double  g,
        double  b
)
{
        double  d;
        int  e;
        
        d = r > g ? r : g;
        if (b > d) d = b;

        if (d <= 1e-32) {
                clr[RED] = clr[GRN] = clr[BLU] = 0;
                clr[EXP] = 0;
                return;
        }

        d = frexp(d, &e) * 256.0 / d;

        if (r > 0.0)
                clr[RED] = r * d;
        else
                clr[RED] = 0;
        if (g > 0.0)
                clr[GRN] = g * d;
        else
                clr[GRN] = 0;
        if (b > 0.0)
                clr[BLU] = b * d;
        else
                clr[BLU] = 0;

        clr[EXP] = e + COLXS;
}


void
colr_color(                     /* convert short to float color */
        COLOR  col,
        COLR  clr
)
{
        double  f;
        
        if (clr[EXP] == 0)
                col[RED] = col[GRN] = col[BLU] = 0.0;
        else {
                f = ldexp(1.0, (int)clr[EXP]-(COLXS+8));
                col[RED] = (clr[RED] + 0.5)*f;
                col[GRN] = (clr[GRN] + 0.5)*f;
                col[BLU] = (clr[BLU] + 0.5)*f;
        }
}


int
bigdiff(                                /* c1 delta c2 > md? */
        COLOR  c1,
        COLOR  c2,
        double  md
)
{
        int  i;

        for (i = 0; i < 3; i++)
                if (colval(c1,i)-colval(c2,i) > md*colval(c2,i) ||
                                colval(c2,i)-colval(c1,i) > md*colval(c1,i))
                        return(1);
        return(0);
}
Revision:	2.26
Committed:	Sat Jul 1 01:31:17 2023 UTC (11 months, 1 week ago) by greg
Content type:	text/plain
Branch:	MAIN
CVS Tags:	rad5R4
Changes since 2.25:	+6 -9 lines
Log Message:	fix: Better fix to potential vulnerability from malicious/corrupted image
#	User	Rev	Content
1	greg	1.1	#ifndef lint
2	greg	2.26	static const char RCSid[] = "$Id: color.c,v 2.25 2023/06/30 15:00:32 greg Exp $";
3	greg	1.1	#endif
4			/*
5			* color.c - routines for color calculations.
6			*
7	greg	2.9	* Externals declared in color.h
8			*/
9
10	greg	2.10	#include "copyright.h"
11	greg	1.1
12			#include <stdio.h>
13	greg	2.9	#include <stdlib.h>
14	greg	2.7	#include <math.h>
15	greg	1.1	#include "color.h"
16	greg	2.14
17			#ifdef getc_unlocked /* avoid horrendous overhead of flockfile */
18	greg	2.15	#undef getc
19			#undef putc
20	greg	2.19	#undef ferror
21	greg	2.14	#define getc getc_unlocked
22			#define putc putc_unlocked
23	greg	2.18	#define ferror ferror_unlocked
24	greg	2.14	#endif
25	greg	1.1
26	greg	1.14	#define MINELEN 8 /* minimum scanline length for encoding */
27	greg	2.6	#define MAXELEN 0x7fff /* maximum scanline length for encoding */
28	greg	1.14	#define MINRUN 4 /* minimum run length */
29
30	greg	2.4
31	greg	2.22	void *
32	greg	2.17	tempbuffer( /* get a temporary buffer */
33			unsigned int len
34			)
35	greg	1.14	{
36	greg	2.22	static void *tempbuf = NULL;
37			static unsigned int tempbuflen = 0;
38	greg	1.14
39	greg	2.22	if (!len) { /* call to free */
40			if (tempbuflen) {
41	greg	2.20	free(tempbuf);
42	greg	2.22	tempbuf = NULL;
43			tempbuflen = 0;
44			}
45			return(NULL);
46	greg	1.14	}
47	greg	2.22	if (len <= tempbuflen) /* big enough already? */
48			return(tempbuf);
49			/* else free & reallocate */
50			if (tempbuflen)
51			free(tempbuf);
52			tempbuf = malloc(len);
53			tempbuflen = len*(tempbuf != NULL);
54	greg	1.14	return(tempbuf);
55			}
56
57
58	greg	2.9	int
59	greg	2.17	fwritecolrs( /* write out a colr scanline */
60			COLR *scanline,
61			int len,
62			FILE *fp
63			)
64	greg	1.1	{
65	greg	2.17	int i, j, beg, cnt = 1;
66	greg	1.14	int c2;
67	greg	1.1
68	schorsch	2.13	if ((len < MINELEN) \| (len > MAXELEN)) /* OOBs, write out flat */
69	greg	1.14	return(fwrite((char *)scanline,sizeof(COLR),len,fp) - len);
70	greg	2.2	/* put magic header */
71			putc(2, fp);
72	greg	1.14	putc(2, fp);
73			putc(len>>8, fp);
74	greg	2.21	putc(len&0xff, fp);
75	greg	1.14	/* put components seperately */
76			for (i = 0; i < 4; i++) {
77			for (j = 0; j < len; j += cnt) { /* find next run */
78			for (beg = j; beg < len; beg += cnt) {
79	greg	2.20	for (cnt = 1; (cnt < 127) & (beg+cnt < len) &&
80	greg	1.14	scanline[beg+cnt][i] == scanline[beg][i]; cnt++)
81			;
82			if (cnt >= MINRUN)
83			break; /* long enough */
84	greg	1.1	}
85	greg	2.20	if ((beg-j > 1) & (beg-j < MINRUN)) {
86	greg	1.15	c2 = j+1;
87			while (scanline[c2++][i] == scanline[j][i])
88			if (c2 == beg) { /* short run */
89			putc(128+beg-j, fp);
90			putc(scanline[j][i], fp);
91			j = beg;
92			break;
93			}
94			}
95			while (j < beg) { /* write out non-run */
96	greg	1.14	if ((c2 = beg-j) > 128) c2 = 128;
97			putc(c2, fp);
98			while (c2--)
99			putc(scanline[j++][i], fp);
100			}
101			if (cnt >= MINRUN) { /* write out run */
102			putc(128+cnt, fp);
103			putc(scanline[beg][i], fp);
104			} else
105			cnt = 0;
106			}
107	greg	1.1	}
108			return(ferror(fp) ? -1 : 0);
109			}
110
111	greg	2.23	/*
112			* An old-format scanline is either a stream of valid RGBE or XYZE real
113			* pixels or at least one real pixel followed by some number of
114			* invalid real pixels of the form (1,1,1,n), where n is a count.
115			* These can themselves be repeated to create a multibyte repeat
116			* count, with the least significant byte first (little-endian order.)
117			* Repeat counts are limited by the size of an int; if a repetition
118			* leads to an overrun, the rest of the the repetition will be
119			* silently ignored.
120			*/
121	greg	2.9	static int
122	greg	2.19	oldreadcolrs( /* read in an old-style colr scanline */
123	greg	2.17	COLR *scanline,
124			int len,
125			FILE *fp
126			)
127	greg	2.9	{
128	greg	2.25	int rshift = 0;
129	greg	2.17	int i;
130	greg	2.9
131			while (len > 0) {
132			scanline[0][RED] = getc(fp);
133			scanline[0][GRN] = getc(fp);
134			scanline[0][BLU] = getc(fp);
135	greg	2.18	scanline[0][EXP] = i = getc(fp);
136			if (i == EOF)
137	greg	2.9	return(-1);
138	greg	2.20	if (scanline[0][GRN] == 1 &&
139			(scanline[0][RED] == 1) &
140	greg	2.25	(scanline[0][BLU] == 1)) {
141	greg	2.20	i = scanline[0][EXP] << rshift;
142			while (i--) {
143	greg	2.9	copycolr(scanline[0], scanline[-1]);
144	greg	2.20	if (--len <= 0)
145			return(0);
146	greg	2.9	scanline++;
147			}
148			rshift += 8;
149			} else {
150			scanline++;
151			len--;
152			rshift = 0;
153			}
154			}
155			return(0);
156			}
157
158	greg	2.23	/*
159			* There are two scanline formats: old and new. The old format
160			* compresses runs of RGBE or XYZE four-byte real pixels; the new
161			* format breaks the pixels into R, G, B, and E lines (or XYZE lines)
162			* which are individually run-length encoded.
163			*
164			* An old-format scanline always begins with a valid real pixel; at
165			* least one of the RGB (or XYZ) values will have its high-order bit
166			* set. A new-format scanline begins with four bytes which are not a
167			* valid real pixel: (2, 2, lenhigh, lenlow) where lenhigh is always
168			* less than 128 and hence never has a high-order bit set.
169			*
170			* A new-format scanline is broken into its RGBE or XYZE components.
171			* Each is output and run-length encoded separately so that a scanline
172			* is broken into four records. In turn, each record is organized
173			* into chunks of up to 128 characters, which begin with a count byte.
174			* If the count byte is greater than 128, the following data byte is
175			* repeated (count-128) times. If not, the count byte is followed by
176			* that many data bytes.
177			*/
178	greg	2.9	int
179	greg	2.17	freadcolrs( /* read in an encoded colr scanline */
180			COLR *scanline,
181			int len,
182			FILE *fp
183			)
184	greg	1.1	{
185	greg	2.17	int i, j;
186	greg	2.6	int code, val;
187	greg	1.14	/* determine scanline type */
188	greg	2.26	if (len <= 0)
189			return(0);
190	greg	1.14	if ((i = getc(fp)) == EOF)
191			return(-1);
192	greg	2.26	scanline[0][RED] = i;
193	greg	1.14	scanline[0][GRN] = getc(fp);
194			scanline[0][BLU] = getc(fp);
195			if ((i = getc(fp)) == EOF)
196			return(-1);
197	greg	2.26	if ((scanline[0][RED] != 2) \| (scanline[0][GRN] != 2) \|
198			(scanline[0][BLU] & 0x80)) {
199	greg	1.14	scanline[0][EXP] = i;
200			return(oldreadcolrs(scanline+1, len-1, fp));
201			}
202			if ((scanline[0][BLU]<<8 \| i) != len)
203			return(-1); /* length mismatch! */
204			/* read each component */
205			for (i = 0; i < 4; i++)
206			for (j = 0; j < len; ) {
207			if ((code = getc(fp)) == EOF)
208			return(-1);
209			if (code > 128) { /* run */
210	greg	2.6	code &= 127;
211	greg	2.9	if ((val = getc(fp)) == EOF)
212			return -1;
213	greg	2.16	if (j + code > len)
214			return -1; /* overrun */
215	greg	2.6	while (code--)
216			scanline[j++][i] = val;
217	greg	2.16	} else { /* non-run */
218			if (j + code > len)
219			return -1; /* overrun */
220	greg	2.9	while (code--) {
221			if ((val = getc(fp)) == EOF)
222			return -1;
223			scanline[j++][i] = val;
224			}
225	greg	2.16	}
226	greg	1.14	}
227	greg	1.1	return(0);
228			}
229
230
231	greg	2.9	int
232	greg	2.17	fwritescan( /* write out a scanline */
233			COLOR *scanline,
234			int len,
235			FILE *fp
236			)
237	greg	1.1	{
238	greg	1.14	COLR *clrscan;
239			int n;
240	greg	2.17	COLR *sp;
241	greg	1.14	/* get scanline buffer */
242			if ((sp = (COLR )tempbuffer(lensizeof(COLR))) == NULL)
243			return(-1);
244			clrscan = sp;
245			/* convert scanline */
246			n = len;
247			while (n-- > 0) {
248			setcolr(sp[0], scanline[0][RED],
249	greg	1.1	scanline[0][GRN],
250			scanline[0][BLU]);
251			scanline++;
252	greg	1.14	sp++;
253	greg	1.1	}
254	greg	1.14	return(fwritecolrs(clrscan, len, fp));
255	greg	1.1	}
256
257
258	greg	2.9	int
259	greg	2.17	freadscan( /* read in a scanline */
260			COLOR *scanline,
261			int len,
262			FILE *fp
263			)
264	greg	1.1	{
265	greg	2.17	COLR *clrscan;
266	greg	1.14
267			if ((clrscan = (COLR )tempbuffer(lensizeof(COLR))) == NULL)
268			return(-1);
269			if (freadcolrs(clrscan, len, fp) < 0)
270			return(-1);
271			/* convert scanline */
272			colr_color(scanline[0], clrscan[0]);
273			while (--len > 0) {
274			scanline++; clrscan++;
275	greg	2.20	if (clrscan[0][GRN] == clrscan[-1][GRN] &&
276			(clrscan[0][RED] == clrscan[-1][RED]) &
277			(clrscan[0][BLU] == clrscan[-1][BLU]) &
278			(clrscan[0][EXP] == clrscan[-1][EXP]))
279	greg	1.14	copycolor(scanline[0], scanline[-1]);
280			else
281			colr_color(scanline[0], clrscan[0]);
282	greg	1.1	}
283			return(0);
284			}
285
286
287	greg	2.9	void
288	greg	2.17	setcolr( /* assign a short color value */
289			COLR clr,
290			double r,
291			double g,
292			double b
293			)
294	greg	1.1	{
295			double d;
296			int e;
297
298			d = r > g ? r : g;
299			if (b > d) d = b;
300
301	greg	1.4	if (d <= 1e-32) {
302	greg	1.1	clr[RED] = clr[GRN] = clr[BLU] = 0;
303			clr[EXP] = 0;
304			return;
305			}
306
307	greg	2.21	d = frexp(d, &e) * 256.0 / d;
308	greg	1.1
309	greg	2.11	if (r > 0.0)
310			clr[RED] = r * d;
311			else
312			clr[RED] = 0;
313			if (g > 0.0)
314			clr[GRN] = g * d;
315			else
316			clr[GRN] = 0;
317			if (b > 0.0)
318			clr[BLU] = b * d;
319			else
320			clr[BLU] = 0;
321
322	greg	1.1	clr[EXP] = e + COLXS;
323			}
324
325
326	greg	2.9	void
327	greg	2.17	colr_color( /* convert short to float color */
328			COLOR col,
329			COLR clr
330			)
331	greg	1.1	{
332	greg	1.6	double f;
333	greg	1.1
334			if (clr[EXP] == 0)
335			col[RED] = col[GRN] = col[BLU] = 0.0;
336			else {
337	greg	1.2	f = ldexp(1.0, (int)clr[EXP]-(COLXS+8));
338	greg	1.1	col[RED] = (clr[RED] + 0.5)*f;
339			col[GRN] = (clr[GRN] + 0.5)*f;
340			col[BLU] = (clr[BLU] + 0.5)*f;
341			}
342	greg	1.6	}
343
344
345	greg	2.9	int
346	greg	2.17	bigdiff( /* c1 delta c2 > md? */
347			COLOR c1,
348			COLOR c2,
349			double md
350			)
351	greg	1.7	{
352	greg	2.17	int i;
353	greg	1.7
354			for (i = 0; i < 3; i++)
355			if (colval(c1,i)-colval(c2,i) > md*colval(c2,i) \|\|
356	greg	2.20	colval(c2,i)-colval(c1,i) > md*colval(c1,i))
357	greg	1.7	return(1);
358			return(0);
359			}