--- ray/src/util/rttree_reduce.c	2017/05/31 03:26:46	2.17
+++ ray/src/util/rttree_reduce.c	2020/05/11 20:26:12	2.19
@@ -1,5 +1,5 @@
 #ifndef lint
-static const char RCSid[] = "$Id: rttree_reduce.c,v 2.17 2017/05/31 03:26:46 greg Exp $";
+static const char RCSid[] = "$Id: rttree_reduce.c,v 2.19 2020/05/11 20:26:12 greg Exp $";
 #endif
 /*
  *  A utility called by genBSDF.pl to reduce tensor tree samples and output
@@ -19,8 +19,8 @@ int	log2g = 4;		/* log2 of grid resolution */
 int	infmt = 'a';		/* input format ('a','f','d') */
 double	pctcull = 95.;		/* target culling percentile */
 
-#define dval3(ix,ox,oy)		datarr[((((ix)<<log2g)+(ox))<<log2g)+(oy)]
-#define dval4(ix,iy,ox,oy)	datarr[((((((ix)<<log2g)+(iy))<<log2g)+(ox))<<log2g)+(oy)]
+#define dval3(ix,ox,oy)		datarr[((size_t)(((ix)<<log2g)+(ox))<<log2g)+(oy)]
+#define dval4(ix,iy,ox,oy)	datarr[((((size_t)(((ix)<<log2g)+(iy))<<log2g)+(ox))<<log2g)+(oy)]
 
 /* Tensor tree node */
 typedef struct ttree_s {
@@ -30,13 +30,13 @@ typedef struct ttree_s {
 } TNODE;
 
 #define HISTLEN		300	/* histogram resolution */
-#define HISTMAX		10.	/* maximum recorded measure in histogram */
+#define HISTMAX		4.	/* maximum recorded measure in histogram */
 
 int	histo[HISTLEN];		/* histogram freq. of variance measure */
 
 double	tthresh;		/* acceptance threshold (TBD) */
 
-#define var_measure(tp)		( ((tp)->vmax - (tp)->vmin) / \
+#define var_measure(tp)		sqrt( ((tp)->vmax - (tp)->vmin) / \
 					(sqrt((tp)->vavg) + .03) )
 #define above_threshold(tp)	(var_measure(tp) > tthresh)