--- ray/src/hd/holofile.c	1997/12/11 19:57:58	3.11
+++ ray/src/hd/holofile.c	1999/03/09 08:40:11	3.39
@@ -1,4 +1,4 @@
-/* Copyright (c) 1997 Silicon Graphics, Inc. */
+/* Copyright (c) 1999 Silicon Graphics, Inc. */
 
 #ifndef lint
 static char SCCSid[] = "$SunId$ SGI";
@@ -13,57 +13,90 @@ static char SCCSid[] = "$SunId$ SGI";
 #include "holo.h"
 
 #ifndef CACHESIZE
-#define CACHESIZE	16	/* default cache size (Mbytes, 0==inf) */
+#ifdef BIGMEM
+#define CACHESIZE	17	/* default cache size (Mbytes, 0==inf) */
+#else
+#define CACHESIZE	5
 #endif
+#endif
 #ifndef FREEBEAMS
-#define FREEBEAMS	512	/* maximum beams to free at a time */
+#define FREEBEAMS	1500	/* maximum beams to free at a time */
 #endif
 #ifndef PCTFREE
-#define PCTFREE		20	/* maximum fraction to free (%) */
+#define PCTFREE		15	/* maximum fraction to free (%) */
 #endif
+#ifndef MAXFRAGB
+#define MAXFRAGB	16	/* fragment blocks/file to track (0==inf) */
+#endif
+#ifndef FF_DEFAULT
+				/* when to free a beam fragment */
+#define FF_DEFAULT	(FF_WRITE|FF_KILL)
+#endif
+#ifndef MINDIRSEL
+				/* minimum directory seek length */
+#define MINDIRSEL	(4*BUFSIZ/sizeof(BEAMI))
+#endif
 
-/* define MAXFRAG if you want to limit fragment tracking memory */
-
 #ifndef BSD
 #define write	writebuf	/* safe i/o routines */
 #define read	readbuf
 #endif
 
-#define FRAGBLK		64	/* number of fragments to allocate at a time */
+#define FRAGBLK		512	/* number of fragments to allocate at a time */
 
-int	hdcachesize = CACHESIZE*1024*1024;	/* target cache size (bytes) */
-unsigned long	hdclock;	/* clock value */
+int	hdfragflags = FF_DEFAULT;	/* tells when to free fragments */
+unsigned	hdcachesize = CACHESIZE*1024*1024;	/* target cache size */
+unsigned long	hdclock;		/* clock value */
 
 HOLO	*hdlist[HDMAX+1];	/* holodeck pointers (NULL term.) */
 
-static struct fragment {
+static struct fraglist {
 	short	nlinks;		/* number of holodeck sections using us */
-	short	nfrags;		/* number of known fragments */
+	short	writerr;	/* write error encountered */
+	int	nfrags;		/* number of known fragments */
 	BEAMI	*fi;		/* fragments, descending file position */
 	long	flen;		/* last known file length */
-} *hdfrag;		/* fragment lists, indexed by file descriptor */
+} *hdfragl;		/* fragment lists, indexed by file descriptor */
 
-static int	nhdfrags;	/* size of hdfrag array */
+static int	nhdfragls;	/* size of hdfragl array */
 
 
+char *
+hdrealloc(ptr, siz, rout)	/* (re)allocate memory, retry then error */
+char	*ptr;
+unsigned	siz;
+char	*rout;
+{
+	register char	*newp;
+					/* call malloc/realloc */
+	if (ptr == NULL) newp = (char *)malloc(siz);
+	else newp = (char *)realloc(ptr, siz);
+					/* check success */
+	if (newp == NULL && rout != NULL) {
+		hdfreecache(25, NULL);	/* free some memory */
+		errno = 0;		/* retry */
+		newp = hdrealloc(ptr, siz, NULL);
+		if (newp == NULL) {	/* give up and report error */
+			sprintf(errmsg, "out of memory in %s", rout);
+			error(SYSTEM, errmsg);
+		}
+	}
+	return(newp);
+}
+
+
 hdattach(fd)		/* start tracking file fragments for some section */
 register int	fd;
 {
-	if (fd >= nhdfrags) {
-		if (nhdfrags)
-			hdfrag = (struct fragment *)realloc((char *)hdfrag,
-					(fd+1)*sizeof(struct fragment));
-		else
-			hdfrag = (struct fragment *)malloc(
-					(fd+1)*sizeof(struct fragment));
-		if (hdfrag == NULL)
-			error(SYSTEM, "out of memory in hdattach");
-		bzero((char *)(hdfrag+nhdfrags),
-				(fd+1-nhdfrags)*sizeof(struct fragment));
-		nhdfrags = fd+1;
+	if (fd >= nhdfragls) {
+		hdfragl = (struct fraglist *)hdrealloc((char *)hdfragl,
+				(fd+1)*sizeof(struct fraglist), "hdattach");
+		bzero((char *)(hdfragl+nhdfragls),
+				(fd+1-nhdfragls)*sizeof(struct fraglist));
+		nhdfragls = fd+1;
 	}
-	hdfrag[fd].nlinks++;
-	hdfrag[fd].flen = lseek(fd, 0L, 2);	/* get file length */
+	hdfragl[fd].nlinks++;
+	hdfragl[fd].flen = lseek(fd, 0L, 2);	/* get file length */
 }
 
 
@@ -73,36 +106,22 @@ register int	fd;
 hdrelease(fd)		/* stop tracking file fragments for some section */
 register int	fd;
 {
-	if (fd < 0 | fd >= nhdfrags || !hdfrag[fd].nlinks)
+	if (fd < 0 | fd >= nhdfragls || !hdfragl[fd].nlinks)
 		return;
-	if (!--hdfrag[fd].nlinks && hdfrag[fd].nfrags) {
-		free((char *)hdfrag[fd].fi);
-		hdfrag[fd].fi = NULL;
-		hdfrag[fd].nfrags = 0;
+	if (!--hdfragl[fd].nlinks && hdfragl[fd].nfrags) {
+		free((char *)hdfragl[fd].fi);
+		hdfragl[fd].fi = NULL;
+		hdfragl[fd].nfrags = 0;
 	}
 }
 
 
-markdirty(hp)			/* mark holodeck section directory dirty */
-register HOLO	*hp;
-{
-	static BEAMI	smudge = {0, -1};
-
-	if (hp->dirty)		/* already marked? */
-		return;
-	hp->dirty = 1;
-	if (lseek(hp->fd, biglob(hp)->fo+(nbeams(hp)-1)*sizeof(BEAMI), 0) < 0
-			|| write(hp->fd, (char *)&smudge,
-					sizeof(BEAMI)) != sizeof(BEAMI))
-		error(SYSTEM, "seek/write error in markdirty");
-}
-
-
 HOLO *
 hdinit(fd, hproto)	/* initialize a holodeck section in a file */
 int	fd;			/* corresponding file descriptor */
 HDGRID	*hproto;		/* holodeck section grid */
 {
+	long	rtrunc;
 	long	fpos;
 	register HOLO	*hp;
 	register int	n;
@@ -123,8 +142,12 @@ HDGRID	*hproto;		/* holodeck section grid */
 		if (read(fd, (char *)(hp->bi+1), n) != n)
 			error(SYSTEM, "failure loading holodeck directory");
 						/* check that it's clean */
-		if (hp->bi[nbeams(hp)].fo < 0)
-			error(USER, "dirty holodeck section");
+		for (n = nbeams(hp); n > 0; n--)
+			if (hp->bi[n].fo < 0) {
+				hp->bi[n].fo = 0;
+				error(WARNING, "dirty holodeck section");
+				break;
+			}
 	} else {			/* assume we're creating it */
 		if ((hp = hdalloc(hproto)) == NULL)
 			goto memerr;
@@ -138,17 +161,29 @@ HDGRID	*hproto;		/* holodeck section grid */
 	hp->fd = fd;	
 	hp->dirty = 0;
 	biglob(hp)->fo = fpos + sizeof(HDGRID);
-	biglob(hp)->nrd = 0;		/* count rays on disk */
-	for (n = nbeams(hp); n > 0; n--)
-		biglob(hp)->nrd += hp->bi[n].nrd;
+					/* start tracking fragments */
+	hdattach(fd);
+					/* check rays on disk */
+	fpos = hdfilen(fd);
+	biglob(hp)->nrd = rtrunc = 0;
+	for (n = hproto == NULL ? nbeams(hp) : 0; n > 0; n--)
+		if (hp->bi[n].nrd)
+			if (hp->bi[n].fo+hp->bi[n].nrd*sizeof(RAYVAL) > fpos) {
+				rtrunc += hp->bi[n].nrd;
+				hp->bi[n].nrd = 0;
+			} else
+				biglob(hp)->nrd += hp->bi[n].nrd;
+	if (rtrunc) {
+		sprintf(errmsg, "truncated section, %ld rays lost (%.1f%%)",
+				rtrunc, 100.*rtrunc/(rtrunc+biglob(hp)->nrd));
+		error(WARNING, errmsg);
+	}
 					/* add to holodeck list */
 	for (n = 0; n < HDMAX; n++)
 		if (hdlist[n] == NULL) {
 			hdlist[n] = hp;
 			break;
 		}
-					/* start tracking fragments (last) */
-	hdattach(fd);
 					/* all done */
 	return(hp);
 memerr:
@@ -156,32 +191,91 @@ memerr:
 }
 
 
+hdmarkdirty(hp, i)		/* mark holodeck directory position dirty */
+register HOLO	*hp;
+int	i;
+{
+	static BEAMI	smudge = {0, -1};
+	int	mindist, minpos;
+	register int	j;
+
+	if (!hp->dirty++) {			/* write smudge first time */
+		if (lseek(hp->fd, biglob(hp)->fo+(i-1)*sizeof(BEAMI), 0) < 0
+				|| write(hp->fd, (char *)&smudge,
+					sizeof(BEAMI)) != sizeof(BEAMI))
+			error(SYSTEM, "seek/write error in hdmarkdirty");
+		hp->dirseg[0].s = i;
+		hp->dirseg[0].n = 1;
+		return;
+	}
+						/* insert into segment list */
+	for (j = hp->dirty; j--; ) {
+		if (!j || hp->dirseg[j-1].s < i) {
+			hp->dirseg[j].s = i;
+			hp->dirseg[j].n = 1;
+			break;
+		}
+		copystruct(hp->dirseg+j, hp->dirseg+(j-1));
+	}
+	do {				/* check neighbors */
+		mindist = nbeams(hp);		/* find closest */
+		for (j = hp->dirty; --j; )
+			if (hp->dirseg[j].s -
+					(hp->dirseg[j-1].s + hp->dirseg[j-1].n)
+					< mindist) {
+				minpos = j;
+				mindist = hp->dirseg[j].s -
+					(hp->dirseg[j-1].s + hp->dirseg[j-1].n);
+			}
+						/* good enough? */
+		if (hp->dirty <= MAXDIRSE && mindist > MINDIRSEL)
+			break;
+		j = minpos - 1;			/* coalesce neighbors */
+		if (hp->dirseg[j].s + hp->dirseg[j].n <
+				hp->dirseg[minpos].s + hp->dirseg[minpos].n)
+			hp->dirseg[j].n = hp->dirseg[minpos].s +
+					hp->dirseg[minpos].n - hp->dirseg[j].s;
+		hp->dirty--;
+		while (++j < hp->dirty)		/* close the gap */
+			copystruct(hp->dirseg+j, hp->dirseg+(j+1));
+	} while (mindist <= MINDIRSEL);
+}
+
+
 int
-hdsync(hp)			/* update directory on disk if necessary */
+hdsync(hp, all)			/* update beams and directory on disk */
 register HOLO	*hp;
+int	all;
 {
 	register int	j, n;
 
-	if (hp == NULL) {		/* do all */
+	if (hp == NULL) {		/* do all holodecks */
 		n = 0;
 		for (j = 0; hdlist[j] != NULL; j++)
-			n += hdsync(hdlist[j]);
+			n += hdsync(hdlist[j], all);
 		return(n);
 	}
-	if (!hp->dirty)			/* check first */
+					/* sync the beams */
+	for (j = (all ? nbeams(hp) : 0); j > 0; j--)
+		if (hp->bl[j] != NULL)
+			hdsyncbeam(hp, j);
+	if (!hp->dirty)			/* directory clean? */
 		return(0);
-	errno = 0;
-	if (lseek(hp->fd, biglob(hp)->fo, 0) < 0)
-		error(SYSTEM, "cannot seek on holodeck file");
-	n = nbeams(hp)*sizeof(BEAMI);
-	if (write(hp->fd, (char *)(hp->bi+1), n) != n)
-		error(SYSTEM, "cannot update holodeck section directory");
-	hp->dirty = 0;
+	errno = 0;			/* write dirty segments */
+	for (j = 0; j < hp->dirty; j++) {
+		if (lseek(hp->fd, biglob(hp)->fo +
+				(hp->dirseg[j].s-1)*sizeof(BEAMI), 0) < 0)
+			error(SYSTEM, "cannot seek on holodeck file");
+		n = hp->dirseg[j].n * sizeof(BEAMI);
+		if (write(hp->fd, (char *)(hp->bi+hp->dirseg[j].s), n) != n)
+			error(SYSTEM, "cannot update section directory");
+	}
+	hp->dirty = 0;			/* all clean */
 	return(1);
 }
 
 
-long
+unsigned
 hdmemuse(all)		/* return memory usage (in bytes) */
 int	all;			/* include overhead (painful) */
 {
@@ -200,11 +294,11 @@ int	all;			/* include overhead (painful) */
 		}
 	}
 	if (all)
-		for (j = 0; j < nhdfrags; j++) {
-			total += sizeof(struct fragment);
-			if (hdfrag[j].nfrags)
+		for (j = 0; j < nhdfragls; j++) {
+			total += sizeof(struct fraglist);
+			if (hdfragl[j].nfrags)
 				total += FRAGBLK*sizeof(BEAMI) *
-					((hdfrag[j].nfrags-1)/FRAGBLK + 1) ;
+					((hdfragl[j].nfrags-1)/FRAGBLK + 1) ;
 		}
 	return(total);
 }
@@ -218,14 +312,14 @@ int	fd;
 
 	if (fd < 0)
 		return(-1);
-	if (fd >= nhdfrags || !hdfrag[fd].nlinks) {
+	if (fd >= nhdfragls || !hdfragl[fd].nlinks) {
 		if ((fpos = lseek(fd, 0L, 1)) < 0)
 			return(-1);
 		flen = lseek(fd, 0L, 2);
 		lseek(fd, fpos, 0);
 		return(flen);
 	}
-	return(hdfrag[fd].flen);
+	return(hdfragl[fd].flen);
 }
 
 
@@ -264,21 +358,19 @@ int	nr;			/* number of new rays desired */
 	RAYVAL	*p;
 	int	n;
 
-	if (nr <= 0)
-		return(NULL);
-	if (i < 1 | i > nbeams(hp))
-		error(CONSISTENCY, "bad beam index given to hdnewrays");
+	if (nr <= 0) return(NULL);
+	CHECK(i < 1 | i > nbeams(hp),
+			CONSISTENCY, "bad beam index given to hdnewrays");
 	if (hp->bl[i] != NULL)
 		hp->bl[i]->tick = hdclock;	/* preempt swap */
 	if (hdcachesize > 0 && hdmemuse(0) >= hdcachesize)
 		hdfreecache(PCTFREE, NULL);	/* free some space */
-	errno = 0;
 	if (hp->bl[i] == NULL) {		/* allocate (and load) */
 		n = hp->bi[i].nrd + nr;
-		if ((hp->bl[i] = (BEAM *)malloc(hdbsiz(n))) == NULL)
-			goto memerr;
+		hp->bl[i] = (BEAM *)hdrealloc(NULL, hdbsiz(n), "hdnewrays");
 		blglob(hp)->nrm += n;
-		if (n = hp->bl[i]->nrm = hp->bi[i].nrd) {
+		if ((n = hp->bl[i]->nrm = hp->bi[i].nrd)) {
+			errno = 0;
 			if (lseek(hp->fd, hp->bi[i].fo, 0) < 0)
 				error(SYSTEM, "seek error on holodeck file");
 			n *= sizeof(RAYVAL);
@@ -287,20 +379,17 @@ int	nr;			/* number of new rays desired */
 				"error reading beam from holodeck file");
 		}
 	} else {				/* just grow in memory */
-		hp->bl[i] = (BEAM *)realloc( (char *)hp->bl[i],
-				hdbsiz(hp->bl[i]->nrm + nr) );
-		if (hp->bl[i] == NULL)
-			goto memerr;
+		hp->bl[i] = (BEAM *)hdrealloc((char *)hp->bl[i],
+				hdbsiz(hp->bl[i]->nrm + nr), "hdnewrays");
 		blglob(hp)->nrm += nr;
 	}
+	if (hdfragflags&FF_ALLOC && hp->bi[i].nrd)
+		hdfreefrag(hp, i);		/* relinquish old fragment */
 	p = hdbray(hp->bl[i]) + hp->bl[i]->nrm;
 	hp->bl[i]->nrm += nr;			/* update in-core structure */
 	bzero((char *)p, nr*sizeof(RAYVAL));
-	hp->bl[i]->tick = ++hdclock;		/* update LRU clock */
-	blglob(hp)->tick = hdclock;
+	blglob(hp)->tick = hp->bl[i]->tick = hdclock++;	/* update LRU clock */
 	return(p);				/* point to new rays */
-memerr:
-	error(SYSTEM, "out of memory in hdnewrays");
 }
 
 
@@ -311,129 +400,260 @@ register int	i;
 {
 	register int	n;
 
-	if (i < 1 | i > nbeams(hp))
-		error(CONSISTENCY, "bad beam index given to hdgetbeam");
+	CHECK(i < 1 | i > nbeams(hp),
+			CONSISTENCY, "bad beam index given to hdgetbeam");
 	if (hp->bl[i] == NULL) {		/* load from disk */
 		if (!(n = hp->bi[i].nrd))
 			return(NULL);
 		if (hdcachesize > 0 && hdmemuse(0) >= hdcachesize)
 			hdfreecache(PCTFREE, NULL);	/* get free space */
-		errno = 0;
-		if ((hp->bl[i] = (BEAM *)malloc(hdbsiz(n))) == NULL)
-			error(SYSTEM, "cannot allocate memory for beam");
+		hp->bl[i] = (BEAM *)hdrealloc(NULL, hdbsiz(n), "hdgetbeam");
 		blglob(hp)->nrm += hp->bl[i]->nrm = n;
+		errno = 0;
 		if (lseek(hp->fd, hp->bi[i].fo, 0) < 0)
 			error(SYSTEM, "seek error on holodeck file");
 		n *= sizeof(RAYVAL);
 		if (read(hp->fd, (char *)hdbray(hp->bl[i]), n) != n)
 			error(SYSTEM, "error reading beam from holodeck file");
+		if (hdfragflags&FF_READ)
+			hdfreefrag(hp, i);	/* relinquish old frag. */
 	}
-	hp->bl[i]->tick = ++hdclock;	/* update LRU clock */
-	blglob(hp)->tick = hdclock;
+	blglob(hp)->tick = hp->bl[i]->tick = hdclock++;	/* update LRU clock */
 	return(hp->bl[i]);
 }
 
 
 int
-hdsyncbi(hp, i)			/* sync beam in memory with beam on disk */
-register HOLO	*hp;
-register int	i;
+hdfilord(hb1, hb2)	/* order beams for quick loading */
+register HDBEAMI	*hb1, *hb2;
 {
-	unsigned int	nrays = hp->bl[i]->nrm;
-	long	nfo;
-	unsigned int	n;
-					/* is current fragment OK? */
-	if (nrays == hp->bi[i].nrd)
-		return(0);
-					/* check file status */
-	if (hp->dirty < 0)
+	register long	c;
+				/* residents go first */
+	if (hb2->h->bl[hb2->b] != NULL)
+		return(hb1->h->bl[hb1->b] == NULL);
+	if (hb1->h->bl[hb1->b] != NULL)
 		return(-1);
+				/* otherwise sort by file descriptor */
+	if ((c = hb1->h->fd - hb2->h->fd))
+		return(c);
+				/* then by position in file */
+	c = hb1->h->bi[hb1->b].fo - hb2->h->bi[hb2->b].fo;
+	return(c > 0 ? 1 : c < 0 ? -1 : 0);
+}
 
-	if (hp->fd >= nhdfrags || !hdfrag[hp->fd].nlinks) /* untracked */
-		hp->bi[i].fo = lseek(hp->fd, 0L, 2);
 
-	else if (hp->bi[i].fo + hp->bi[i].nrd*sizeof(RAYVAL) ==
-			hdfrag[hp->fd].flen)		/* EOF special case */
-		hdfrag[hp->fd].flen = (nfo=hp->bi[i].fo) + nrays*sizeof(RAYVAL);
+hdloadbeams(hb, n, bf)	/* load a list of beams in optimal order */
+register HDBEAMI	*hb;	/* list gets sorted by hdfilord() */
+int	n;			/* list length */
+int	(*bf)();		/* callback function (optional) */
+{
+	unsigned	origcachesize, memuse;
+	int	bytesloaded, needbytes, bytes2free;
+	register BEAM	*bp;
+	register int	i;
+					/* precheck consistency */
+	if (n <= 0) return;
+	for (i = n; i--; )
+		if (hb[i].h==NULL || hb[i].b<1 | hb[i].b>nbeams(hb[i].h))
+			error(CONSISTENCY, "bad beam in hdloadbeams");
+					/* sort list for optimal access */
+	qsort((char *)hb, n, sizeof(HDBEAMI), hdfilord);
+	bytesloaded = 0;		/* run through loaded beams */
+	for ( ; n && (bp = hb->h->bl[hb->b]) != NULL; n--, hb++) {
+		bp->tick = hdclock;	/* preempt swap */
+		bytesloaded += bp->nrm;
+		if (bf != NULL)
+			(*bf)(bp, hb);
+	}
+	bytesloaded *= sizeof(RAYVAL);
+	if ((origcachesize = hdcachesize) > 0) {
+		needbytes = 0;		/* figure out memory needs */
+		for (i = n; i--; )
+			needbytes += hb[i].h->bi[hb[i].b].nrd;
+		needbytes *= sizeof(RAYVAL);
+		do {				/* free enough memory */
+			memuse = hdmemuse(0);
+			bytes2free = needbytes - (int)(hdcachesize-memuse);
+			if (bytes2free > (int)(memuse - bytesloaded))
+				bytes2free = memuse - bytesloaded;
+		} while (bytes2free > 0 &&
+				hdfreecache(100*bytes2free/memuse, NULL) < 0);
+		hdcachesize = 0;		/* load beams w/o swap */
+	}
+	for (i = 0; i < n; i++)
+		if ((bp = hdgetbeam(hb[i].h, hb[i].b)) != NULL && bf != NULL)
+			(*bf)(bp, hb+i);
+	hdcachesize = origcachesize;	/* resume dynamic swapping */
+}
 
-	else {						/* general case */
-		register struct fragment	*f = &hdfrag[hp->fd];
-		register int	j, k;
-					/* relinquish old fragment */
-		if (hp->bi[i].nrd) {
-			j = f->nfrags++;
-#ifdef MAXFRAG
-			if (j >= MAXFRAG-1)
-				f->nfrags--;
+
+int
+hdfreefrag(hp, i)			/* free a file fragment */
+HOLO	*hp;
+int	i;
+{
+	register BEAMI	*bi = &hp->bi[i];
+	register struct fraglist	*f;
+	register int	j, k;
+
+	if (bi->nrd <= 0)
+		return(0);
+	DCHECK(hp->fd < 0 | hp->fd >= nhdfragls || !hdfragl[hp->fd].nlinks,
+			CONSISTENCY, "bad file descriptor in hdfreefrag");
+	f = &hdfragl[hp->fd];
+	if (f->nfrags % FRAGBLK == 0) {	/* delete empty remnants */
+		for (j = k = 0; k < f->nfrags; j++, k++) {
+			while (f->fi[k].nrd == 0)
+				if (++k >= f->nfrags)
+					goto endloop;
+			if (k > j)
+				copystruct(f->fi+j, f->fi+k);
+		}
+	endloop:
+		f->nfrags = j;
+	}
+	j = f->nfrags++;		/* allocate a slot in free list */
+#if MAXFRAGB
+	if (j >= MAXFRAGB*FRAGBLK) {
+		f->nfrags = j--;	/* stop list growth */
+		if (bi->nrd <= f->fi[j].nrd)
+			return(0);	/* new one no better than discard */
+	}
 #endif
-			if (j % FRAGBLK == 0) {		/* more frag. space */
-				if (f->fi == NULL)
-					f->fi = (BEAMI *)malloc(
-							FRAGBLK*sizeof(BEAMI));
-				else
-					f->fi = (BEAMI *)realloc((char *)f->fi,
-						(j+FRAGBLK)*sizeof(BEAMI));
-				if (f->fi == NULL)
-					error(SYSTEM,
-						"out of memory in hdsyncbi");
-			}
-			for ( ; ; j--) {	/* insert in descending list */
-				if (!j || hp->bi[i].fo < f->fi[j-1].fo) {
-					f->fi[j].fo = hp->bi[i].fo;
-					f->fi[j].nrd = hp->bi[i].nrd;
-					break;
-				}
-				copystruct(f->fi+j, f->fi+j-1);
-			}
+	if (j % FRAGBLK == 0) {		/* more (or less) free list space */
+		register BEAMI	*newp;
+		if (f->fi == NULL)
+			newp = (BEAMI *)malloc((j+FRAGBLK)*sizeof(BEAMI));
+		else
+			newp = (BEAMI *)realloc((char *)f->fi,
+					(j+FRAGBLK)*sizeof(BEAMI));
+		if (newp == NULL) {
+			f->nfrags--;	/* graceful failure */
+			return(0);
+		}
+		f->fi = newp;
+	}
+	for ( ; ; j--) {		/* insert in descending list */
+		if (!j || bi->fo < f->fi[j-1].fo) {
+			f->fi[j].fo = bi->fo;
+			f->fi[j].nrd = bi->nrd;
+			break;
+		}
+		copystruct(f->fi+j, f->fi+(j-1));
+	}
 					/* coalesce adjacent fragments */
-			for (j = k = 0; k < f->nfrags; j++, k++) {
-				if (k > j)
-					copystruct(f->fi+j, f->fi+k);
-				while (k+1 < f->nfrags && f->fi[k+1].fo +
-						f->fi[k+1].nrd*sizeof(RAYVAL)
-							== f->fi[j].fo) {
-					f->fi[j].fo -=
-						f->fi[++k].nrd*sizeof(RAYVAL);
-					f->fi[j].nrd += f->fi[k].nrd;
-				}
+						/* successors never empty */
+	if (j && f->fi[j-1].fo == f->fi[j].fo + f->fi[j].nrd*sizeof(RAYVAL)) {
+		f->fi[j].nrd += f->fi[j-1].nrd;
+		f->fi[j-1].nrd = 0;
+	}
+	for (k = j+1; k < f->nfrags; k++)	/* get non-empty predecessor */
+		if (f->fi[k].nrd) {
+			if (f->fi[j].fo == f->fi[k].fo +
+					f->fi[k].nrd*sizeof(RAYVAL)) {
+				f->fi[k].nrd += f->fi[j].nrd;
+				f->fi[j].nrd = 0;
 			}
-			f->nfrags = j;
+			break;
 		}
-		k = -1;			/* find closest-sized fragment */
-		for (j = nrays ? f->nfrags : 0; j-- > 0; )
-			if (f->fi[j].nrd >= nrays &&
-					(k < 0 || f->fi[j].nrd < f->fi[k].nrd))
-				if (f->fi[k=j].nrd == nrays)
-					break;
-		if (k < 0) {		/* no fragment -- extend file */
-			nfo = f->flen;
-			f->flen += nrays*sizeof(RAYVAL);
-		} else {		/* else use fragment */
-			nfo = f->fi[k].fo;
-			if (f->fi[k].nrd == nrays) {	/* delete fragment */
-				f->nfrags--;
-				for (j = k; j < f->nfrags; j++)
-					copystruct(f->fi+j, f->fi+j+1);
-			} else {			/* else shrink it */
-				f->fi[k].fo += nrays*sizeof(RAYVAL);
-				f->fi[k].nrd -= nrays;
-			}
-		}
+	biglob(hp)->nrd -= bi->nrd;		/* tell fragment it's free */
+	bi->nrd = 0;
+	bi->fo = 0L;
+	hdmarkdirty(hp, i);			/* assume we'll reallocate */
+	return(1);
+}
+
+
+int
+hdfragOK(fd, listlen, listsiz)	/* get fragment list status for file */
+int	fd;
+int	*listlen;
+register int4	*listsiz;
+{
+	register struct fraglist	*f;
+	register int	i;
+
+	if (fd < 0 | fd >= nhdfragls || !(f = &hdfragl[fd])->nlinks)
+		return(0);		/* listless */
+	if (listlen != NULL)
+		*listlen = f->nfrags;
+	if (listsiz != NULL)
+		for (i = f->nfrags, *listsiz = 0; i--; )
+			*listsiz += f->fi[i].nrd;
+#if MAXFRAGB
+	return(f->nfrags < MAXFRAGB*FRAGBLK);
+#else
+	return(1);			/* list never fills up */
+#endif
+}
+
+
+long
+hdallocfrag(fd, nrays)		/* allocate a file fragment */
+int	fd;
+unsigned int4	nrays;
+{
+	register struct fraglist	*f;
+	register int	j;
+	long	nfo;
+
+	if (nrays == 0)
+		return(-1L);
+	DCHECK(fd < 0 | fd >= nhdfragls || !hdfragl[fd].nlinks,
+			CONSISTENCY, "bad file descriptor in hdallocfrag");
+	f = &hdfragl[fd];
+	for (j = f->nfrags; j-- > 0; )	/* first fit algorithm */
+		if (f->fi[j].nrd >= nrays)
+			break;
+	if (j < 0) {			/* no fragment -- extend file */
+		nfo = f->flen;
+		f->flen += nrays*sizeof(RAYVAL);
+	} else {			/* else use fragment */
+		nfo = f->fi[j].fo;
+		f->fi[j].fo += nrays*sizeof(RAYVAL);
+		f->fi[j].nrd -= nrays;
 	}
-	if (nrays) {		/* write the new fragment */
+	return(nfo);
+}
+
+
+int
+hdsyncbeam(hp, i)		/* sync beam in memory with beam on disk */
+register HOLO	*hp;
+register int	i;
+{
+	int	fragfreed;
+	unsigned int4	nrays;
+	unsigned int	n;
+	long	nfo;
+					/* check file status */
+	if (hdfragl[hp->fd].writerr)
+		return(-1);
+	DCHECK(i < 1 | i > nbeams(hp),
+			CONSISTENCY, "bad beam index in hdsyncbeam");
+					/* is current fragment OK? */
+	if (hp->bl[i] == NULL || (nrays = hp->bl[i]->nrm) == hp->bi[i].nrd)
+		return(0);
+					/* relinquish old fragment? */
+	fragfreed = hdfragflags&FF_WRITE && hp->bi[i].nrd && hdfreefrag(hp,i);
+	if (nrays) {			/* get and write new fragment */
+		nfo = hdallocfrag(hp->fd, nrays);
 		errno = 0;
 		if (lseek(hp->fd, nfo, 0) < 0)
 			error(SYSTEM, "cannot seek on holodeck file");
 		n = hp->bl[i]->nrm * sizeof(RAYVAL);
 		if (write(hp->fd, (char *)hdbray(hp->bl[i]), n) != n) {
-			hp->dirty = -1;		/* avoid recursive error */
-			error(SYSTEM, "write error in hdsyncbi");
+			hdfragl[hp->fd].writerr++;
+			hdsync(NULL, 0);	/* sync directories */
+			error(SYSTEM, "write error in hdsyncbeam");
 		}
-	}
+		hp->bi[i].fo = nfo;
+	} else
+		hp->bi[i].fo = 0L;
 	biglob(hp)->nrd += nrays - hp->bi[i].nrd;
 	hp->bi[i].nrd = nrays;
-	hp->bi[i].fo = nfo;
-	markdirty(hp);		/* section directory now out of date */
+	if (!fragfreed)
+		hdmarkdirty(hp, i);		/* need to flag dir. ent. */
 	return(1);
 }
 
@@ -451,21 +671,27 @@ register int	i;
 			nchanged += hdfreebeam(hdlist[i], 0);
 		return(nchanged);
 	}
+	if (hdfragl[hp->fd].writerr)	/* check for file error */
+		return(0);
 	if (i == 0) {			/* clear entire holodeck */
+		if (blglob(hp)->nrm == 0)
+			return(0);		/* already clear */
 		nchanged = 0;
 		for (i = nbeams(hp); i > 0; i--)
 			if (hp->bl[i] != NULL)
 				nchanged += hdfreebeam(hp, i);
+		DCHECK(blglob(hp)->nrm != 0,
+				CONSISTENCY, "bad beam count in hdfreebeam");
 		return(nchanged);
 	}
-	if (i < 1 | i > nbeams(hp))
-		error(CONSISTENCY, "bad beam index to hdfreebeam");
+	DCHECK(i < 1 | i > nbeams(hp),
+			CONSISTENCY, "bad beam index to hdfreebeam");
 	if (hp->bl[i] == NULL)
 		return(0);
 					/* check for additions */
 	nchanged = hp->bl[i]->nrm - hp->bi[i].nrd;
 	if (nchanged)
-		hdsyncbi(hp, i);			/* write new fragment */
+		hdsyncbeam(hp, i);		/* write new fragment */
 	blglob(hp)->nrm -= hp->bl[i]->nrm;
 	free((char *)hp->bl[i]);		/* free memory */
 	hp->bl[i] = NULL;
@@ -478,7 +704,6 @@ hdkillbeam(hp, i)		/* delete beam from holodeck */
 register HOLO	*hp;
 register int	i;
 {
-	static BEAM	emptybeam;
 	int	nchanged;
 
 	if (hp == NULL) {		/* clobber all holodecks */
@@ -488,93 +713,108 @@ register int	i;
 		return(nchanged);
 	}
 	if (i == 0) {			/* clobber entire holodeck */
+		if (biglob(hp)->nrd == 0 & blglob(hp)->nrm == 0)
+			return(0);		/* already empty */
 		nchanged = 0;
+		nchanged = 0;
 		for (i = nbeams(hp); i > 0; i--)
 			if (hp->bi[i].nrd > 0 || hp->bl[i] != NULL)
 				nchanged += hdkillbeam(hp, i);
-#ifdef DEBUG
-		if (biglob(hp)->nrd != 0 | blglob(hp)->nrm != 0)
-			error(CONSISTENCY, "bad beam count in hdkillbeam");
-#endif
+		DCHECK(biglob(hp)->nrd != 0 | blglob(hp)->nrm != 0,
+				CONSISTENCY, "bad beam count in hdkillbeam");
 		return(nchanged);
 	}
-	if (i < 1 | i > nbeams(hp))
-		error(CONSISTENCY, "bad beam index to hdkillbeam");
+	DCHECK(i < 1 | i > nbeams(hp),
+			CONSISTENCY, "bad beam index to hdkillbeam");
 	if (hp->bl[i] != NULL) {	/* free memory */
 		blglob(hp)->nrm -= nchanged = hp->bl[i]->nrm;
 		free((char *)hp->bl[i]);
+		hp->bl[i] = NULL;
 	} else
 		nchanged = hp->bi[i].nrd;
-	if (hp->bi[i].nrd) {		/* free file fragment */
-		hp->bl[i] = &emptybeam;
-		hdsyncbi(hp, i);
+	if (hp->bi[i].nrd && !(hdfragflags&FF_KILL && hdfreefrag(hp,i))) {
+		biglob(hp)->nrd -= hp->bi[i].nrd;	/* free failed */
+		hp->bi[i].nrd = 0;
+		hp->bi[i].fo = 0L;
+		hdmarkdirty(hp, i);
 	}
-	hp->bl[i] = NULL;
 	return(nchanged);
 }
 
 
-hdlrulist(ha, ba, n, hp)	/* add beams from holodeck to LRU list */
-register HOLO	*ha[];			/* section list (NULL terminated) */
-register int	ba[];			/* beam index to go with section */
-int	n;				/* length of arrays minus 1 */
+int
+hdlrulist(hb, nents, n, hp)	/* add beams from holodeck to LRU list */
+register HDBEAMI	*hb;		/* beam list */
+int	nents;				/* current list length */
+int	n;				/* maximum list length */
 register HOLO	*hp;			/* section we're adding from */
 {
 	register int	i, j;
-	int	nents;
-					/* find last entry in LRU list */
-	for (j = 0; ha[j] != NULL; j++)
-		;
-	nents = j;
 					/* insert each beam from hp */
-	for (i = nbeams(hp); i > 0; i--) {
+	for (i = 1; i <= nbeams(hp); i++) {
 		if (hp->bl[i] == NULL)		/* check if loaded */
 			continue;
-		if ((j = ++nents) > n)		/* grow list if we can */
+#if 0
+		if (hp->bl[i]->tick == hdclock)	/* preempt swap? */
+			continue;
+#endif
+		if ((j = ++nents) >= n)		/* grow list if we can */
 			nents--;
 		for ( ; ; ) {			/* bubble into place */
 			if (!--j || hp->bl[i]->tick >=
-					ha[j-1]->bl[ba[j-1]]->tick) {
-				ha[j] = hp;
-				ba[j] = i;
+					hb[j-1].h->bl[hb[j-1].b]->tick) {
+				hb[j].h = hp;
+				hb[j].b = i;
 				break;
 			}
-			ha[j] = ha[j-1];
-			ba[j] = ba[j-1];
+			copystruct(hb+j, hb+(j-1));
 		}
 	}
-	ha[nents] = NULL;		/* all done */
-	ba[nents] = 0;
+	return(nents);			/* return new list length */
 }
 
 
+int
 hdfreecache(pct, honly)		/* free up cache space, writing changes */
 int	pct;				/* maximum percentage to free */
 register HOLO	*honly;			/* NULL means check all */
 {
-	HOLO	*hp[FREEBEAMS+1];
-	int	bn[FREEBEAMS+1];
+	HDBEAMI	hb[FREEBEAMS];
 	int	freetarget;
+	int	n;
 	register int	i;
+#ifdef DEBUG
+	unsigned	membefore;
+
+	membefore = hdmemuse(0);
+#endif
 						/* compute free target */
 	freetarget = (honly != NULL) ? blglob(honly)->nrm :
 			hdmemuse(0)/sizeof(RAYVAL) ;
 	freetarget = freetarget*pct/100;
+	if (freetarget <= 0)
+		return(0);
 						/* find least recently used */
-	hp[0] = NULL;
-	bn[0] = 0;
+	n = 0;
 	if (honly != NULL)
-		hdlrulist(hp, bn, FREEBEAMS, honly);
+		n = hdlrulist(hb, n, FREEBEAMS, honly);
 	else
 		for (i = 0; hdlist[i] != NULL; i++)
-			hdlrulist(hp, bn, FREEBEAMS, hdlist[i]);
+			n = hdlrulist(hb, n, FREEBEAMS, hdlist[i]);
 						/* free LRU beams */
-	for (i = 0; hp[i] != NULL; i++) {
-		hdfreebeam(hp[i], bn[i]);
-		if ((freetarget -= hp[i]->bi[bn[i]].nrd) <= 0)
+	for (i = 0; i < n; i++) {
+		hdfreebeam(hb[i].h, hb[i].b);
+		if ((freetarget -= hb[i].h->bi[hb[i].b].nrd) <= 0)
 			break;
 	}
-	hdsync(honly);		/* synchronize directories as necessary */
+	hdsync(honly, 0);	/* synchronize directories as necessary */
+#ifdef DEBUG
+	sprintf(errmsg,
+	"%dK before, %dK after hdfreecache (%dK total), %d rays short\n",
+		membefore>>10, hdmemuse(0)>>10, hdmemuse(1)>>10, freetarget);
+	wputs(errmsg);
+#endif
+	return(-freetarget);	/* return how far past goal we went */
 }
 
 
@@ -586,6 +826,8 @@ register HOLO	*hp;		/* NULL means clean up all */
 	if (hp == NULL) {		/* NULL means clean up everything */
 		while (hdlist[0] != NULL)
 			hddone(hdlist[0]);
+		free((char *)hdfragl);
+		hdfragl = NULL; nhdfragls = 0;
 		return;
 	}
 					/* flush all data and free memory */