[gs-cvs] rev 8660 - branches/mtrender/src
ray at ghostscript.com
ray at ghostscript.com
Thu Apr 24 12:24:09 PDT 2008
Author: ray
Date: 2008-04-24 12:24:08 -0700 (Thu, 24 Apr 2008)
New Revision: 8660
Modified:
branches/mtrender/src/gxclmem.c
branches/mtrender/src/gxclmem.h
Log:
Add support for opening a 'memfile' from multiple threads for reading. This
requires copying the memfile header and the 'logical block' chain. The
'physical blocks' that comrise the bulk of the allocation are shared and
are 'read only'. This change allows the mult-threaded clist rendering to
work with BAND_LIST_STORAGE=memory.
Also change the default algorithm for the number of decompression buffers
(raw blocks) to limit to 64. The threshold for compression is increased
to a fairly large value, suitable for host based machines. Embedded clients
should use a 'smarter' method that starts compression depending on the
current memory available, dynamically, rather than a static compile time
approach.
Modified: branches/mtrender/src/gxclmem.c
===================================================================
--- branches/mtrender/src/gxclmem.c 2008-04-23 14:06:11 UTC (rev 8659)
+++ branches/mtrender/src/gxclmem.c 2008-04-24 19:24:08 UTC (rev 8660)
@@ -131,14 +131,14 @@
least as large as the fixed overhead of the compressor plus the
decompressor, plus the expected compressed size of a block that size.
*/
-static const long COMPRESSION_THRESHOLD = 32000000;
+static const long COMPRESSION_THRESHOLD = 500000000; /* 0.5 Gb for host machines */
#define NEED_TO_COMPRESS(f)\
((f)->ok_to_compress && (f)->total_space > COMPRESSION_THRESHOLD)
- /* FOR NOW ALLOCATE 1 raw buffer for every 32 blocks (at least 8) */
-#define GET_NUM_RAW_BUFFERS( f ) \
- max(f->log_length/MEMFILE_DATA_SIZE/32, 8)
+ /* FOR NOW ALLOCATE 1 raw buffer for every 32 blocks (at least 8, no more than 64) */
+#define GET_NUM_RAW_BUFFERS( f ) \
+ min(64, max(f->log_length/MEMFILE_DATA_SIZE/32, 8))
#define MALLOC(f, siz, cname)\
(void *)gs_alloc_bytes((f)->data_memory, siz, cname)
@@ -154,6 +154,7 @@
static int memfile_init_empty(MEMFILE * f);
static int memfile_set_memory_warning(clist_file_ptr cf, int bytes_left);
static int memfile_fclose(clist_file_ptr cf, const char *fname, bool delete);
+static int memfile_get_pdata(MEMFILE * f);
/************************************************/
/* #define DEBUG /- force statistics -/ */
@@ -225,20 +226,90 @@
clist_file_ptr /*MEMFILE * */ * pf,
gs_memory_t *mem, gs_memory_t *data_mem, bool ok_to_compress)
{
- MEMFILE *f = 0;
+ MEMFILE *f = NULL;
int code = 0;
- /* We don't implement reopening an existing file. */
- if (fname[0] != 0 || fmode[0] != 'w') {
- code = gs_note_error(gs_error_invalidfileaccess);
+ *pf = NULL; /* in case we have an error */
+
+ /* fname[0] == 0 if this is not reopening */
+ /* memfile file names begin with a flag byte == 0xff */
+ if (fname[0] == 0xff || fmode[0] == 'r') {
+ MEMFILE *base_f = NULL;
+
+ /* reopening an existing file. */
+ code = sscanf(fname+1, "0x%x", &base_f);
+ if (code != 1)
+ gs_note_error(gs_error_ioerror);
+
+ /* We need to 'clone' this memfile so that each reader instance */
+ /* will be able to maintain it's own 'state' */
+ f = gs_alloc_struct(mem, MEMFILE, &st_MEMFILE,
+ "memfile_fopen_instance(MEMFILE)");
+ if (f == NULL) {
+ eprintf1("memfile_open_scratch(%s): gs_alloc_struct failed\n", fname);
+ code = gs_note_error(gs_error_VMerror);
+ goto finish;
+ }
+ memcpy(f, base_f, sizeof(MEMFILE));
+ f->memory = mem;
+ f->data_memory = data_mem;
+ f->compress_state = 0; /* Not used by reader instance */
+ f->decompress_state = 0; /* make clean for GC, or alloc'n failure */
+ f->reservePhysBlockChain = NULL;
+ f->reservePhysBlockCount = 0;
+ f->reserveLogBlockChain = NULL;
+ f->reserveLogBlockCount = 0;
+ f->openlist = base_f->openlist;
+ base_f->openlist = f; /* link this one in to the base memfile */
+ f->base_memfile = base_f;
+ f->log_curr_pos = 0;
+ f->raw_head = NULL;
+ f->error_code = 0;
+
+ if (f->log_head->phys_blk->data_limit != NULL) {
+ /* The file is compressed, so we need to copy the logical block */
+ /* list so that it is unique to this instance, and initialize */
+ /* the decompressor. */
+ LOG_MEMFILE_BLK *log_block, *new_log_block;
+ int i;
+ int num_log_blocks = (f->log_length + MEMFILE_DATA_SIZE - 1) / MEMFILE_DATA_SIZE;
+ const stream_state *decompress_proto = clist_decompressor_state(NULL);
+ const stream_template *decompress_template = decompress_proto->template;
+
+ new_log_block = MALLOC(f, num_log_blocks * sizeof(LOG_MEMFILE_BLK), "memfile_fopen" );
+ if (new_log_block == NULL)
+ code = gs_note_error(gs_error_VMerror);
+
+ /* copy the logical blocks to the new list just allocated */
+ for (log_block=f->log_head, i=0; log_block != NULL; log_block=log_block->link, i++) {
+ new_log_block[i].phys_blk = log_block->phys_blk;
+ new_log_block[i].phys_pdata = log_block->phys_pdata;
+ new_log_block[i].raw_block = NULL;
+ new_log_block[i].link = log_block->link == NULL ? NULL : new_log_block + i + 1;
+ }
+ f->log_head = new_log_block;
+
+ /* NB: don't need compress_state for reading */
+ f->decompress_state =
+ gs_alloc_struct(mem, stream_state, decompress_template->stype,
+ "memfile_open_scratch(decompress_state)");
+ if (f->decompress_state == 0) {
+ eprintf1("memfile_open_scratch(%s): gs_alloc_struct failed\n", fname);
+ code = gs_note_error(gs_error_VMerror);
+ goto finish;
+ }
+ memcpy(f->decompress_state, decompress_proto,
+ gs_struct_type_size(decompress_template->stype));
+ f->decompress_state->memory = mem;
+ if (decompress_template->set_defaults)
+ (*decompress_template->set_defaults) (f->decompress_state);
+ }
+ f->log_curr_blk = f->log_head;
+ memfile_get_pdata(f); /* set up the initial block */
+
goto finish;
}
-
- /* There is no need to set fname in this implementation, */
- /* but we do it anyway. */
- fname[0] = (ok_to_compress ? 'a' : 'b');
- fname[1] = 0;
-
+ fname[0] = 0; /* no file name yet */
f = gs_alloc_struct(mem, MEMFILE, &st_MEMFILE,
"memfile_open_scratch(MEMFILE)");
if (f == NULL) {
@@ -251,6 +322,8 @@
/* init an empty file, BEFORE allocating de/compress state */
f->compress_state = 0; /* make clean for GC, or alloc'n failure */
f->decompress_state = 0;
+ f->openlist = NULL;
+ f->base_memfile = NULL;
f->total_space = 0;
f->reservePhysBlockChain = NULL;
f->reservePhysBlockCount = 0;
@@ -298,16 +371,16 @@
}
f->total_space = 0;
+ /* Return the address of this memfile as a string for use in future clist_fopen calls */
+ fname[0] = 0xff; /* a flag that this is a memfile name */
+ sprintf(fname+1, "0x%0x", f);
+
#ifdef DEBUG
- /* If this is the start, init some statistics. */
- /* Hack: we know the 'a' file is opened first. */
- if (*fname == 'a') {
tot_compressed = 0;
tot_raw = 0;
tot_cache_miss = 0;
tot_cache_hits = 0;
tot_swap_out = 0;
- }
#endif
finish:
if (code < 0) {
@@ -326,9 +399,41 @@
{
MEMFILE *const f = (MEMFILE *)cf;
- /* We don't implement closing without deletion. */
- if (!delete)
- return_error(gs_error_invalidfileaccess);
+ if (!delete) {
+ if (f->base_memfile) {
+ MEMFILE *prev_f;
+
+ /* Here we need to delete this instance from the 'openlist' */
+ /* in case this file was opened for 'read' on a previously */
+ /* written file (base_memfile != NULL) */
+ for (prev_f = f->base_memfile; prev_f != NULL; prev_f = prev_f->openlist)
+ if (prev_f->openlist == f)
+ break;
+ if (prev_f == NULL) {
+ eprintf1("Could not find 0x%x on memfile openlist\n", f);
+ return_error(gs_error_invalidfileaccess);
+ }
+ prev_f->openlist = f->openlist; /* link around the one being fclosed */
+ /* Now delete this MEMFILE reader instance */
+ /* NB: we don't delete 'base' instances until we delete */
+ /* deallocate de/compress state */
+ gs_free_object(f->memory, f->decompress_state,
+ "memfile_close_and_unlink(decompress_state)");
+ gs_free_object(f->memory, f->compress_state,
+ "memfile_close_and_unlink(compress_state)");
+ /* deallocate the memfile object proper */
+ gs_free_object(f->memory, f, "memfile_close_and_unlink(MEMFILE)");
+ }
+ return 0;
+ }
+
+ /* If there are open read memfile structures, set them so that */
+ /* future accesses will return errors rather than causing SEGV */
+ if (f->openlist) {
+ /* TODO: do the cleanup rather than just giving an error */
+ eprintf1("Attempt to delete a memfile still open for read: 0x%0x\n", f);
+ return_error(gs_error_invalidfileaccess);
+ }
memfile_free_mem(f);
/* Free reserve blocks; don't do it in memfile_free_mem because */
@@ -360,11 +465,14 @@
static int
memfile_unlink(const char *fname)
{
- /*
- * Since we have no way to represent a memfile other than by the
- * pointer, we don't (can't) implement unlinking.
- */
- return_error(gs_error_invalidfileaccess);
+ int code;
+ MEMFILE *f;
+
+ /* memfile file names begin with a flag byte == 0xff */
+ if (fname[0] == 0xff && (code = sscanf(fname+1, "0x%x", &f) == 1)) {
+ return memfile_fclose((clist_file_ptr)f, fname, true);
+ } else
+ return_error(gs_error_invalidfileaccess);
}
/* ---------------- Writing ---------------- */
@@ -546,11 +654,6 @@
code = (*f->compress_state->template->init) (f->compress_state);
if (code < 0)
return_error(gs_error_VMerror); /****** BOGUS ******/
- if (f->decompress_state->template->init != 0)
- code = (*f->decompress_state->template->init)
- (f->decompress_state);
- if (code < 0)
- return_error(gs_error_VMerror); /****** BOGUS ******/
f->compressor_initialized = true;
}
/* Write into the new physical block we just allocated, */
@@ -672,7 +775,7 @@
static int
memfile_get_pdata(MEMFILE * f)
{
- int i, num_raw_buffers, status;
+ int code, i, num_raw_buffers, status;
LOG_MEMFILE_BLK *bp = f->log_curr_blk;
if (bp->phys_blk->data_limit == NULL) {
@@ -726,6 +829,12 @@
num_raw_buffers = i + 1; /* if MALLOC failed, then OK */
if_debug1(':', "[:]Number of raw buffers allocated=%d\n",
num_raw_buffers);
+ if (f->decompress_state->template->init != 0)
+ code = (*f->decompress_state->template->init)
+ (f->decompress_state);
+ if (code < 0)
+ return_error(gs_error_VMerror);
+
} /* end allocating the raw buffer pool (first time only) */
if (bp->raw_block == NULL) {
#ifdef DEBUG
@@ -821,7 +930,7 @@
/* is always full size. */
} /* end else (when data was compressed) */
- return (0);
+ return 0;
}
/* ---------------- Reading ---------------- */
@@ -960,47 +1069,7 @@
/* Free up memory that was allocated for the memfile */
bp = f->log_head;
-/******************************************************************
- * The following was the original algorithm here. This algorithm has a bug:
- * the second loop references the physical blocks again after they have been
- * freed.
- ******************************************************************/
-
-#if 0 /**************** *************** */
-
if (bp != NULL) {
- /* Free the physical blocks that make up the compressed data */
- PHYS_MEMFILE_BLK *pphys = (f->log_head)->phys_blk;
-
- if (pphys->data_limit != NULL) {
- /* the data was compressed, free the chain of blocks */
- while (pphys != NULL) {
- PHYS_MEMFILE_BLK *tmpphys = pphys->link;
-
- FREE(f, pphys, "memfile_free_mem(pphys)");
- pphys = tmpphys;
- }
- }
- }
- /* free the logical blocks */
- while (bp != NULL) {
- /* if this logical block was not compressed, free the phys_blk */
- if (bp->phys_blk->data_limit == NULL) {
- FREE(f, bp->phys_blk, "memfile_free_mem(phys_blk)");
- }
- tmpbp = bp->link;
- FREE(f, bp, "memfile_free_mem(log_blk)");
- bp = tmpbp;
- }
-
-#else /**************** *************** */
-# if 1 /**************** *************** */
-
-/****************************************************************
- * This algorithm is correct (we think).
- ****************************************************************/
-
- if (bp != NULL) {
/* Null out phys_blk pointers to compressed data. */
PHYS_MEMFILE_BLK *pphys = bp->phys_blk;
@@ -1030,35 +1099,6 @@
bp = tmpbp;
}
-/***********************************************************************
- * This algorithm appears to be both simpler and free of the bug that
- * occasionally causes the older one to reference freed blocks; but in
- * fact it can miss blocks, because the very last compressed logical block
- * can have spill into a second physical block, which is not referenced by
- * any logical block.
- ***********************************************************************/
-
-# else /**************** *************** */
-
- {
- PHYS_MEMFILE_BLK *prev_phys = 0;
-
- while (bp != NULL) {
- PHYS_MEMFILE_BLK *phys = bp->phys_blk;
-
- if (phys != prev_phys) {
- FREE(f, phys, "memfile_free_mem(phys_blk)");
- prev_phys = phys;
- }
- tmpbp = bp->link;
- FREE(f, bp, "memfile_free_mem(log_blk)");
- bp = tmpbp;
- }
- }
-
-# endif /**************** *************** */
-#endif /**************** *************** */
-
f->log_head = NULL;
/* Free any internal compressor state. */
Modified: branches/mtrender/src/gxclmem.h
===================================================================
--- branches/mtrender/src/gxclmem.h 2008-04-23 14:06:11 UTC (rev 8659)
+++ branches/mtrender/src/gxclmem.h 2008-04-24 19:24:08 UTC (rev 8660)
@@ -49,7 +49,7 @@
typedef struct PHYS_MEMFILE_BLK {
struct PHYS_MEMFILE_BLK *link;
char *data_limit; /* end of data when compressed */
- /* NULL if not compressed */
+ /* data_limit is NULL if not compressed */
char data_spare[4]; /* used during de-compress */
char data[MEMFILE_DATA_SIZE];
} PHYS_MEMFILE_BLK;
@@ -65,6 +65,21 @@
gs_memory_t *memory; /* storage allocator */
gs_memory_t *data_memory; /* storage allocator for data */
bool ok_to_compress; /* if true, OK to compress this file */
+ /*
+ * We need to maintain a linked list of other structs that
+ * have beed opened on this 'file'. This allows for a writer
+ * to create a file, close it (but not unlink it), then other
+ * 'opens' of the same file, using the pseudo file-name that
+ * consists of a flag 0xff byte followed by the address as
+ * a 0x____ string.
+ *
+ * When the file is 'unlinked' it must check for other open
+ * 'readers' and set the 'READER INSTANCE' values of the
+ * cloned structures so that calls into 'read' and 'seek'
+ * etc., will be harmless.
+ */
+ struct MEMFILE_s *openlist;
+ struct MEMFILE_s *base_memfile; /* reader instances set this to the file that was written */
/*
* Reserve memory blocks: these are used to guarantee that a
* given-sized write (or sequence of writes) will always succeed.
@@ -87,21 +102,21 @@
int reserveLogBlockCount; /* count of entries on reserveLogBlockChain */
/* logical file properties */
LOG_MEMFILE_BLK *log_head;
- LOG_MEMFILE_BLK *log_curr_blk;
+ LOG_MEMFILE_BLK *log_curr_blk; /******* READER INSTANCE *******/
int64_t log_length; /* updated during write */
- int64_t log_curr_pos; /* updated during seek, close, read */
- char *pdata; /* raw data */
- char *pdata_end;
+ int64_t log_curr_pos; /* updated during seek, close, read */ /******* READER INSTANCE *******/
+ char *pdata; /* raw data */ /******* READER INSTANCE *******/
+ char *pdata_end; /******* READER INSTANCE *******/
/* physical file properties */
int64_t total_space; /* so we know when to start compress */
- PHYS_MEMFILE_BLK *phys_curr; /* NULL if not compressing */
- RAW_BUFFER *raw_head, *raw_tail;
- int error_code; /* used by CLIST_ferror */
- stream_cursor_read rd; /* use .ptr, .limit */
- stream_cursor_write wt; /* use .ptr, .limit */
+ PHYS_MEMFILE_BLK *phys_curr; /* NULL if not compressing */ /******* READER INSTANCE *******/
+ RAW_BUFFER *raw_head, *raw_tail ; /******* READER INSTANCE *******/
+ int error_code; /* used by CLIST_ferror */ /******* READER INSTANCE *******/
+ stream_cursor_read rd; /* use .ptr, .limit */ /******* READER INSTANCE *******/
+ stream_cursor_write wt; /* use .ptr, .limit */ /******* READER INSTANCE *******/
bool compressor_initialized;
stream_state *compress_state;
- stream_state *decompress_state;
+ stream_state *decompress_state; /******* READER INSTANCE *******/
};
#ifndef MEMFILE_DEFINED
#define MEMFILE_DEFINED
More information about the gs-cvs
mailing list