[gs-cvs] rev 8056 - trunk/gs/src
ray at ghostscript.com
ray at ghostscript.com
Sun Jun 17 23:37:59 PDT 2007
Author: ray
Date: 2007-06-17 23:37:58 -0700 (Sun, 17 Jun 2007)
New Revision: 8056
Modified:
trunk/gs/src/gdevdrop.c
trunk/gs/src/gdevdsp.c
trunk/gs/src/gdevmem.c
trunk/gs/src/gdevppla.c
trunk/gs/src/gdevprn.c
trunk/gs/src/gdevxini.c
trunk/gs/src/gxccman.c
trunk/gs/src/gxclist.c
trunk/gs/src/gxdevmem.h
trunk/gs/src/gximag3x.c
trunk/gs/src/gxmclip.c
trunk/gs/src/gxpcmap.c
Log:
Fixes for ulong overflow in memory device buffer sizing that caused
problems with very large (>4Gb on 32-bit machines) page buffer sizes
not invoking clist mode. Also the internal 'mask' image allocation
in ImageType3 would silently wrap around, causing SEGV crashes.
Lastly, the tiff*nc devices are fixed so that they never write to
file when the file will exceed 2Gb (max signed int) since the device
uses 'ftell' to calculate offsets in the file. This last case could
be fixed to use 64 bit file I/O or at least to keep an internal
offset so that multipage tiffs could be > 2Gb as long as each page
was less than 2Gb. Other TIFF devices that compress were not changed
since the probability of overflow is less and we don't want to be
overly agressive about throwing a 'rangecheck' on the basis of the
uncompressed size.
Bugs 688808, 688815 and 689080 for customer 190 (and others).
DETAILS:
The buf_device/bitmap sizing functions now return a success code,
setting the size using a pointer passed as a parameter. A bitmap
that overflows max_ulong now throws gs_error_VMerror so that the
caller can take appropriate recovery action (such as gdevprn using
a clist instead of a page buffer).
The ImageType 3 mask probably needs an internal clist method (such
as was done for the pattern accumulator) to avoid this overflow.
As mentioned above, the TIFF devices, particularly the '*nc' type
(not compressed) could benefit from using 64-bit file I/O calls
for those that want to fill up large hard drives quickly.
EXPECTED DIFFERENCES:
None.
Modified: trunk/gs/src/gdevdrop.c
===================================================================
--- trunk/gs/src/gdevdrop.c 2007-06-17 17:55:26 UTC (rev 8055)
+++ trunk/gs/src/gdevdrop.c 2007-06-18 06:37:58 UTC (rev 8056)
@@ -369,6 +369,7 @@
gx_device_memory mdev;
union { long l; void *p; } mdev_storage[20];
uint row_raster = bitmap_raster(width * depth);
+ ulong size_from_mem_device;
gs_rop3_t trans_rop = gs_transparent_rop(lop);
bool uses_d = rop3_uses_D(trans_rop);
bool uses_s = rop3_uses_S(trans_rop);
@@ -451,11 +452,13 @@
mdev.width = width;
mdev.height = block_height;
mdev.color_info.num_components = rop_depth >> 3;
- if (gdev_mem_data_size(&mdev, width, block_height) <= sizeof(mdev_storage)) {
+ if (gdev_mem_data_size(&mdev, width, block_height, &size_from_mem_device) >= 0 &&
+ size_from_mem_device <= sizeof(mdev_storage)) {
/* Use the locally allocated storage. */
mdev.base = (byte *)mdev_storage;
- mdev.line_ptrs = (byte **)
- (mdev.base + gdev_mem_bits_size(&mdev, mdev.width, mdev.height));
+ if ((code = gdev_mem_bits_size(&mdev, mdev.width, mdev.height, &size_from_mem_device)) < 0)
+ return code;
+ mdev.line_ptrs = (byte **) (mdev.base + size_from_mem_device);
} else {
mdev.bitmap_memory = mem;
}
Modified: trunk/gs/src/gdevdsp.c
===================================================================
--- trunk/gs/src/gdevdsp.c 2007-06-17 17:55:26 UTC (rev 8055)
+++ trunk/gs/src/gdevdsp.c 2007-06-18 06:37:58 UTC (rev 8056)
@@ -1351,8 +1351,9 @@
* so we can place the bitmap in special memory.
*/
ddev->mdev->line_pointer_memory = ddev->mdev->memory;
- ddev->ulBitmapSize = gdev_mem_bits_size(ddev->mdev,
- ddev->mdev->width, ddev->mdev->height);
+ if (gdev_mem_bits_size(ddev->mdev, ddev->mdev->width, ddev->mdev->height,
+ &(ddev->ulBitmapSize)) < 0)
+ return_error(gs_error_VMerror);
/* allocate bitmap using an allocator not subject to GC */
if (ddev->callback->display_memalloc
Modified: trunk/gs/src/gdevmem.c
===================================================================
--- trunk/gs/src/gdevmem.c 2007-06-17 17:55:26 UTC (rev 8055)
+++ trunk/gs/src/gdevmem.c 2007-06-18 06:37:58 UTC (rev 8056)
@@ -209,9 +209,11 @@
* The computation for planar devices is a little messier. Each plane
* must pad its scan lines, and then we must pad again for the pointer
* tables (one table per plane).
+ *
+ * Return VMerror if the size exceeds max ulong
*/
-ulong
-gdev_mem_bits_size(const gx_device_memory * dev, int width, int height)
+int
+gdev_mem_bits_size(const gx_device_memory * dev, int width, int height, ulong *psize)
{
int num_planes = dev->num_planes;
gx_render_plane_t plane1;
@@ -225,18 +227,27 @@
planes = &plane1, plane1.depth = dev->color_info.depth, num_planes = 1;
for (size = 0, pi = 0; pi < num_planes; ++pi)
size += bitmap_raster(width * planes[pi].depth);
- return ROUND_UP(size * height, ARCH_ALIGN_PTR_MOD);
+ if (size > (max_ulong - ARCH_ALIGN_PTR_MOD) / (ulong)height)
+ return_error(gs_error_VMerror);
+ *psize = ROUND_UP(size * height, ARCH_ALIGN_PTR_MOD);
+ return 0;
}
ulong
gdev_mem_line_ptrs_size(const gx_device_memory * dev, int width, int height)
{
return (ulong)height * sizeof(byte *) * max(dev->num_planes, 1);
}
-ulong
-gdev_mem_data_size(const gx_device_memory * dev, int width, int height)
+int
+gdev_mem_data_size(const gx_device_memory * dev, int width, int height, ulong *psize)
{
- return gdev_mem_bits_size(dev, width, height) +
- gdev_mem_line_ptrs_size(dev, width, height);
+ ulong bits_size;
+ ulong line_ptrs_size = gdev_mem_line_ptrs_size(dev, width, height);
+
+ if (gdev_mem_bits_size(dev, width, height, &bits_size) < 0 ||
+ bits_size > max_ulong - line_ptrs_size)
+ return_error(gs_error_VMerror);
+ *psize = bits_size + line_ptrs_size;
+ return 0;
}
/*
* Do the inverse computation: given a width (in pixels) and a buffer size,
@@ -248,6 +259,7 @@
{
int height;
ulong max_height;
+ ulong data_size;
if (page_uses_transparency) {
/*
@@ -271,8 +283,12 @@
* Because of alignment rounding, the just-computed height might
* be too large by a small amount. Adjust it the easy way.
*/
- while (gdev_mem_data_size(dev, width, height) > size)
+ do {
+ gdev_mem_data_size(dev, width, height, &data_size);
+ if (data_size <= size)
+ break;
--height;
+ } while (data_size > size);
}
return height;
}
@@ -293,14 +309,16 @@
gdev_mem_open_scan_lines(gx_device_memory *mdev, int setup_height)
{
bool line_pointers_adjacent = true;
+ ulong size;
if (setup_height < 0 || setup_height > mdev->height)
return_error(gs_error_rangecheck);
if (mdev->bitmap_memory != 0) {
/* Allocate the data now. */
- ulong size = gdev_mem_bitmap_size(mdev);
+ if (gdev_mem_bitmap_size(mdev, &size) < 0)
+ return_error(gs_error_VMerror);
- if ((uint) size != size)
+ if ((uint) size != size) /* ulong may be bigger than uint */
return_error(gs_error_limitcheck);
mdev->base = gs_alloc_bytes(mdev->bitmap_memory, (uint)size,
"mem_open");
@@ -319,9 +337,10 @@
mdev->foreign_line_pointers = false;
line_pointers_adjacent = false;
}
- if (line_pointers_adjacent)
- mdev->line_ptrs = (byte **)
- (mdev->base + gdev_mem_bits_size(mdev, mdev->width, mdev->height));
+ if (line_pointers_adjacent) {
+ gdev_mem_bits_size(mdev, mdev->width, mdev->height, &size);
+ mdev->line_ptrs = (byte **)(mdev->base + size);
+ }
mdev->raster = gdev_mem_raster(mdev);
return gdev_mem_set_line_ptrs(mdev, NULL, 0, NULL, setup_height);
}
Modified: trunk/gs/src/gdevppla.c
===================================================================
--- trunk/gs/src/gdevppla.c 2007-06-17 17:55:26 UTC (rev 8055)
+++ trunk/gs/src/gdevppla.c 2007-06-18 06:37:58 UTC (rev 8056)
@@ -121,7 +121,8 @@
height, for_band);
mdev.color_info = target->color_info;
gdev_prn_set_planar(&mdev, target);
- space->bits = gdev_mem_bits_size(&mdev, target->width, height);
+ if (gdev_mem_bits_size(&mdev, target->width, height, &(space->bits)) < 0)
+ return_error(gs_error_VMerror);
space->line_ptrs = gdev_mem_line_ptrs_size(&mdev, target->width, height);
space->raster = bitmap_raster(target->width * mdev.planes[0].depth);
return 0;
Modified: trunk/gs/src/gdevprn.c
===================================================================
--- trunk/gs/src/gdevprn.c 2007-06-17 17:55:26 UTC (rev 8055)
+++ trunk/gs/src/gdevprn.c 2007-06-18 06:37:58 UTC (rev 8056)
@@ -232,6 +232,7 @@
int save_height = 0x0badf00d; /* Quiet compiler */
bool is_command_list = false; /* Quiet compiler */
bool save_is_command_list = false; /* Quiet compiler */
+ bool size_ok = 0;
int ecode = 0;
int pass;
gs_memory_t *buffer_memory =
@@ -274,11 +275,16 @@
/* Init clist/mem device-specific fields */
memset(ppdev->skip, 0, sizeof(ppdev->skip));
- ppdev->printer_procs.buf_procs.size_buf_device
- (&buf_space, pdev, NULL, pdev->height, false);
- if (ppdev->page_uses_transparency)
- pdf14_trans_buffer_size = new_height
- * (ESTIMATED_PDF14_ROW_SPACE(new_width) >> 3);
+ size_ok = ppdev->printer_procs.buf_procs.size_buf_device
+ (&buf_space, pdev, NULL, pdev->height, false) >= 0;
+ if (ppdev->page_uses_transparency)
+ if (new_height < max_ulong/(ESTIMATED_PDF14_ROW_SPACE(new_width) >> 3))
+ pdf14_trans_buffer_size = new_height
+ * (ESTIMATED_PDF14_ROW_SPACE(new_width) >> 3);
+ else {
+ size_ok = 0;
+ pdf14_trans_buffer_size = 0;
+ }
mem_space = buf_space.bits + buf_space.line_ptrs
+ pdf14_trans_buffer_size;
@@ -304,7 +310,7 @@
else {
is_command_list = space_params.banding_type == BandingAlways ||
mem_space >= space_params.MaxBitmap ||
- mem_space != (uint)mem_space; /* too big to allocate */
+ !size_ok; /* too big to allocate */
}
if (!is_command_list) {
/* Try to allocate memory for full memory buffer */
@@ -1062,12 +1068,16 @@
{
gx_device_memory mdev;
+ space->line_ptrs = 0; /* */
+ space->bits = 0; /* clear in case of failure */
+ space->raster = 0; /* */
mdev.color_info.depth =
(render_plane && render_plane->index >= 0 ? render_plane->depth :
target->color_info.depth);
mdev.width = target->width;
mdev.num_planes = 0;
- space->bits = gdev_mem_bits_size(&mdev, target->width, height);
+ if (gdev_mem_bits_size(&mdev, target->width, height, &(space->bits)) < 0)
+ return_error(gs_error_VMerror);
space->line_ptrs = gdev_mem_line_ptrs_size(&mdev, target->width, height);
space->raster = gdev_mem_raster(&mdev);
return 0;
Modified: trunk/gs/src/gdevxini.c
===================================================================
--- trunk/gs/src/gdevxini.c 2007-06-17 17:55:26 UTC (rev 8055)
+++ trunk/gs/src/gdevxini.c 2007-06-18 06:37:58 UTC (rev 8056)
@@ -550,8 +550,8 @@
byte *buffer;
ulong space;
- space = gdev_mem_data_size(mdev, xdev->width, xdev->height);
- if (space > xdev->MaxBitmap) {
+ if (gdev_mem_data_size(mdev, xdev->width, xdev->height, &space) < 0 ||
+ space > xdev->MaxBitmap) {
buffered = false;
goto setup;
}
Modified: trunk/gs/src/gxccman.c
===================================================================
--- trunk/gs/src/gxccman.c 2007-06-17 17:55:26 UTC (rev 8055)
+++ trunk/gs/src/gxccman.c 2007-06-18 06:37:58 UTC (rev 8056)
@@ -562,7 +562,7 @@
int log2_yscale = pscale->y;
int log2_depth = ilog2(depth);
uint nwidth_bits = (iwidth >> log2_xscale) << log2_depth;
- ulong isize, icdsize;
+ ulong isize, icdsize, isize2;
uint iraster;
cached_char *cc;
gx_device_memory mdev;
@@ -611,7 +611,7 @@
pdev->retained = retained;
pdev->width = iwidth;
pdev->height = iheight;
- isize = gdev_mem_bitmap_size(pdev);
+ gdev_mem_bitmap_size(pdev, &isize); /* Assume less than max_ulong */
pdev->HWResolution[0] = HWResolution0;
pdev->HWResolution[1] = HWResolution1;
} else {
@@ -630,8 +630,9 @@
dev->rc = rc;
dev->width = iwidth;
dev->height = 2 << log2_yscale;
- isize = gdev_mem_bitmap_size(dev) +
- gdev_mem_bitmap_size(dev2);
+ gdev_mem_bitmap_size(dev, &isize); /* Assume less than max_ulong */
+ gdev_mem_bitmap_size(dev2, &isize2); /* Assume less than max_ulong */
+ isize += isize2; /* Assume less than max_ulong */
dev->HWResolution[0] = HWResolution0 * (1 >> log2_xscale);
dev->HWResolution[1] = HWResolution1 * (1 >> log2_yscale);
}
@@ -663,8 +664,9 @@
if (dev2) { /* The second device is an alpha device that targets */
/* the real storage for the character. */
byte *bits = cc_bits(cc);
- uint bsize = (uint) gdev_mem_bitmap_size(dev2);
+ ulong bsize;
+ gdev_mem_bitmap_size(dev2, &bsize);
memset(bits, 0, bsize);
dev2->base = bits;
(*dev_proc(dev2, open_device)) ((gx_device *) dev2);
@@ -681,10 +683,13 @@
gx_open_cache_device(gx_device_memory * dev, cached_char * cc)
{
byte *bits = cc_bits(cc);
+ ulong bsize;
+ gdev_mem_bitmap_size(dev, &bsize);
+
dev->width = cc->width;
dev->height = cc->height;
- memset((char *)bits, 0, (uint) gdev_mem_bitmap_size(dev));
+ memset((char *)bits, 0, bsize);
dev->base = bits;
(*dev_proc(dev, open_device)) ((gx_device *) dev); /* initialize */
}
Modified: trunk/gs/src/gxclist.c
===================================================================
--- trunk/gs/src/gxclist.c 2007-06-17 17:55:26 UTC (rev 8055)
+++ trunk/gs/src/gxclist.c 2007-06-18 06:37:58 UTC (rev 8056)
@@ -274,6 +274,7 @@
gx_device_clist_writer * const cdev =
&((gx_device_clist *)dev)->writer;
int nbands;
+ ulong space;
if (dev->procs.open_device == pattern_clist_open_device) {
/* We don't need bands really. */
@@ -281,7 +282,8 @@
cdev->nbands = 1;
return 0;
}
- if (gdev_mem_data_size(bdev, band_width, band_height) > data_size)
+ if (gdev_mem_data_size(bdev, band_width, band_height, &space) < 0 ||
+ space > data_size)
return_error(gs_error_rangecheck);
cdev->page_band_height = band_height;
nbands = (cdev->target->height + band_height - 1) / band_height;
@@ -359,10 +361,10 @@
* The band height is fixed, so the band buffer requirement
* is completely determined.
*/
- uint band_data_size =
- gdev_mem_data_size(&bdev, band_width, band_height);
+ ulong band_data_size;
- if (band_data_size >= band_space)
+ if (gdev_mem_data_size(&bdev, band_width, band_height, &band_data_size) < 0 ||
+ band_data_size >= band_space)
return_error(gs_error_rangecheck);
bits_size = min(band_space - band_data_size, data_size >> 1);
} else {
Modified: trunk/gs/src/gxdevmem.h
===================================================================
--- trunk/gs/src/gxdevmem.h 2007-06-17 17:55:26 UTC (rev 8055)
+++ trunk/gs/src/gxdevmem.h 2007-06-18 06:37:58 UTC (rev 8056)
@@ -177,17 +177,17 @@
* size includes both the bitmap and the line pointers.
*/
/* bits only */
-ulong gdev_mem_bits_size(const gx_device_memory *mdev, int width,
- int height);
+int gdev_mem_bits_size(const gx_device_memory *mdev, int width,
+ int height, ulong *size);
/* line pointers only */
ulong gdev_mem_line_ptrs_size(const gx_device_memory *mdev, int width,
int height);
/* bits + line pointers */
-ulong gdev_mem_data_size(const gx_device_memory *mdev, int width,
- int height);
+int gdev_mem_data_size(const gx_device_memory *mdev, int width,
+ int height, ulong *size);
-#define gdev_mem_bitmap_size(mdev)\
- gdev_mem_data_size(mdev, (mdev)->width, (mdev)->height)
+#define gdev_mem_bitmap_size(mdev, size)\
+ gdev_mem_data_size(mdev, (mdev)->width, (mdev)->height, size)
/*
* Do the inverse computation: given the device width and a buffer size,
Modified: trunk/gs/src/gximag3x.c
===================================================================
--- trunk/gs/src/gximag3x.c 2007-06-17 17:55:26 UTC (rev 8055)
+++ trunk/gs/src/gximag3x.c 2007-06-18 06:37:58 UTC (rev 8056)
@@ -489,6 +489,8 @@
gx_device_memory *midev;
int code;
+ if (height > max_ulong/width) /* protect against overflow in bitmap size */
+ return_error(gs_error_VMerror);
if (mdproto == 0)
return_error(gs_error_rangecheck);
midev = gs_alloc_struct(mem, gx_device_memory, &st_device_memory,
Modified: trunk/gs/src/gxmclip.c
===================================================================
--- trunk/gs/src/gxmclip.c 2007-06-17 17:55:26 UTC (rev 8055)
+++ trunk/gs/src/gxmclip.c 2007-06-18 06:37:58 UTC (rev 8056)
@@ -13,6 +13,7 @@
/* $Id$ */
/* Mask clipping support */
+#include "gserrors.h"
#include "gx.h"
#include "gxdevice.h"
#include "gxdevmem.h"
@@ -81,17 +82,20 @@
buffer_height = bits->size.y;
gs_make_mem_mono_device(&cdev->mdev, 0, 0);
for (;;) {
+ ulong bitmap_size = max_ulong;
+
if (buffer_height <= 0) {
/*
* The tile is too wide to buffer even one scan line.
* We could do copy_mono in chunks, but for now, we punt.
*/
cdev->mdev.base = 0;
- return 0;
+ return_error(gs_error_VMerror);
}
cdev->mdev.width = buffer_width;
cdev->mdev.height = buffer_height;
- if (gdev_mem_bitmap_size(&cdev->mdev) <= tile_clip_buffer_size)
+ gdev_mem_bitmap_size(&cdev->mdev, &bitmap_size);
+ if (bitmap_size <= tile_clip_buffer_size)
break;
buffer_height--;
}
Modified: trunk/gs/src/gxpcmap.c
===================================================================
--- trunk/gs/src/gxpcmap.c 2007-06-17 17:55:26 UTC (rev 8055)
+++ trunk/gs/src/gxpcmap.c 2007-06-18 06:37:58 UTC (rev 8056)
@@ -587,16 +587,19 @@
mdev.width = ctile->tmask.size.x;
mdev.height = ctile->tmask.size.y;
/*mdev.color_info.depth = 1;*/
- used = gdev_mem_bitmap_size(&mdev);
+ gdev_mem_bitmap_size(&mdev, &used);
gs_free_object(mem, ctile->tmask.data,
"free_pattern_cache_entry(mask data)");
ctile->tmask.data = 0; /* for GC */
}
if (ctile->tbits.data != 0) {
+ ulong tbits_used = 0;
+
mdev.width = ctile->tbits.size.x;
mdev.height = ctile->tbits.size.y;
mdev.color_info.depth = ctile->depth;
- used += gdev_mem_bitmap_size(&mdev);
+ gdev_mem_bitmap_size(&mdev, &tbits_used);
+ used += tbits_used;
gs_free_object(mem, ctile->tbits.data,
"free_pattern_cache_entry(bits data)");
ctile->tbits.data = 0; /* for GC */
@@ -624,7 +627,7 @@
{
gx_pattern_cache *pcache;
const gs_pattern1_instance_t *pinst;
- ulong used = 0;
+ ulong used = 0, mask_used = 0;
gx_bitmap_id id;
gx_color_tile *ctile;
int code = ensure_pattern_cache(pis);
@@ -664,9 +667,11 @@
keep:;
}
if (mbits != 0)
- used += gdev_mem_bitmap_size(mbits);
- if (mmask != 0)
- used += gdev_mem_bitmap_size(mmask);
+ gdev_mem_bitmap_size(mbits, &used);
+ if (mmask != 0) {
+ gdev_mem_bitmap_size(mmask, &mask_used);
+ used += mask_used;
+ }
} else {
gx_device_clist *cdev = (gx_device_clist *)fdev;
gx_device_clist_writer * cldev = (gx_device_clist_writer *)cdev;
More information about the gs-cvs
mailing list