This patch exists to address CERT Vulnerability Report VU#102441 (CVE ID CAN-2005-4095) related to a potential X server exploit when an X client allocates large pixmaps. *** xc/programs/Xserver/afb/afbpixmap.c Tue Jun 3 08:11:07 1997 --- xc/programs/Xserver/afb/afbpixmap.c Tue Sep 20 10:34:17 2005 *************** afbCreatePixmap(pScreen, width, height, *** 76,81 **** --- 76,84 ---- int datasize; int paddedWidth; + if ((width > MAXSHORT) || (height > MAXSHORT)) + return NullPixmap; + paddedWidth = BitmapBytePad(width); datasize = height * paddedWidth * depth; pPixmap = AllocatePixmap(pScreen, datasize); *** xc/programs/Xserver/cfb/cfbpixmap.c Fri Dec 14 12:59:23 2001 --- xc/programs/Xserver/cfb/cfbpixmap.c Tue Sep 20 10:34:17 2005 *************** cfbCreatePixmap (pScreen, width, height, *** 73,78 **** --- 72,80 ---- int datasize; int paddedWidth; + if ((width > MAXSHORT) || (height > MAXSHORT)) + return NullPixmap; + paddedWidth = PixmapBytePad(width, depth); datasize = height * paddedWidth; pPixmap = AllocatePixmap(pScreen, datasize); *** xc/programs/Xserver/dix/pixmap.c Fri Dec 14 12:59:32 2001 --- xc/programs/Xserver/dix/pixmap.c Tue Sep 20 10:34:17 2005 *************** AllocatePixmap(pScreen, pixDataSize) *** 126,132 **** unsigned size; int i; ! pPixmap = (PixmapPtr)xalloc(pScreen->totalPixmapSize + pixDataSize); if (!pPixmap) return NullPixmap; ppriv = (DevUnion *)(pPixmap + 1); --- 125,133 ---- unsigned size; int i; ! if (((unsigned)(-1) - pScreen->totalPixmapSize) < (unsigned)pixDataSize) ! return NullPixmap; ! pPixmap = xalloc(pScreen->totalPixmapSize + (unsigned)pixDataSize); if (!pPixmap) return NullPixmap; ppriv = (DevUnion *)(pPixmap + 1); *************** AllocatePixmap(pScreen, pixDataSize) *** 144,150 **** ppriv->ptr = (pointer)NULL; } #else ! pPixmap = (PixmapPtr)xalloc(sizeof(PixmapRec) + pixDataSize); #endif return pPixmap; } --- 145,151 ---- ppriv->ptr = (pointer)NULL; } #else ! pPixmap = xalloc(sizeof(PixmapRec) + (unsigned)pixDataSize); #endif return pPixmap; } *** xc/programs/Xserver/fb/fbpixmap.c Mon Sep 16 12:05:34 2002 --- xc/programs/Xserver/fb/fbpixmap.c Tue Sep 20 10:34:17 2005 *************** fbCreatePixmapBpp (ScreenPtr pScreen, in *** 37,42 **** --- 37,45 ---- int adjust; int base; + if ((width > MAXSHORT) || (height > MAXSHORT)) + return NullPixmap; + paddedWidth = ((width * bpp + FB_MASK) >> FB_SHIFT) * sizeof (FbBits); datasize = height * paddedWidth; #ifdef PIXPRIV *** xc/programs/Xserver/hw/xfree86/xaa/xaaInit.c Thu Jul 19 12:50:16 2001 --- xc/programs/Xserver/hw/xfree86/xaa/xaaInit.c Tue Sep 20 10:34:17 2005 *************** XAACreatePixmap(ScreenPtr pScreen, int w *** 479,486 **** ScrnInfoPtr pScrn = xf86Screens[pScreen->myNum]; XAAPixmapPtr pPriv; PixmapPtr pPix = NULL; ! int size = w * h; if (!infoRec->offscreenDepthsInitialized) XAAInitializeOffscreenDepths (pScreen); --- 479,490 ---- ScrnInfoPtr pScrn = xf86Screens[pScreen->myNum]; XAAPixmapPtr pPriv; PixmapPtr pPix = NULL; ! int size; ! ! if ((w > MAXSHORT) || (h > MAXSHORT)) ! return NullPixmap; + size = w * h; if (!infoRec->offscreenDepthsInitialized) XAAInitializeOffscreenDepths (pScreen); *** xc/programs/Xserver/hw/xfree86/xf4bpp/ppcPixmap.c Thu Apr 27 10:26:49 2000 --- xc/programs/Xserver/hw/xfree86/xf4bpp/ppcPixmap.c Tue Sep 20 10:34:17 2005 *************** xf4bppCreatePixmap( pScreen, width, heig *** 89,96 **** TRACE(("xf4bppCreatePixmap(pScreen=0x%x, width=%d, height=%d, depth=%d)\n", pScreen, width, height, depth)) ; ! if ( depth > 8 ) ! return (PixmapPtr) NULL ; size = PixmapBytePad(width, depth); pPixmap = AllocatePixmap (pScreen, (height * size)); --- 89,96 ---- TRACE(("xf4bppCreatePixmap(pScreen=0x%x, width=%d, height=%d, depth=%d)\n", pScreen, width, height, depth)) ; ! if ((depth > 8) || (width > MAXSHORT) || (height > MAXSHORT)) ! return NullPixmap; size = PixmapBytePad(width, depth); pPixmap = AllocatePixmap (pScreen, (height * size)); *** xc/programs/Xserver/ilbm/ilbmpixmap.c Sat Aug 17 19:54:01 1996 --- xc/programs/Xserver/ilbm/ilbmpixmap.c Tue Sep 20 10:34:17 2005 *************** ilbmCreatePixmap(pScreen, width, height, *** 78,83 **** --- 78,86 ---- int datasize; int paddedWidth; + if ((width > MAXSHORT) || (height > MAXSHORT)) + return NullPixmap; + paddedWidth = BitmapBytePad(width); datasize = height * paddedWidth * depth; pPixmap = AllocatePixmap(pScreen, datasize); *** xc/programs/Xserver/iplan2p4/iplpixmap.c Mon Dec 17 13:00:46 2001 --- xc/programs/Xserver/iplan2p4/iplpixmap.c Tue Sep 20 10:34:17 2005 *************** iplCreatePixmap (pScreen, width, height, *** 76,83 **** PixmapPtr pPixmap; int datasize; int paddedWidth; ! int ipad=INTER_PLANES*2 - 1; paddedWidth = PixmapBytePad(width, depth); paddedWidth = (paddedWidth + ipad) & ~ipad; datasize = height * paddedWidth; --- 76,87 ---- PixmapPtr pPixmap; int datasize; int paddedWidth; ! int ipad; + if ((width > MAXSHORT) || (height > MAXSHORT)) + return NullPixmap; + + ipad = INTER_PLANES*2 - 1; paddedWidth = PixmapBytePad(width, depth); paddedWidth = (paddedWidth + ipad) & ~ipad; datasize = height * paddedWidth; *** xc/programs/Xserver/mfb/mfbpixmap.c Fri Dec 14 13:00:10 2001 --- xc/programs/Xserver/mfb/mfbpixmap.c Tue Sep 20 10:34:17 2005 *************** mfbCreatePixmap (pScreen, width, height, *** 75,82 **** int datasize; int paddedWidth; ! if (depth != 1) return NullPixmap; paddedWidth = BitmapBytePad(width); datasize = height * paddedWidth; pPixmap = AllocatePixmap(pScreen, datasize); --- 75,83 ---- int datasize; int paddedWidth; ! if ((depth != 1) || (width > MAXSHORT) || (height > MAXSHORT)) return NullPixmap; + paddedWidth = BitmapBytePad(width); datasize = height * paddedWidth; pPixmap = AllocatePixmap(pScreen, datasize); *** xc/programs/Xserver/os/utils.c Wed Feb 2 19:01:14 2005 --- xc/programs/Xserver/os/utils.c Tue Sep 20 10:34:17 2005 *************** set_font_authorizations(char **authoriza *** 1243,1261 **** void * Xalloc(unsigned long amount) { ! register pointer ptr; ! ! if ((long)amount <= 0) ! return NULL; /* aligned extra on long word boundary */ amount = (amount + (sizeof(long) - 1)) & ~(sizeof(long) - 1); #ifdef MEMBUG if (!Must_have_memory && Memory_fail && ((random() % MEM_FAIL_SCALE) < Memory_fail)) return NULL; #endif ! if ((ptr = (pointer)malloc(amount))) { return ptr; } if (Must_have_memory) --- 1243,1261 ---- void * Xalloc(unsigned long amount) { ! pointer ptr; /* aligned extra on long word boundary */ amount = (amount + (sizeof(long) - 1)) & ~(sizeof(long) - 1); + + if ((long)amount <= 0) + return NULL; #ifdef MEMBUG if (!Must_have_memory && Memory_fail && ((random() % MEM_FAIL_SCALE) < Memory_fail)) return NULL; #endif ! if ((ptr = malloc(amount))) { return ptr; } if (Must_have_memory) *************** XNFalloc(unsigned long amount) *** 1273,1284 **** { register pointer ptr; ! if ((long)amount <= 0) ! return NULL; /* aligned extra on long word boundary */ amount = (amount + (sizeof(long) - 1)) & ~(sizeof(long) - 1); ! ptr = (pointer)malloc(amount); if (!ptr) FatalError("Out of memory"); --- 1273,1288 ---- { register pointer ptr; ! if (amount == 0) ! return NULL; /* aligned extra on long word boundary */ amount = (amount + (sizeof(long) - 1)) & ~(sizeof(long) - 1); ! ! if ((long)amount <= 0) ! FatalError("Bad request for memory"); ! ! ptr = malloc(amount); if (!ptr) FatalError("Out of memory"); *************** XNFalloc(unsigned long amount) *** 1292,1302 **** void * Xcalloc(unsigned long amount) { ! unsigned long *ret; ! ret = Xalloc (amount); if (ret) ! bzero ((void *) ret, (int) amount); return ret; } --- 1296,1306 ---- void * Xcalloc(unsigned long amount) { ! pointer ret; ! ret = Xalloc(amount); if (ret) ! bzero (ret, (int) amount); return ret; } *************** Xcalloc(unsigned long amount) *** 1307,1319 **** void * XNFcalloc(unsigned long amount) { ! unsigned long *ret; ! ret = Xalloc (amount); ! if (ret) ! bzero ((char *) ret, (int) amount); ! else if ((long)amount > 0) FatalError("Out of memory"); return ret; } --- 1311,1327 ---- void * XNFcalloc(unsigned long amount) { ! pointer ret; ! if (amount == 0) ! return NULL; ! ! ret = Xalloc(amount); ! if (!ret) FatalError("Out of memory"); + + bzero (ret, (int) amount); + return ret; } *************** Xrealloc(pointer ptr, unsigned long amou *** 1336,1345 **** return NULL; } amount = (amount + (sizeof(long) - 1)) & ~(sizeof(long) - 1); if (ptr) ! ptr = (pointer)realloc((char *)ptr, amount); else ! ptr = (pointer)malloc(amount); if (ptr) return ptr; if (Must_have_memory) --- 1344,1355 ---- return NULL; } amount = (amount + (sizeof(long) - 1)) & ~(sizeof(long) - 1); + if ((long)amount <= 0) + return NULL; if (ptr) ! ptr = realloc(ptr, amount); else ! ptr = malloc(amount); if (ptr) return ptr; if (Must_have_memory) *************** Xrealloc(pointer ptr, unsigned long amou *** 1355,1366 **** void * XNFrealloc(pointer ptr, unsigned long amount) { ! if (( ptr = (pointer)Xrealloc( ptr, amount ) ) == NULL) { ! if ((long)amount > 0) FatalError( "Out of memory" ); } ! return ((unsigned long *)ptr); } /***************** --- 1365,1376 ---- void * XNFrealloc(pointer ptr, unsigned long amount) { ! if ((ptr = Xrealloc(ptr, amount)) == NULL) { ! if (amount != 0) FatalError( "Out of memory" ); } ! return ptr; } /*****************