File n_VNC-Don-t-let-VNC-access-the-framebuffer-dire... of Package xorg-x11-server

Overview Repositories Revisions Requests Users Attributes Meta

File n_VNC-Don-t-let-VNC-access-the-framebuffer-directly-an.patch of Package xorg-x11-server (Revision 6a68f286c9a349231c833eebde671905)

Currently displaying revision 6a68f286c9a349231c833eebde671905 , Show latest

From: Michal Srb <msrb@suse.cz>
Date: Tue, 6 Sep 2011 13:08:25 +0200
Subject: [PATCH 4/6] VNC: Don't let VNC access the framebuffer directly any more.
Patch-Mainline: Currently no upstream project.
Git-commit: 3e0de1d95b3ffd3988016b2d3f40f577393ad046
Signed-off: Egbert Eich <eich@suse.de> 
References: bnc #653915

It seems that accessing the framebuffer directly is not a good idea anymore.
This patch will let the tight encoding read the screen data using GetImage. It
may be little slower, but not dramatically - it already does few GetImage calls
on every repaint now.

Signed-off-by: Egbert Eich <eich@freedesktop.org>
---
 hw/vnc/tight.c           |   69 ++++++++++++++++++++++++++++++---------------
 hw/vnc/vncext.c          |   19 ++----------
 hw/xfree86/vnc/vncInit.c |   11 ++-----
 hw/xfree86/vnc/vncint.h  |    2 -
 4 files changed, 53 insertions(+), 48 deletions(-)

diff --git a/hw/vnc/tight.c b/hw/vnc/tight.c
index 5c54736..f27c73e 100644
--- a/hw/vnc/tight.c
+++ b/hw/vnc/tight.c
@@ -109,15 +109,17 @@ static unsigned char *tightAfterBuf = NULL;
 
 static int *prevRowBuf = NULL;
 
+static unsigned char* fakeFrameBuffer = NULL;
+
 
 /* Prototypes for static functions. */
 
-static void FindBestSolidArea (ScreenPtr pScreen, int x, int y, int w, int h,
+static void FindBestSolidArea (rfbClientPtr cl, int x, int y, int w, int h,
                                CARD32 colorValue, int *w_ptr, int *h_ptr);
-static void ExtendSolidArea   (ScreenPtr pScreen, int x, int y, int w, int h,
+static void ExtendSolidArea   (rfbClientPtr cl, int x, int y, int w, int h,
                                CARD32 colorValue,
                                int *x_ptr, int *y_ptr, int *w_ptr, int *h_ptr);
-static Bool CheckSolidTile    (ScreenPtr pScreen, int x, int y, int w, int h,
+static Bool CheckSolidTile    (rfbClientPtr cl, int x, int y, int w, int h,
                                CARD32 *colorPtr, Bool needSameColor);
 static Bool CheckSolidTile8   (ScreenPtr pScreen, int x, int y, int w, int h,
                                CARD32 *colorPtr, Bool needSameColor);
@@ -126,6 +128,7 @@ static Bool CheckSolidTile16  (ScreenPtr pScreen, int x, int y, int w, int h,
 static Bool CheckSolidTile32  (ScreenPtr pScreen, int x, int y, int w, int h,
                                CARD32 *colorPtr, Bool needSameColor);
 
+static Bool SendRectEncodingTight(rfbClientPtr cl, int x, int y, int w, int h);
 static Bool SendRectSimple    (rfbClientPtr cl, int x, int y, int w, int h);
 static Bool SendSubrect       (rfbClientPtr cl, int x, int y, int w, int h);
 static Bool SendTightHeader   (rfbClientPtr cl, int x, int y, int w, int h);
@@ -211,6 +214,25 @@ rfbSendRectEncodingTight(rfbClientPtr cl,
 			int x, int y, int w, int h)
 {
     VNCSCREENPTR(cl->pScreen);
+  
+    /* Copy the rectangle to the fake buffer for CheckSolidTile functions. */
+
+    if(!fakeFrameBuffer) fakeFrameBuffer = malloc(pVNC->width * pVNC->height * cl->format.bitsPerPixel / 8);
+    (*cl->translateFn)(cl->pScreen, cl->translateLookupTable, 
+                        &pVNC->rfbServerFormat,
+                        &cl->format, fakeFrameBuffer + (y * pVNC->width * cl->format.bitsPerPixel / 8),
+                        pVNC->paddedWidthInBytes, pVNC->width, h, 0, y);
+    
+    /* Call the inner part */
+    
+    return SendRectEncodingTight(cl, x, y, w, h);
+}
+
+static Bool
+SendRectEncodingTight(rfbClientPtr cl,
+		      int x, int y, int w, int h)
+{
+    VNCSCREENPTR(cl->pScreen);
     int nMaxRows;
     CARD32 colorValue;
     int dx, dy, dw, dh;
@@ -247,7 +269,7 @@ rfbSendRectEncodingTight(rfbClientPtr cl,
         nMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
         nMaxRows = maxRectSize / nMaxWidth;
     }
-
+    
     /* Try to find large solid-color areas and send them separately. */
 
     for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {
@@ -269,11 +291,11 @@ rfbSendRectEncodingTight(rfbClientPtr cl,
             dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w) ?
                 MAX_SPLIT_TILE_SIZE : (x + w - dx);
 
-            if (CheckSolidTile(cl->pScreen, dx, dy, dw, dh, &colorValue, FALSE)) {
+            if (CheckSolidTile(cl, dx, dy, dw, dh, &colorValue, FALSE)) {
 
                 /* Get dimensions of solid-color area. */
 
-                FindBestSolidArea(cl->pScreen, dx, dy, w - (dx - x), h - (dy - y),
+                FindBestSolidArea(cl, dx, dy, w - (dx - x), h - (dy - y),
 				  colorValue, &w_best, &h_best);
 
                 /* Make sure a solid rectangle is large enough
@@ -286,7 +308,7 @@ rfbSendRectEncodingTight(rfbClientPtr cl,
                 /* Try to extend solid rectangle to maximum size. */
 
                 x_best = dx; y_best = dy;
-                ExtendSolidArea(cl->pScreen, x, y, w, h, colorValue,
+                ExtendSolidArea(cl, x, y, w, h, colorValue,
                                 &x_best, &y_best, &w_best, &h_best);
 
                 /* Send rectangles at top and left to solid-color area. */
@@ -295,7 +317,7 @@ rfbSendRectEncodingTight(rfbClientPtr cl,
                      !SendRectSimple(cl, x, y, w, y_best-y) )
                     return FALSE;
                 if ( x_best != x &&
-                     !rfbSendRectEncodingTight(cl, x, y_best,
+                     !SendRectEncodingTight(cl, x, y_best,
                                                x_best-x, h_best) )
                     return FALSE;
 
@@ -316,11 +338,11 @@ rfbSendRectEncodingTight(rfbClientPtr cl,
                 /* Send remaining rectangles (at right and bottom). */
 
                 if ( x_best + w_best != x + w &&
-                     !rfbSendRectEncodingTight(cl, x_best+w_best, y_best,
+                     !SendRectEncodingTight(cl, x_best+w_best, y_best,
                                                w-(x_best-x)-w_best, h_best) )
                     return FALSE;
                 if ( y_best + h_best != y + h &&
-                     !rfbSendRectEncodingTight(cl, x, y_best+h_best,
+                     !SendRectEncodingTight(cl, x, y_best+h_best,
                                                w, h-(y_best-y)-h_best) )
                     return FALSE;
 
@@ -339,7 +361,7 @@ rfbSendRectEncodingTight(rfbClientPtr cl,
 }
 
 static void
-FindBestSolidArea(ScreenPtr pScreen,
+FindBestSolidArea(rfbClientPtr cl,
 		  int x, int y, int w, int h,
 		  CARD32 colorValue,
 		  int *w_ptr, int *h_ptr)
@@ -357,13 +379,13 @@ FindBestSolidArea(ScreenPtr pScreen,
         dw = (w_prev > MAX_SPLIT_TILE_SIZE) ?
             MAX_SPLIT_TILE_SIZE : w_prev;
 
-        if (!CheckSolidTile(pScreen, x, dy, dw, dh, &colorValue, TRUE))
+        if (!CheckSolidTile(cl, x, dy, dw, dh, &colorValue, TRUE))
             break;
 
         for (dx = x + dw; dx < x + w_prev;) {
             dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w_prev) ?
                 MAX_SPLIT_TILE_SIZE : (x + w_prev - dx);
-            if (!CheckSolidTile(pScreen, dx, dy, dw, dh, &colorValue, TRUE))
+            if (!CheckSolidTile(cl, dx, dy, dw, dh, &colorValue, TRUE))
                 break;
 	    dx += dw;
         }
@@ -380,7 +402,7 @@ FindBestSolidArea(ScreenPtr pScreen,
 }
 
 static void
-ExtendSolidArea(ScreenPtr pScreen, int x, int y, int w, int h,
+ExtendSolidArea(rfbClientPtr cl, int x, int y, int w, int h,
 		CARD32 colorValue,
 		int *x_ptr, int *y_ptr, int *w_ptr, int *h_ptr)
 {
@@ -388,7 +410,7 @@ ExtendSolidArea(ScreenPtr pScreen, int x, int y, int w, int h,
 
     /* Try to extend the area upwards. */
     for ( cy = *y_ptr - 1;
-          cy >= y && CheckSolidTile(pScreen, *x_ptr, cy, *w_ptr, 1, &colorValue, TRUE);
+          cy >= y && CheckSolidTile(cl, *x_ptr, cy, *w_ptr, 1, &colorValue, TRUE);
           cy-- );
     *h_ptr += *y_ptr - (cy + 1);
     *y_ptr = cy + 1;
@@ -396,13 +418,13 @@ ExtendSolidArea(ScreenPtr pScreen, int x, int y, int w, int h,
     /* ... downwards. */
     for ( cy = *y_ptr + *h_ptr;
           cy < y + h &&
-              CheckSolidTile(pScreen, *x_ptr, cy, *w_ptr, 1, &colorValue, TRUE);
+              CheckSolidTile(cl, *x_ptr, cy, *w_ptr, 1, &colorValue, TRUE);
           cy++ );
     *h_ptr += cy - (*y_ptr + *h_ptr);
 
     /* ... to the left. */
     for ( cx = *x_ptr - 1;
-          cx >= x && CheckSolidTile(pScreen, cx, *y_ptr, 1, *h_ptr, &colorValue, TRUE);
+          cx >= x && CheckSolidTile(cl, cx, *y_ptr, 1, *h_ptr, &colorValue, TRUE);
           cx-- );
     *w_ptr += *x_ptr - (cx + 1);
     *x_ptr = cx + 1;
@@ -410,7 +432,7 @@ ExtendSolidArea(ScreenPtr pScreen, int x, int y, int w, int h,
     /* ... to the right. */
     for ( cx = *x_ptr + *w_ptr;
           cx < x + w &&
-              CheckSolidTile(pScreen, cx, *y_ptr, 1, *h_ptr, &colorValue, TRUE);
+              CheckSolidTile(cl, cx, *y_ptr, 1, *h_ptr, &colorValue, TRUE);
           cx++ );
     *w_ptr += cx - (*x_ptr + *w_ptr);
 }
@@ -423,11 +445,12 @@ ExtendSolidArea(ScreenPtr pScreen, int x, int y, int w, int h,
  */
 
 static Bool
-CheckSolidTile(ScreenPtr pScreen, int x, int y, int w, int h, CARD32 *colorPtr,
+CheckSolidTile(rfbClientPtr cl, int x, int y, int w, int h, CARD32 *colorPtr,
 	       Bool needSameColor)
 {
-    VNCSCREENPTR(pScreen);
-    switch(pVNC->rfbServerFormat.bitsPerPixel) {
+  ScreenPtr pScreen = cl->pScreen;
+
+  switch(cl->format.bitsPerPixel) {
     case 32:
         return CheckSolidTile32(pScreen, x, y, w, h, colorPtr, needSameColor);
     case 16:
@@ -449,7 +472,7 @@ static Bool                                                                   \
     int dx, dy;                                                               \
                                                                               \
     fbptr = (CARD##bpp *)                                                     \
-        &pVNC->pfbMemory[y * pVNC->paddedWidthInBytes + x * (bpp/8)]; \
+        &fakeFrameBuffer[(y * pVNC->width + x) * (bpp/8)];                    \
                                                                               \
     colorValue = *fbptr;                                                      \
     if (needSameColor && (CARD32)colorValue != *colorPtr)                     \
@@ -460,7 +483,7 @@ static Bool                                                                   \
             if (colorValue != fbptr[dx])                                      \
                 return FALSE;                                                 \
         }                                                                     \
-        fbptr = (CARD##bpp *)((CARD8 *)fbptr + pVNC->paddedWidthInBytes);     \
+        fbptr = (CARD##bpp *)((CARD8 *)fbptr + pVNC->width * (bpp/8));        \
     }                                                                         \
                                                                               \
     *colorPtr = (CARD32)colorValue;                                           \
diff --git a/hw/vnc/vncext.c b/hw/vnc/vncext.c
index ea913b7..534f3f5 100644
--- a/hw/vnc/vncext.c
+++ b/hw/vnc/vncext.c
@@ -702,15 +702,7 @@ CreateResourceTypes(void)
 
 static unsigned long vncExtGeneration = 0;
 #if XFREE86VNC
-extern Bool VNCInit(ScreenPtr pScreen, unsigned char *FBStart);
-
-/* copied from miscrinit.c */
-typedef struct
-{
-    pointer pbits; /* pointer to framebuffer */
-    int width;    /* delta to add to a framebuffer addr to move one row down */
-} miScreenInitParmsRec, *miScreenInitParmsPtr;
-
+extern Bool VNCInit(ScreenPtr pScreen);
 
 static Bool
 vncCreateScreenResources(ScreenPtr pScreen)
@@ -719,9 +711,6 @@ vncCreateScreenResources(ScreenPtr pScreen)
     CreateScreenResourcesProcPtr CreateScreenResources =
         (CreateScreenResourcesProcPtr)
         dixLookupPrivate(&pScreen->devPrivates, vncCreateScreenResourcesKey);
-    miScreenInitParmsPtr pScrInitParms;
-
-    pScrInitParms = (miScreenInitParmsPtr)pScreen->devPrivate;
 
     if ( pScreen->CreateScreenResources != vncCreateScreenResources ) {
         /* Can't find hook we are hung on */
@@ -732,9 +721,6 @@ vncCreateScreenResources(ScreenPtr pScreen)
                    (void *) pScreen->CreateScreenResources );
     }
 
-    /* Now do our stuff */
-    VNCInit(pScreen, pScrInitParms->pbits);
-
     /* Unhook this function ... */
     pScreen->CreateScreenResources = CreateScreenResources;
     dixSetPrivate(&pScreen->devPrivates, vncCreateScreenResourcesKey, NULL);
@@ -744,6 +730,9 @@ vncCreateScreenResources(ScreenPtr pScreen)
         ret = (*pScreen->CreateScreenResources)(pScreen);
     }
 
+    /* Now do our stuff */
+    VNCInit(pScreen);
+
 #ifdef DEBUG
     ErrorF("vncCreateScreenResources() returns %d\n", ret);
 #endif
diff --git a/hw/xfree86/vnc/vncInit.c b/hw/xfree86/vnc/vncInit.c
index 4a124fb..8b2fa5f 100644
--- a/hw/xfree86/vnc/vncInit.c
+++ b/hw/xfree86/vnc/vncInit.c
@@ -49,7 +49,7 @@ extern void VncExtensionInit(void);
 
 extern void vncInitMouse(void);
 extern void vncInitKeyb(void);
-Bool VNCInit(ScreenPtr pScreen, unsigned char *FBStart);
+Bool VNCInit(ScreenPtr pScreen);
 
 #ifndef XFree86LOADER
 static unsigned long VNCGeneration = 0;
@@ -151,7 +151,7 @@ void rfbLogPerror(char *str)
  * Called by vncCreateScreenResources()
  */
 Bool
-VNCInit(ScreenPtr pScreen, unsigned char *FBStart)
+VNCInit(ScreenPtr pScreen)
 {
     ScrnInfoPtr pScrn = xf86Screens[pScreen->myNum];
     VisualPtr visual;
@@ -164,9 +164,6 @@ VNCInit(ScreenPtr pScreen, unsigned char *FBStart)
     PictureScreenPtr	ps;
 #endif
 
-    if (!FBStart)
-	return FALSE;
-
 #ifndef XFree86LOADER
     if (VNCGeneration != serverGeneration) {
 	VncExtensionInit();
@@ -287,9 +284,7 @@ VNCInit(ScreenPtr pScreen, unsigned char *FBStart)
     pScreenPriv->depth = pScrn->depth;
     pScreenPriv->paddedWidthInBytes = PixmapBytePad(pScrn->displayWidth, pScrn->depth);
     pScreenPriv->bitsPerPixel = rfbBitsPerPixel(pScrn->depth);
-    pScreenPriv->pfbMemory = FBStart;
-    pScreenPriv->oldpfbMemory = FBStart;
-
+    
     pScreenPriv->cursorIsDrawn = TRUE;
     pScreenPriv->dontSendFramebufferUpdate = FALSE;
 
diff --git a/hw/xfree86/vnc/vncint.h b/hw/xfree86/vnc/vncint.h
index 18a3630..9e4a36f 100644
--- a/hw/xfree86/vnc/vncint.h
+++ b/hw/xfree86/vnc/vncint.h
@@ -44,13 +44,11 @@ typedef struct {
     size_t		buf_filled;
     int			maxFd;
     fd_set		allFds;
-    unsigned char *	oldpfbMemory;
     Bool		rfbAlwaysShared;
     Bool		rfbNeverShared;
     Bool		rfbDontDisconnect;
     Bool		rfbUserAccept;
     Bool		rfbViewOnly;
-    unsigned char *	pfbMemory;
     int 		paddedWidthInBytes;
     ColormapPtr 	rfbInstalledColormap;
     ColormapPtr		savedColormap;
-- 
1.7.3.4

Places

File n_VNC-Don-t-let-VNC-access-the-framebuffer-directly-an.patch of Package xorg-x11-server (Revision 6a68f286c9a349231c833eebde671905)

Places