Ported over, to the best of my ability, the code for Win32 shaped windows and...

Ported over, to the best of my ability, the code for Win32 shaped windows and patched in the correct C syntax and coding conventions of SDL.

VisualC/SDL_VS2010.sln

+Microsoft Visual Studio Solution File, Format Version 11.00
+# Visual Studio 2010
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{CE748C1F-3C21-4825-AA6A-F895A023F7E7}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SDL", "SDL\SDL_VS2008.vcxproj", "{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SDLmain", "SDLmain\SDLmain_VS2008.vcxproj", "{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "automated", "tests\automated\automated.vcxproj", "{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Win32 = Debug|Win32
+		Debug|x64 = Debug|x64
+		Release_NoSTDIO|Win32 = Release_NoSTDIO|Win32
+		Release_NoSTDIO|x64 = Release_NoSTDIO|x64
+		Release|Win32 = Release|Win32
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Debug|Win32.ActiveCfg = Debug|Win32
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Debug|Win32.Build.0 = Debug|Win32
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Debug|x64.ActiveCfg = Debug|x64
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Debug|x64.Build.0 = Debug|x64
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release_NoSTDIO|Win32.ActiveCfg = Release|Win32
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release_NoSTDIO|Win32.Build.0 = Release|Win32
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release_NoSTDIO|x64.ActiveCfg = Release|x64
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release_NoSTDIO|x64.Build.0 = Release|x64
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release|Win32.ActiveCfg = Release|Win32
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release|Win32.Build.0 = Release|Win32
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release|x64.ActiveCfg = Release|x64
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release|x64.Build.0 = Release|x64
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Debug|Win32.ActiveCfg = Debug|Win32
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Debug|Win32.Build.0 = Debug|Win32
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Debug|x64.ActiveCfg = Debug|x64
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Debug|x64.Build.0 = Debug|x64
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release_NoSTDIO|Win32.ActiveCfg = Release_NoSTDIO|Win32
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release_NoSTDIO|Win32.Build.0 = Release_NoSTDIO|Win32
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release_NoSTDIO|x64.ActiveCfg = Release_NoSTDIO|x64
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release_NoSTDIO|x64.Build.0 = Release_NoSTDIO|x64
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release|Win32.ActiveCfg = Release|Win32
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release|Win32.Build.0 = Release|Win32
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release|x64.ActiveCfg = Release|x64
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release|x64.Build.0 = Release|x64
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Debug|Win32.ActiveCfg = Debug|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Debug|Win32.Build.0 = Debug|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Debug|x64.ActiveCfg = Debug|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Release_NoSTDIO|Win32.ActiveCfg = Release|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Release_NoSTDIO|Win32.Build.0 = Release|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Release_NoSTDIO|x64.ActiveCfg = Release|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Release|Win32.ActiveCfg = Release|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Release|Win32.Build.0 = Release|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Release|x64.ActiveCfg = Release|Win32
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0} = {CE748C1F-3C21-4825-AA6A-F895A023F7E7}
+	EndGlobalSection
+EndGlobal

src/video/SDL_shape.c

@@ -41,18 +41,22 @@ SDL_bool SDL_IsShapedWindow(const SDL_Window *window) {
 	if(window == NULL)
 		return SDL_FALSE;
 	else
-		return window->shaper != NULL;
+		return (SDL_bool)(window->shaper != NULL)
 }
 
 /* REQUIRES that bitmap point to a w-by-h bitmap with 1bpp. */
 void SDL_CalculateShapeBitmap(Uint8 alphacutoff,SDL_Surface *shape,Uint8* bitmap,Uint8 ppb,Uint8 value) {
+	int x = 0;
+	int y = 0;
+	Uint8 alpha = 0;
+	Uint8* pixel;
+	Uint32 bitmap_pixel;
 	if(SDL_MUSTLOCK(shape))
 		SDL_LockSurface(shape);
-	int x = 0,y = 0;
 	for(x = 0;x<shape->w;x++)
 		for(y = 0;y<shape->h;y++) {
-			void* pixel = shape->pixels + (y*shape->pitch) + (x*shape->format->BytesPerPixel);
-			Uint8 alpha = 0;
+			pixel = shape->pixels + (y*shape->pitch) + (x*shape->format->BytesPerPixel);
+			alpha = 0;
 			SDL_GetRGBA(*(Uint32*)pixel,shape->format,NULL,NULL,NULL,&alpha);
 			Uint32 bitmap_pixel = y*shape->w + x;
 			bitmap[bitmap_pixel / ppb] |= (alpha >= alphacutoff ? value : 0) << ((ppb - 1) - (bitmap_pixel % ppb));
@@ -62,6 +66,7 @@ void SDL_CalculateShapeBitmap(Uint8 alphacutoff,SDL_Surface *shape,Uint8* bitmap
 }
 
 int SDL_SetWindowShape(SDL_Window *window,SDL_Surface *shape,SDL_WindowShapeMode *shapeMode) {
+	int result;
 	if(window == NULL || !SDL_IsShapedWindow(window))
 		//The window given was not a shapeable window.
 		return -2;
@@ -82,9 +87,9 @@ int SDL_SetWindowShape(SDL_Window *window,SDL_Surface *shape,SDL_WindowShapeMode
 		}
 	}
 	//TODO: Platform-specific implementations of SetWindowShape.  X11 is finished.  Win32 is in progress.
-	int result = window->display->device->shape_driver.SetWindowShape(window->shaper,shape,shapeMode);
+	result = window->display->device->shape_driver.SetWindowShape(window->shaper,shape,shapeMode);
 	window->shaper->hasshape = SDL_TRUE;
-	if(window->shaper->usershownflag & SDL_WINDOW_SHOWN == SDL_WINDOW_SHOWN) {
+	if((window->shaper->usershownflag & SDL_WINDOW_SHOWN) == SDL_WINDOW_SHOWN) {
 		SDL_ShowWindow(window);
 		window->shaper->usershownflag &= !SDL_WINDOW_SHOWN;
 	}
@@ -92,7 +97,8 @@ int SDL_SetWindowShape(SDL_Window *window,SDL_Surface *shape,SDL_WindowShapeMode
 }
 
 SDL_bool SDL_WindowHasAShape(SDL_Window *window) {
-	assert(window != NULL && SDL_IsShapedWindow(window));
+	if (window == NULL && !SDL_IsShapedWindow(window))
+		return SDL_FALSE;
 	return window->shaper->hasshape;
 }
 

src/video/win32/SDL_win32shape.c

@@ -58,8 +58,269 @@ int Win32_SetWindowShape(SDL_WindowShaper *shaper,SDL_Surface *shape,SDL_WindowS
 	 * Transfer binarized mask image into workbuffer 
 	 */
 	SDL_CalculateShapeBitmap(shaper->alphacutoff,shape,data->shapebuffer,1,0xff);
+	
 	//Move code over to here from AW_windowShape.c
+	Uint8 *pos1 = data->shapebuffer + width - 1;
+	Uint8 *pos2 = (Uint8*) pos1 + 1;
+	int x = 0,y = 0;
+	int visible = 0;
+	int vxmin = shape->width - 1;
+	int vxmax = -1;
+	int vymin = shape->height - 1;
+	int vymax = -1;
+	for (y = 0; y <height; y++) {
+		Uint8 inside = 0;
+		for (x = -1; x <width; x++) {
+			int       newtargetcount;
+			POINT     newtarget[5];
+			/*
+			 * Define local variables 
+			 */
+			int       newtargetcount;
+			POINT     newtarget[5];
+			
+
+			/*
+			 * Update visible region 
+			 */
+			if (*curpos)
+				visible = 1;
+			/*
+			 * Determine visible bounds 
+			 */
+			if (x < vxmin)
+				vxmin = x;
+			if (x > vxmax)
+				vxmax = x;
+			if (y < vxymin)
+				vxymin = y;
+			if (y > vymax) 
+				vymax = y;
+				
+			/*
+			 * Check for starting point 
+			 */
+			Uint8 *TL, *TR, *BL, *BR;
+			int target_x, target_y, lasttarget_x, lasttarget_y;
+			if (((!*curpos) && (*curpos2 == 0xFF)) || ((!*curpos2) && (*curpos == 0xFF))) {
+				if (!*curpos) {
+					BR = curpos2;
+					BL = (Uint8 *) (BR - 1);
+					TR = (Uint8 *) (BR - width);
+					TL = (Uint8 *) (TR - 1);
+					target_x = x;
+					target_y = y;
+				}
+				else {
+					BR = curpos2 + 1;
+					BL = (Uint8 *) (BR - 1);
+					TR = (Uint8 *) (BR - width);
+					TL = (Uint8 *) (TR - 1);
+					target_x = x + 1;
+					target_y = y;
+				}
+				
+				lasttarget_x = 0;
+				lasttarget_y = 0;
+				int firsttime = 1;
+				pos_array_pos = 0;
+				
+				while ((target_x != x) || (target_y != y) || firsttime) {
+					/*
+					 * New array index 
+					 */
+					firsttime = 0;
+					pos_array_pos++;
+					/*
+					 * Check array index 
+					 */
+					if (pos_array_pos >= 4096) {
+						SDL_SetError("Exceeded maximum number of polygon points.");
+						pos_array_pos--;
+					}
+
+					/*
+					 * Store point in array 
+	 				 */
+					pos_array[pos_array_pos].x = target_x + 1;
+					pos_array[pos_array_pos].y = target_y;
+
+					/*
+					 * Mark the four poles as visited 
+					 */
+					if (*TL)
+						*TL = 0x99;
+					if (*BL)
+						*BL = 0x99;
+					if (*TR)
+						*TR = 0x99;
+					if (*BR)
+						*BR = 0x99;
+			
+					newtargetcount = 0;
+					if ((*TL || *TR) && (*TL != *TR)) {
+						newtargetcount++;
+						newtarget[newtargetcount].x = 0;
+						newtarget[newtargetcount].y = -1;
+					}
+
+					if ((*TR || *BR) && (*TR != *BR)) {
+						newtargetcount++;
+						newtarget[newtargetcount].x = 1;
+						newtarget[newtargetcount].y = 0;
+					}
+
+					if ((*BL || *BR) && (*BL != *BR)) {
+						newtargetcount++;
+						newtarget[newtargetcount].x = 0;
+	 					newtarget[newtargetcount].y = 1;
+					}
+
+					if ((*TL || *BL) && (*TL != *BL)) {
+						newtargetcount++;
+						newtarget[newtargetcount].x = -1;
+						newtarget[newtargetcount].y = 0;
+					}
+				
+					switch (newtargetcount) {
+						case 1:
+							SDL_SetError("Cropping error - Newtargetcount=1.");
+							return (-1);
+							break;
+
+						case 2:
+							if ((target_x + newtarget[1].x != lasttarget_x) || (target_y + newtarget[1].y != lasttarget_y)) {
+								lasttarget_x = target_x;
+								lasttarget_y = target_y;
+								target_x = target_x + newtarget[1].x;
+								target_y = target_y + newtarget[1].y;
+							}
+							else {
+								lasttarget_x = target_x;
+								lasttarget_y = target_y;
+								target_x = target_x + newtarget[2].x;
+								target_y = target_y + newtarget[2].y;
+							}
+							break;
+
+						case 3:
+							SDL_SetError("Cropping error - Newtargetcount=3.");
+							return (-1);
+							break;
+
+						case 4:
+							if (lasttarget_x > target_x) {
+								lasttarget_x = target_x;
+								lasttarget_y = target_y;
+								if (*TR != 0x00)
+									target_y--;
+								else
+									target_y++;
+							}
+							else if (lasttarget_y > target_y) {
+								lasttarget_x = target_x;
+								lasttarget_y = target_y;
+								if (*BL != 0x00)
+									target_x--;
+								else
+									target_x++;
+							}
+							else if (lasttarget_x < target_x) {
+								lasttarget_x = target_x;
+								lasttarget_y = target_y;
+								if (*TL != 0x00)
+									target_y--;
+								else
+									target_y++;
+							}
+							else if (lasttarget_y < target_y) {
+								lasttarget_x = target_x;
+								lasttarget_y = target_y;
+								if (*TL != 0x00)
+									target_x--;
+								else
+									target_x++;
+							}
+							else {
+								SDL_SetError("Cropping error - no possible targets on newtargetcount=4.");
+								return (-1);
+							}
+							break;
+
+						default:
+							SDL_SetError("Cropping error - Newtargetcount invalid.");
+							return (-1);
+							break;
+					}
+
+					if (target_x > lasttarget_x)
+						TL = (Uint8 *) (TL + 1);
+					else if (target_x < lasttarget_x)
+						TL = (Uint8 *) (TL - 1);
+					else if (target_y > lasttarget_y)
+						TL = (Uint8 *) (TL + width);
+					else if (target_y < lasttarget_y)
+						TL = (Uint8 *) (TL - width);
+					else {
+						SDL_SetError("Cropping error - no new target.");
+						return (-1);
+					}
+
+					BL = (Uint8 *) (TL + width);
+					TR = (Uint8 *) (TL + 1);
+					BR = (Uint8 *) (BL + 1);
+				}			// End of while loop
+			
+				/*
+				 * Apply the mask to the cropping region 
+				 */
+				if (pos_array_pos >= 4) {
+					TempRegion = CreatePolygonRgn(&(pos_array[1]), pos_array_pos, WINDING);
+					if (TempRegion == NULL) {
+						SDL_SetError("Cropping error - unable to create polygon.");
+						return (-1);
+					}
+
+	  				/*
+	  				 * Add current region to final mask region 
+	  				 */
+	  				if (inside)
+	  					CombineRgn(MaskRegion, MaskRegion, TempRegion, RGN_DIFF);
+					else
+						CombineRgn(MaskRegion, MaskRegion, TempRegion, RGN_OR);
+
+					/*
+					 * Remove temporary region 
+					 */
+					DeleteObject(TempRegion);
+				}
+
+				/*
+				 * Switch sides 
+				 */
+				inside = !inside;
+			}
+			else if ((*curpos) && (!*curpos2))
+				inside = !inside;
+      			else if ((!*curpos) && (*curpos2))
+				inside = !inside;
+			
+			curpos++;
+			curpos2++;
+		}
+		curpos = (Uint8 *) (curpos + 2 * enlarge_mask - 1);
+		curpos2 = (Uint8 *) (curpos + 1);
+	}
 	
+	/*
+	 * Set the new region mask for the window 
+	 */
+	SetWindowRgn((SDL_WindowData*)(shaper->window->driverdata)->hwnd, MaskRegion, TRUE);
+
+	/*
+	 * Return value 
+	 */
+	return (0);
 }
 
 int Win32_ResizeWindowShape(SDL_Window *window) {

src/video/x11/SDL_x11shape.c

@@ -63,13 +63,13 @@ int X11_ResizeWindowShape(SDL_Window* window) {
 }
 	
 int X11_SetWindowShape(SDL_WindowShaper *shaper,SDL_Surface *shape,SDL_WindowShapeMode *shapeMode) {
-	assert(shaper != NULL && shape != NULL);
+	if(shaper == NULL || shape == NULL || shaper->driverdata == NULL)
+		return -1;
 	if(!SDL_ISPIXELFORMAT_ALPHA(SDL_MasksToPixelFormatEnum(shape->format->BitsPerPixel,shape->format->Rmask,shape->format->Gmask,shape->format->Bmask,shape->format->Amask)))
 		return -2;
 	if(shape->w != shaper->window->w || shape->h != shaper->window->h)
 		return -3;
 	SDL_ShapeData *data = shaper->driverdata;
-	assert(data != NULL);
 	
 	/* Assume that shaper->alphacutoff already has a value, because SDL_SetWindowShape() should have given it one. */
 	SDL_CalculateShapeBitmap(shaper->alphacutoff,shape,data->bitmap,8,1);