For each point rotate it by using this matrix:

that is what you are doing and it is OK.
And translate it by using

And that what you are doing and that is ok as well.
That is your code with the fixed code, mentioned by the others:
#include <Windows.h>
#include <math.h>
#define ID_TIMER 1
LRESULT CALLBACK WndProc(HWND , UINT , WPARAM , LPARAM );
void WINAPI Rotate(POINT arr[5]);
int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, PSTR szCmdLine, int cmdShow)
{
TCHAR szAppName[] = TEXT("HelloWin32");
HWND hWnd;
MSG msg;
WNDCLASS wndClass;
wndClass.style = CS_HREDRAW | CS_VREDRAW;
wndClass.lpfnWndProc = WndProc;
wndClass.cbClsExtra = wndClass.cbWndExtra = 0;
wndClass.hInstance = hInstance;
wndClass.hIcon = LoadIcon(0, IDI_APPLICATION);
wndClass.hCursor = LoadCursor(0, IDC_ARROW);
wndClass.hbrBackground = (HBRUSH) GetStockObject(1);
wndClass.lpszMenuName = 0;
wndClass.lpszClassName = szAppName;
if (!RegisterClass(&wndClass))
{
MessageBox(0, TEXT("Failed to register window class"), TEXT("Error"), MB_OK | MB_DEFBUTTON1 | MB_ICONERROR);
return 1;
}
hWnd = CreateWindow(szAppName, TEXT("Hello World Win32!!"), WS_OVERLAPPEDWINDOW,
CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT,
NULL, NULL, hInstance, NULL);
if (!hWnd)
{
MessageBox(NULL, TEXT("Failed to create the window"), TEXT("Win32 Error"), MB_ICONERROR);
return 1;
}
SetTimer(hWnd, ID_TIMER, 400, NULL);
ShowWindow(hWnd, cmdShow);
UpdateWindow(hWnd);
while (GetMessage(&msg, NULL, 0, 0))
{
TranslateMessage(&msg);
DispatchMessage(&msg);
}
return msg.wParam;
}
LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM w, LPARAM l)
{
static HDC hdc;
static PAINTSTRUCT ps;
static POINT arr[5];
static int i;
static RECT rect;
switch (message)
{
case WM_CREATE:
GetClientRect(hWnd, &rect);
arr[0].x = 200;
arr[0].y = 100;
arr[1].x = 200;
arr[1].y = 200;
arr[2].x = 100;
arr[2].y = 200;
arr[3].x = 100;
arr[3].y = 100;
arr[4].x = arr[0].x;
arr[4].y = arr[0].y;
return 0;
case WM_TIMER:
Rotate(arr);
InvalidateRect(hWnd, &rect, TRUE);
return 0;
case WM_DESTROY:
KillTimer(hWnd, ID_TIMER);
PostQuitMessage(0);
return 0;
case WM_PAINT:
hdc = BeginPaint(hWnd, &ps);
MoveToEx(hdc, arr[0].x, arr[0].y, NULL);
for (i = 1; i < 5; ++i) LineTo(hdc, arr[i].x, arr[i].y);
EndPaint(hWnd, &ps);
return 0;
case WM_SIZE:
GetClientRect(hWnd, &rect);
return 0;
default:
return DefWindowProc(hWnd, message, w, l);
}
}
void WINAPI Rotate(POINT arr[5])
{
static const POINT origin = { 150, 150 };
static int i;
static const double angle = 0.3;
static const int direction = 1; // 1 or -1
for (i = 0; i < 5; ++i)
{
POINT temp;
arr[i].x -= origin.x;
arr[i].y -= origin.y;
temp.x = arr[i].x;
temp.y = arr[i].y;
arr[i].x = ceil(arr[i].x * cos(angle)) - ceil(direction * arr[i].y * sin(angle));
arr[i].y = ceil(direction * temp.x * sin(angle)) + ceil(temp.y * cos(angle));
arr[i].x += origin.x;
arr[i].y += origin.y;
}
}
The changes I made:
for (i = 0; i < 5; ++i)
{
POINT temp;
arr[i].x -= origin.x;
arr[i].y -= origin.y;
temp.x = arr[i].x;
temp.y = arr[i].y;
arr[i].x = ceil(arr[i].x * cos(angle)) - ceil(direction * arr[i].y * sin(angle));
arr[i].y = ceil(direction * temp.x * sin(angle)) + ceil(temp.y * cos(angle));
arr[i].x += origin.x;
arr[i].y += origin.y;
}
while declare the points like that:
arr[0].x = 200;
arr[0].y = 100;
arr[1].x = 200;
arr[1].y = 200;
arr[2].x = 100;
arr[2].y = 200;
arr[3].x = 100;
arr[3].y = 100;
arr[4].x = arr[0].x;
arr[4].y = arr[0].y;
That is a win32 project in visual studio
Comment:
Could be nice solution for people who look for:
How to rotate a rectangular in win32 application using WINAPI