You can always create form on form. It's not happiest solution, but it works. I beleive the best way to solve this problem would be by utilizing GDI+ or D2D, but unfortunately, I couldn't figure it out, so I went with "form on form" approach:
Layered form:
unit uLayeredForm;
interface
uses
Winapi.Windows, Winapi.Messages, System.SysUtils, System.Classes, System.Types,
Vcl.Graphics, Vcl.Controls, Vcl.Forms, Vcl.Dialogs, Vcl.Imaging.PngImage;
type
TfrmLayered = class(TForm)
procedure FormActivate(Sender: TObject);
private
FParentForm: TForm;
procedure SetAlphaBackground(const AResourceName: String);
public
constructor Create(AOwner: TComponent; const ABitmapResourceName: String); reintroduce;
procedure UpdatePosition;
end;
var
frmLayered: TfrmLayered;
implementation
{$R *.dfm}
constructor TfrmLayered.Create(AOwner: TComponent; const ABitmapResourceName: String);
begin
inherited Create(AOwner);
FParentForm := AOwner as TForm;
SetAlphaBackground(ABitmapResourceName);
end;
procedure TfrmLayered.FormActivate(Sender: TObject);
begin
if (Active) and (FParentForm.Visible) and (Assigned(FParentForm)) then
FParentForm.SetFocus;
end;
procedure TfrmLayered.UpdatePosition;
begin
if Assigned(FParentForm) then
begin
Left := FParentForm.Left - (ClientWidth - FParentForm.ClientWidth) div 2 - 1;
Top := FParentForm.Top - (ClientHeight - FParentForm.ClientHeight) div 2 - 1;
end;
end;
procedure TfrmLayered.SetAlphaBackground(const AResourceName: String);
var
blend_func: TBlendFunction;
imgpos : TPoint;
imgsize : TSize;
exStyle : DWORD;
png : TPngImage;
bmp : TBitmap;
begin
// enable window layering
exStyle := GetWindowLongA(Handle, GWL_EXSTYLE);
if ((exStyle and WS_EX_LAYERED) = 0) then
SetWindowLong(Handle, GWL_EXSTYLE, exStyle or WS_EX_LAYERED);
png := TPngImage.Create;
try
png.LoadFromResourceName(HInstance, AResourceName);
bmp := TBitmap.Create;
try
bmp.Assign(png);
// resize the form
ClientWidth := bmp.Width;
ClientHeight := bmp.Height;
// position image on form
imgpos := Point(0, 0);
imgsize.cx := bmp.Width;
imgsize.cy := bmp.Height;
// setup alpha blending parameters
blend_func.BlendOp := AC_SRC_OVER;
blend_func.BlendFlags := 0;
blend_func.SourceConstantAlpha := 255;
blend_func.AlphaFormat := AC_SRC_ALPHA;
UpdateLayeredWindow(Handle, 0, nil, @imgsize, bmp.Canvas.Handle, @imgpos, 0, @blend_func, ULW_ALPHA);
finally
bmp.Free;
end;
finally
png.Free;
end;
end;
end.
Main form:
unit uMainForm;
interface
uses
uLayeredForm,
Winapi.Windows, Winapi.Messages, System.SysUtils, System.Variants, System.Classes, Vcl.Graphics,
Vcl.Controls, Vcl.Forms, Vcl.Dialogs, Vcl.ExtCtrls;
type
TfrmMain = class(TForm)
imgClose: TImage;
procedure FormCreate(Sender: TObject);
procedure FormMouseDown(Sender: TObject; Button: TMouseButton; Shift: TShiftState; X, Y: Integer);
procedure FormShow(Sender: TObject);
procedure FormDestroy(Sender: TObject);
procedure FormHide(Sender: TObject);
procedure imgCloseClick(Sender: TObject);
private
FLayeredForm: TfrmLayered;
protected
procedure WMMove(var AMessage: TMessage); message WM_MOVE;
public
end;
var
frmMain: TfrmMain;
implementation
{$R *.dfm}
uses
uCommon, Vcl.Themes, Vcl.Styles.FormStyleHooks;
procedure TfrmMain.FormCreate(Sender: TObject);
begin
{$IFDEF DEBUG} ReportMemoryLeaksOnShutdown := TRUE; {$ENDIF}
FLayeredForm := TfrmLayered.Create(self, 'MainBackground');
FLayeredForm.Visible := TRUE;
end;
procedure TfrmMain.FormDestroy(Sender: TObject);
begin
FLayeredForm.Free;
end;
procedure TfrmMain.FormHide(Sender: TObject);
begin
FLayeredForm.Hide;
end;
procedure TfrmMain.WMMove(var AMessage: TMessage);
begin
if Assigned(FLayeredForm) then
FLayeredForm.UpdatePosition;
inherited;
end;
procedure TfrmMain.FormMouseDown(Sender: TObject; Button: TMouseButton; Shift: TShiftState; X, Y: Integer);
begin
FormMove(self, Button, Shift, X, Y);
end;
procedure TfrmMain.FormShow(Sender: TObject);
begin
if Assigned(FLayeredForm) then
begin
FLayeredForm.Show;
FLayeredForm.UpdatePosition;
end;
end;
procedure TfrmMain.imgCloseClick(Sender: TObject);
begin
Close;
end;
initialization
TStyleManager.Engine.RegisterStyleHook(TfrmMain, TFormStyleHookBackground);
TFormStyleHookBackground.BackGroundSettings.Color := clBlack;
TFormStyleHookBackground.BackGroundSettings.Enabled := TRUE;
end.
As you can see, you will have to do a bit of manual work to make two forms behave as one, but this code should get you started.
Since I needed form with smooth rounded borders, following screenshot is what I got as an end result. I coloured top form in gray, specifically for this post, for easier distinction between it and layered black form:

You can clearly see the difference between aliased gray form borders (made by SetWindowRgn() and CreateRoundRectRgn() APIs), and antialiased black form borders.