UnhookWindowsHookEx produces ERROR_INVALID_HOOK_HANDLE?

Question

I'm trying to implement a background application in WPF that does some things only when right-clicking the top of the screen, so I tried hooking WH_MOUSE_LL, which works great. The problem is unhooking the callback when exiting the application, which works perfectly fine in C, but gives me ERROR_INVALID_HOOK_HANDLE in C# even though the hook handle passed to UnhookWindowsHookEx is exactly the same handle I got from SetWindowsHookEx (examples below).

EDIT: I just created a bare bones C# console application registering the same hook and instantly unregistering it and it works fine too, so I think WPF might cause the problems.

Here is the C# code that does not work, problem is in the destructor:

using System;
using System.Runtime.InteropServices;

public class LowLevelMouseHook {
    #region Win32 API
    private const int WH_MOUSE_LL = 14;
    private const ulong WM_RBUTTONDOWN = 0x0204;

    [UnmanagedFunctionPointer(CallingConvention.StdCall)]
    private delegate long LowLevelMouseProc (int nCode, UIntPtr wParam, IntPtr lParam);

    private struct POINT {
        long x;
        long y;
    }

    private struct MSLLHOOKSTRUCT {
        POINT pt;
        int mouseData;
        int flags;
        int time;
        UIntPtr dwExtraInfo;
    }

    [DllImport("User32.dll", EntryPoint = "CallNextHookEx", SetLastError = true)]
    private static extern long CallNextHookEx (IntPtr hHook, int nCode, UIntPtr wParam, IntPtr lParam);

    [DllImport("Kernel32.dll", EntryPoint = "GetLastError", SetLastError = true)]
    private static extern uint GetLastError ();

    [DllImport("Kernel32.dll", EntryPoint = "LoadLibraryW", CharSet = CharSet.Unicode, SetLastError = true)]
    private static extern IntPtr LoadLibrary (string lpFileName);

    [DllImport("User32.dll", EntryPoint = "SetWindowsHookExW", SetLastError = true)]
    private static extern IntPtr SetWindowsHookEx (int idHook, Delegate lpfn, IntPtr hMod, uint dwThreadId);

    [DllImport("User32.dll", EntryPoint = "UnhookWindowsHookEx", SetLastError = true)]
    private static extern byte UnhookWindowsHookEx (IntPtr hHook);
    #endregion

    // There shall only be one instance of this class
    private static LowLevelMouseHook instance = null;

    private IntPtr hHook;
    private LowLevelMouseProc hProc; // If I don't store the delegate, I get some CallbackOnCollectedDelegate exception

    private LowLevelMouseHook () {
        this.hProc = new LowLevelMouseProc(MouseProc);
        this.hHook = SetWindowsHookEx(WH_MOUSE_LL, this.hProc, LoadLibrary("User32.dll"), 0);
    }

    ~LowLevelMouseHook () {
        //
        // PROBLEM:
        // UnhookWindowsHookEx always returns 0 and GetLastError returns ERROR_INVALID_HOOK_HANDLE (1404)
        // even though this.hHook is still the same value SetWindowsHookEx returned.
        //
        if (UnhookWindowsHookEx(this.hHook) == 0)
            System.Windows.Forms.MessageBox.Show($"Error {GetLastError()} unhooking WH_MOUSE_LL", "UnhookWindowsHookEx Error");

        LowLevelMouseHook.instance = null;
    }

    // Create new LowLevelMouseHook instance.
    // This is called during the Startup-Event of Application.
    public static void Init () {
        LowLevelMouseHook.instance = new LowLevelMouseHook();
    }

    private static long MouseProc (int nCode, UIntPtr wParam, IntPtr lParam) {
        if ((ulong)wParam == WM_RBUTTONDOWN) {
            ; // Things to do; that works fine
        }

        return CallNextHookEx(LowLevelMouseHook.instance.hHook, nCode, wParam, lParam);
    }
}

Here is the C code that works perfectly fine:

#include <stdio.h>
#include <Windows.h>

FILE *file;
HHOOK hHook;

LRESULT CALLBACK LowLevelMouseProc_Test (int nCode, WPARAM wParam, LPARAM lParam) {
    PMSLLHOOKSTRUCT pHookStruct = (PMSLLHOOKSTRUCT)lParam;
    fprintf(file, "MouseProc_Test(nCode = %i, wParam = %lu, lParam->pt = (%i, %i))\n", nCode, wParam, pHookStruct->pt.x, pHookStruct->pt.y);
    return CallNextHookEx(hHook, nCode, wParam, lParam);
}

VOID CALLBACK TimerProc_Test (HWND hWnd, UINT uMsg, UINT_PTR idEvent, DWORD dwTime) {
    PostQuitMessage(0);
}

int WINAPI WinMain (HINSTANCE hInstance, HINSTANCE hPrevInstance, PSTR szCmdLine, int nShow) {
    fopen_s(&file, "wm_mouse_ll.txt", "w");
    hHook = SetWindowsHookExW(WH_MOUSE_LL, LowLevelMouseProc_Test, LoadLibrary("User32.dll"), 0);

    if (hHook != 0) {
        fprintf(file, "SetWindowsHookExW Success (hHook = 0x%p)\n", hHook);

        SetTimer(NULL, 0, 5000, TimerProc_Test);

        MSG msg = {0};
        while (GetMessageW(&msg, NULL, 0, 0) != 0) {
            TranslateMessage(&msg);
            DispatchMessageW(&msg);
        }

        if (UnhookWindowsHookEx(hHook) != 0)
            fputs("UnhookWindowsHookEx Success\n", file);
        else
            fprintf(file, "UnhookWindowsHookEx Error %u\n", GetLastError());
    } else {
        fprintf(file, "SetWindowsHookExW Error %u\n", GetLastError());
    }

    fclose(file);
    return 0;
}

score 0 · Accepted Answer · answered Aug 04 '18 at 18:57

Ok, I simply restructured LowLevelMouseHook a little bit and added a custom Main() method that looks like the one below. UnhookWindowsHookEx now succeeds.

public static class EntryPoint {
    public static void Main () {
        LowLevelMouseHook.Begin() // Here SetWindowsHookEx is called
        App.Main()                // Original entrypoint
        LowLevelMouseHook.End()   // Here UnhookWindowsHookEx is called
    }
}

UnhookWindowsHookEx produces ERROR_INVALID_HOOK_HANDLE?

1 Answers1