HTTP GET request to API behind AzureAD authentication with ASP.NET Core MVC

Question

The code below gets a token which I then use to try and fetch some data from an API which is behind AzureAD authentication. I get a token back, but when I use it to try and reach the API, I get "login to your account" in apiResponse.

What is wrong with my authorization?

    var recoAadAppId = "xxxxxxxxxxxxxx";
    var callerAadAppId = "xxxxxxxxxxxxxx";
    var callerAadTenantId = "xxxxxxxxxxxxxx";
    var token = await AcquireTokenWithSecret(callerAadAppId, callerAadTenantId, recoAadAppId);
    
    var httpClient = new HttpClient();
    httpClient.DefaultRequestHeaders.Authorization = AuthenticationHeaderValue.Parse(token.CreateAuthorizationHeader());
    using (var response = await httpClient.GetAsync("https://redacted/app/rest/buildQueue"))
    {
        string apiResponse = await response.Content.ReadAsStringAsync();
    }

    public static Task<AuthenticationResult> AcquireTokenWithSecret(
           string callerAadAppId, string callerTenantId, string recoAadAppId)
    {
        var secret = "mysecret";
        var app = ConfidentialClientApplicationBuilder.Create(callerAadAppId).WithAuthority($"https://login.microsoftonline.com/{callerTenantId}").WithClientSecret(secret).Build();
        var scopes = new[] { $"{recoAadAppId}/.default" };
        return app.AcquireTokenForClient(scopes).ExecuteAsync(CancellationToken.None);
    }