Is it possible to ignore all authentication schemes if authorization fails and just return a 401 errorcode?

Question

I am developing a web-api which uses some authorization against another of our applications to verify that a user is still logged in and may continue with the action they've requested. But the server responds with a 500 internal server error when the user is not Authenticated, stack trace can be found below. I seem to be unable to wrestle the configuration into a correct state. From the stack trace, I can tell that it tries to fall back to some default authentication scheme, which I haven't defined and ideally aren't interested in as we always wish to authorize against the 3rd party instead and simply fail if it returns false.

Startup.cs

public void ConfigureServices(IServiceCollection services)
{
    services.AddScoped<IAuthService>(_ =>
        new AuthService(damInfo.BaseUrl, damInfo.SystemUsername, damInfo.SystemPassword));
    services.AddScoped<IAuthorizationHandler, AuthenticationHandler>();

    services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_1);

    services.AddAuthorization(options =>
    {
        options.AddPolicy("ServicePolicy", policy =>
            policy.Requirements.Add(new ServiceRequirement("")));
        /*options.DefaultPolicy = new AuthorizationPolicyBuilder().AddAuthenticationSchemes(Constants.NoOp).RequireAuthenticatedUser().Build(); // keep#1#*/
        options.InvokeHandlersAfterFailure = false;
    });

    services.AddAuthentication(options =>
    {
        options.DefaultScheme = Constants.NoOp;
        options.DefaultChallengeScheme = Constants.NoOp;
    });
}

public void Configure(IApplicationBuilder app, IHostingEnvironment env)
{
    if (env.IsDevelopment())
        app.UseDeveloperExceptionPage();
    else
        app.UseHsts();

    app.UseCors(builder =>
    {
        builder.AllowAnyHeader();
        builder.AllowAnyMethod();
        builder.AllowAnyOrigin();
    });

    app.UseHttpsRedirection();
    app.UseMvc();
}

Policy:

using Microsoft.AspNetCore.Authorization;

namespace Core.Services.Authentication
{
    public class ServiceRequirement : IAuthorizationRequirement
    {
        public string CurrentToken { get; private set; }

        public ServiceRequirement(string token)
        {
            CurrentToken = token;
        }       
    }
}

Authorize attribute:

[Authorize(Policy = "ServicePolicy")]

Auth handler:

using System.Net.Http.Headers;
using System.Threading.Tasks;
using Digizuite.Services.Authentication;
using Microsoft.AspNetCore.Authorization;
using Microsoft.Extensions.Primitives;
using Microsoft.Net.Http.Headers;

namespace Core.Services.Authentication
{
    public class AuthenticationHandler : AuthorizationHandler<ServiceRequirement>
    {
        readonly IAuthService _authService;

        /// <summary>
        /// Authentication handler that check if user is logged in
        /// </summary>
        /// <param name="authService"></param>
        public AuthenticationHandler(IAuthService authService)
        {
            _authService = authService;
        }

        /// <summary>
        /// Handle requirement and check isLoggedIn
        /// </summary>
        /// <param name="context"></param>
        /// <param name="requirement"></param>
        /// <returns></returns>
        protected override async Task HandleRequirementAsync(AuthorizationHandlerContext context,
            ServiceRequirement requirement)
        {
            var authFilterCtx = (Microsoft.AspNetCore.Mvc.Filters.AuthorizationFilterContext)context.Resource;
            var httpContext = authFilterCtx.HttpContext;

            var headers = httpContext.Request.Headers[HeaderNames.Authorization];

            if (StringValues.IsNullOrEmpty(headers) == true)
            {
                context.Fail();
            }
            else
            {
                var auth = AuthenticationHeaderValue.Parse(headers);
                var loginResponse = _authService.IsLoggedInRequest(auth.Parameter).Result;
                if (loginResponse.IsLoggedIn == false)
                {
                    context.Fail();
                }
                else
                {
                    context.Succeed(requirement);
                }
            }
        }
    }
}

StackTrace:

fail: Microsoft.AspNetCore.Server.Kestrel[13]
      Connection id "0HLOC48NR5MP4", Request id "0HLOC48NR5MP4:00000001": An unhandled exception was thrown by the application.
System.InvalidOperationException: No authenticationScheme was specified, and there was no DefaultChallengeScheme found.
   at Microsoft.AspNetCore.Authentication.AuthenticationService.ChallengeAsync(HttpContext context, String scheme, AuthenticationProperties properties)
   at Microsoft.AspNetCore.Mvc.ChallengeResult.ExecuteResultAsync(ActionContext context)
   at Microsoft.AspNetCore.Mvc.Internal.ResourceInvoker.InvokeResultAsync(IActionResult result)
   at Microsoft.AspNetCore.Mvc.Internal.ResourceInvoker.InvokeAlwaysRunResultFilters()
   at Microsoft.AspNetCore.Mvc.Internal.ResourceInvoker.InvokeFilterPipelineAsync()
   at Microsoft.AspNetCore.Mvc.Internal.ResourceInvoker.InvokeAsync()
   at Microsoft.AspNetCore.Builder.RouterMiddleware.Invoke(HttpContext httpContext)
   at Microsoft.AspNetCore.Cors.Infrastructure.CorsMiddleware.Invoke(HttpContext context)
   at Microsoft.AspNetCore.Authentication.AuthenticationMiddleware.Invoke(HttpContext context)
   at Digizuite.Logging2.LogClient.LoggerRequestMiddleware.InvokeAsync(HttpContext context, RequestDelegate next) in C:\dev\digizuite.core\Libs\LogClient\LoggerRequestMiddleware.cs:line 40
   at Microsoft.AspNetCore.Builder.UseMiddlewareExtensions.<>c__DisplayClass5_1.<<UseMiddlewareInterface>b__1>d.MoveNext()