Validate Azure AD generated JWT token's signature in .NetCore

Question

Largely following on the comment here. I have the below code. (V1 token) and validation in .NetCore 3.1. This always returns false and I am not able to figure. Could someone help to find out what I am doing incorrect here. TIA

public async Task<bool> ValidateAsync(string token)
    {
        ConfigurationManager<OpenIdConnectConfiguration> configManager = new ConfigurationManager<OpenIdConnectConfiguration>(this.tokenSettings.StsDiscoveryEndpoint, new OpenIdConnectConfigurationRetriever());
        OpenIdConnectConfiguration config = configManager.GetConfigurationAsync(CancellationToken.None).Result;
        var signingKeys = config.SigningKeys;
        TokenValidationParameters validationParameters = new TokenValidationParameters
        {
            ValidIssuer = this.tokenSettings.IssuerValidator,
            ValidAudience = this.tokenSettings.Audience,
            IssuerSigningKeys = signingKeys,
            ValidateAudience = true,
            ValidateIssuer = true,
            ValidateLifetime = true,
            SignatureValidator = (token, parameter) => new JwtSecurityToken(token)
        };
        var allParts = token.Split(".");
        var header = allParts[0];
        var payload = allParts[1];
        var signature = allParts[2];
        using (var client = new HttpClient())
        {
            discoveryKeys = await client.GetStringAsync("https://login.microsoftonline.com/common/discovery/keys");
        }
        var azureKeys = JObject.Parse(discoveryKeys);
        var kidKey = JObject.Parse(Encoding.UTF8.GetString(Convert.FromBase64String(header)))["kid"].ToString();
        var signatureKeyIdentifier = azureKeys["keys"].Children().Where(x => x["kid"].ToString() == kidKey);
        var signatureKey = signatureKeyIdentifier.Select(x => x["x5c"]).FirstOrDefault().ToArray()[0].ToObject<byte[]>();

        using (X509Certificate2 certificate = new X509Certificate2(signatureKey))
        {
            byte[] bytesToSign = System.Text.UTF8Encoding.UTF8.GetBytes(string.Join(".", header, payload));
            byte[] signatureToCheck = System.Text.UTF8Encoding.UTF8.GetBytes(signature);
            bool isValid;
            using (RSA rsa = certificate.GetRSAPublicKey())
            {
                try
                {
                    isValid = rsa.VerifyHash(bytesToSign, signatureToCheck, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
                }
                catch (Exception e)
                {

                }
            }
        }
        return isValid;
        }