we are currently working on a server program written in Java that uses TLS. To achieve this, we used the Java SSLServerSocket. However, everytime a client tried to connect to the server, it caused a javax.net.ssl.SSLHandshakeException when the server attempted to start the handshake. To resolve this, we extracted the relevant code part and wrote a debugging server and client program, but still the same errors occurred.
We have the latest version of Java installed (8u111) and it's a regular Oracle version. The cipher suite we chose should definitely be supported since it was on the list from SSLSocket.getSupportedCipherSuites().
We read other threads on this forum suggesting various things like disabling ciphers using Diffie-Hellman or using a cipher with RSA signing instead of ECDSA, but nothing fixed it.
The server program
package ch.debugging.server
import org.bouncycastle.asn1.x500.X500Name;
import org.bouncycastle.cert.X509CertificateHolder;
import org.bouncycastle.cert.X509v3CertificateBuilder;
import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.operator.ContentSigner;
import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
import org.bouncycastle.util.encoders.Base64;
import javax.net.ssl.*;
import java.io.*;
import java.math.BigInteger;
import java.net.BindException;
import java.security.*;
import java.security.cert.X509Certificate;
import java.util.Date;
public class Main {
public static void main(String[] args) {
KeyStore keystore = null;
try {
BouncyCastleProvider bc = new BouncyCastleProvider();
File keystoreFile = new File("D:\\keystore.asdf");
keystore = KeyStore.getInstance("JKS");
//Create new keystore if there is none
if (!keystoreFile.exists()) {
SecureRandom r = SecureRandom.getInstanceStrong();
KeyPairGenerator keygen = KeyPairGenerator.getInstance("EC", bc);
keygen.initialize(384, r);
KeyPair keys = keygen.generateKeyPair();
System.out.println("Keys successfully generated");
//Parameters for the certificate
X500Name issuer = new X500Name("C=, ST=, L=, O=, OU=, CN=");
X500Name subject = issuer; //self-signed certificate: issuer and subject are the same
BigInteger serial = BigInteger.valueOf(r.nextLong());
Date notBefore = new Date(System.currentTimeMillis());
Date notAfter = new Date(System.currentTimeMillis() + (1000 * 3600 * 24 * 365 * 10)); //valid for one year from now
//Build the certificate
X509v3CertificateBuilder certGen = new JcaX509v3CertificateBuilder(issuer, serial, notBefore, notAfter, subject, keys.getPublic());
ContentSigner signer = new JcaContentSignerBuilder("SHA512WithECDSA").build(keys.getPrivate());
X509CertificateHolder certHolder = certGen.build(signer);
X509Certificate[] certChain = {new JcaX509CertificateConverter().setProvider(bc).getCertificate(certHolder)};
System.out.println(Base64.toBase64String(certChain[0].getEncoded()));
System.out.println("Certificate built");
//Save keys and the certificate to keystore
keystore.load(null, "123456".toCharArray());
keystore.setCertificateEntry("Certificate", certChain[0]);
keystore.store(new FileOutputStream(keystoreFile), "123456".toCharArray());
System.out.println("Certificate saved in keystore");
}
//load keystore if it exists
else {
keystore.load(new FileInputStream(keystoreFile), "123456".toCharArray());
System.out.println("Keystore loaded");
}
} catch(Exception e) {
e.printStackTrace();
System.exit(1);
}
try {
SecureRandom r = SecureRandom.getInstanceStrong();
//Prepare SSLServerSocket
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
kmf.init(keystore,"123456".toCharArray());
tmf.init(keystore);
SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
sslContext.init(kmf.getKeyManagers(),tmf.getTrustManagers(),r);
//Create and set up server socket
SSLServerSocketFactory factory = sslContext.getServerSocketFactory();
SSLServerSocket serverSocket = (SSLServerSocket) factory.createServerSocket(6060);
serverSocket.setEnabledProtocols(new String[]{"TLSv1.2"});
serverSocket.setEnabledCipherSuites(new String[]{"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"});
serverSocket.setEnableSessionCreation(true);
serverSocket.setNeedClientAuth(false);
System.out.println("Waiting for incoming connections...");
SSLSocket s = (SSLSocket) serverSocket.accept();
s.startHandshake();
System.out.println("------Connection established------");
BufferedReader in = new BufferedReader(new InputStreamReader(s.getInputStream()));
while (true) {
System.out.println(in.readLine());
}
} catch(BindException bind) {
System.out.println("Failed to bind port 6060. Is it already in use?");
bind.printStackTrace();
} catch(IOException io) {
io.printStackTrace();
} catch(Exception kse) {
kse.printStackTrace();
}
}
}
The client program
import javax.net.ssl.*;
import javax.swing.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.PrintWriter;
import java.net.InetSocketAddress;
import java.security.KeyStore;
import java.security.SecureRandom;
public class main {
public static void main(String[] args) {
try {
//Load keystore
KeyStore keystore = KeyStore.getInstance("JKS");
keystore.load(new FileInputStream(new File("D:\\keystore.asdf")), "123456".toCharArray());
//Prepare SSLServerSocket
SecureRandom r = SecureRandom.getInstanceStrong();
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
kmf.init(keystore,"123456".toCharArray());
tmf.init(keystore);
//initialize SSLContext
SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
sslContext.init(kmf.getKeyManagers(),tmf.getTrustManagers(),r);
SSLSocketFactory factory = sslContext.getSocketFactory();
SSLSocket socket = (SSLSocket) factory.createSocket();
socket.setEnabledProtocols(new String[]{"TLSv1.2"});
socket.setEnabledCipherSuites(new String[]{"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"});
socket.setEnableSessionCreation(true);
socket.connect(new InetSocketAddress("localhost",6060));
socket.startHandshake();
PrintWriter out = new PrintWriter(socket.getOutputStream(),true);
while (true) {
String msg = JOptionPane.showInputDialog(null,"Message");
if (!msg.equalsIgnoreCase("")) {
out.println(msg);
} else {
socket.close();
break;
}
}
} catch(Exception e) {
e.printStackTrace();
}
}
}
The output of the server program (with -Djavax.net.debug=ssl,handshake)
Keystore loaded
adding as trusted cert:
Subject: CN=, OU=, O=, L=, ST=, C=
Issuer: CN=, OU=, O=, L=, ST=, C=
Algorithm: EC; Serial number: 0x-dde391bb99f8ce8
Valid from Fri Dec 30 13:55:36 CET 2016 until Fri Jan 20 17:32:03 CET 2017
trigger seeding of SecureRandom
done seeding SecureRandom
Waiting for incoming connections...
Allow unsafe renegotiation: false
Allow legacy hello messages: true
Is initial handshake: true
Is secure renegotiation: false
main, READ: TLSv1.2 Handshake, length = 140
*** ClientHello, TLSv1.2
RandomCookie: GMT: 1483105145 bytes = { 70, 211, 55, 52, 24, 196, 174, 198, 164, 57, 157, 130, 143, 21, 91, 86, 53, 112, 138, 55, 208, 249, 218, 82, 245, 224, 32, 184 }
Session ID: {}
Cipher Suites: [TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384]
Compression Methods: { 0 }
Extension elliptic_curves, curve names: {secp256r1, sect163k1, sect163r2, secp192r1, secp224r1, sect233k1, sect233r1, sect283k1, sect283r1, secp384r1, sect409k1, sect409r1, secp521r1, sect571k1, sect571r1, secp160k1, secp160r1, secp160r2, sect163r1, secp192k1, sect193r1, sect193r2, secp224k1, sect239k1, secp256k1}
Extension ec_point_formats, formats: [uncompressed]
Extension signature_algorithms, signature_algorithms: SHA512withECDSA, SHA512withRSA, SHA384withECDSA, SHA384withRSA, SHA256withECDSA, SHA256withRSA, SHA256withDSA, SHA1withECDSA, SHA1withRSA, SHA1withDSA
Extension renegotiation_info, renegotiated_connection:
%% Initialized: [Session-1, SSL_NULL_WITH_NULL_NULL]
%% Invalidated: [Session-1, SSL_NULL_WITH_NULL_NULL]
main, SEND TLSv1.2 ALERT: fatal, description = handshake_failure
main, WRITE: TLSv1.2 Alert, length = 2
main, called closeSocket()
main, handling exception: javax.net.ssl.SSLHandshakeException: no cipher suites in common
javax.net.ssl.SSLHandshakeException: no cipher suites in common
at sun.security.ssl.Alerts.getSSLException(Alerts.java:192)
at sun.security.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1949)
at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:302)
at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:292)
at sun.security.ssl.ServerHandshaker.chooseCipherSuite(ServerHandshaker.java:1035)
at sun.security.ssl.ServerHandshaker.clientHello(ServerHandshaker.java:738)
at sun.security.ssl.ServerHandshaker.processMessage(ServerHandshaker.java:221)
at sun.security.ssl.Handshaker.processLoop(Handshaker.java:979)
at sun.security.ssl.Handshaker.process_record(Handshaker.java:914)
at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1062)
at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375)
at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403)
at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387)
at ch.debugging.server.Main.main(Main.java:93)
The output of the client program
javax.net.ssl.SSLHandshakeException: Received fatal alert: handshake_failure
at sun.security.ssl.Alerts.getSSLException(Alerts.java:192)
at sun.security.ssl.Alerts.getSSLException(Alerts.java:154)
at sun.security.ssl.SSLSocketImpl.recvAlert(SSLSocketImpl.java:2023)
at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1125)
at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375)
at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403)
at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387)
at main.main(main.java:38)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
Thanks for your help.
