I am trying to establish a TLS connection between my Java Client and the Floodlight SDN controller (https://github.com/floodlight/floodlight). Every time I try to decrypt the first message Floodlight sends after the handshake, a AEADBadTagException occurs. This only happens for the first message, the following messages can be decrypted successfully. How can I prevent this Exception?
My Client:
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.*;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.SelectionKey;
import java.nio.channels.Selector;
import java.nio.channels.SocketChannel;
import java.security.cert.X509Certificate;
import java.util.Arrays;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class Client {
private Selector selector;
private SocketChannel upstreamSocketChannel;
private final ByteBuffer receiveBuffer = ByteBuffer.allocate(16384);
private SSLEngine upstreamSslEngine;
private ByteBuffer myNetData;
private ByteBuffer peerAppData;
private ByteBuffer peerNetData;
private ExecutorService executor = Executors.newSingleThreadExecutor();
private final Logger log = LoggerFactory.getLogger(Client.class);
public Client() throws IOException {
selector = Selector.open();
setupSslContexts();
String upstreamIp = "127.0.0.1";
int upstreamPort = 6653;
upstreamSocketChannel = selector.provider().openSocketChannel();
upstreamSocketChannel.connect(new InetSocketAddress(upstreamIp, upstreamPort));
upstreamSslEngine.beginHandshake();
if(!doHandshake(upstreamSocketChannel, upstreamSslEngine)){
throw new IOException("upstream ssl handshake failed!");
}else{
System.out.println("handshake suceeded");
}
upstreamSocketChannel.configureBlocking(false);
upstreamSocketChannel.register(selector, SelectionKey.OP_READ);
new Thread(this::loop).start();
}
private void setupSslContexts(){
try{
TrustManager trm = new X509TrustManager() {
public X509Certificate[] getAcceptedIssuers() {
return null;
}
public void checkClientTrusted(X509Certificate[] certs, String authType) {}
public void checkServerTrusted(X509Certificate[] certs, String authType) {}
};
SSLContext upstreamContext = SSLContext.getInstance("TLS");
upstreamContext.init(null, new TrustManager[] { trm }, null);
upstreamSslEngine = upstreamContext.createSSLEngine();
upstreamSslEngine.setUseClientMode(true);
upstreamSslEngine.setEnabledProtocols(upstreamSslEngine.getSupportedProtocols());
upstreamSslEngine.setEnabledCipherSuites(upstreamSslEngine.getSupportedCipherSuites());
SSLSession dummySession = upstreamContext.createSSLEngine().getSession();
myNetData = ByteBuffer.allocate(dummySession.getPacketBufferSize());
peerAppData = ByteBuffer.allocate(dummySession.getApplicationBufferSize());
peerNetData = ByteBuffer.allocate(dummySession.getPacketBufferSize());
dummySession.invalidate();
}catch (Exception e){
log.error("Failed to setup SSL Context!");
e.printStackTrace();
}
}
protected boolean doHandshake(SocketChannel socketChannel, SSLEngine engine) throws IOException {
log.debug("About to do handshake...");
SSLEngineResult result;
SSLEngineResult.HandshakeStatus handshakeStatus;
// NioSslPeer's fields myAppData and peerAppData are supposed to be large enough to hold all message data the peer
// will send and expects to receive from the other peer respectively. Since the messages to be exchanged will usually be less
// than 16KB long the capacity of these fields should also be smaller. Here we initialize these two local buffers
// to be used for the handshake, while keeping client's buffers at the same size.
int appBufferSize = engine.getSession().getApplicationBufferSize();
ByteBuffer myAppData = ByteBuffer.allocate(appBufferSize);
ByteBuffer peerAppData = ByteBuffer.allocate(appBufferSize);
myNetData.clear();
peerNetData.clear();
handshakeStatus = engine.getHandshakeStatus();
while (handshakeStatus != SSLEngineResult.HandshakeStatus.FINISHED && handshakeStatus != SSLEngineResult.HandshakeStatus.NOT_HANDSHAKING) {
switch (handshakeStatus) {
case NEED_UNWRAP:
if (socketChannel.read(peerNetData) < 0) {
if (engine.isInboundDone() && engine.isOutboundDone()) {
return false;
}
try {
engine.closeInbound();
} catch (SSLException e) {
log.error("This engine was forced to close inbound, without having received the proper SSL/TLS close notification message from the peer, due to end of stream.");
}
engine.closeOutbound();
// After closeOutbound the engine will be set to WRAP state, in order to try to send a close message to the client.
handshakeStatus = engine.getHandshakeStatus();
break;
}
peerNetData.flip();
try {
result = engine.unwrap(peerNetData, peerAppData);
peerNetData.compact();
handshakeStatus = result.getHandshakeStatus();
} catch (SSLException sslException) {
sslException.printStackTrace();
log.error("A problem was encountered while processing the data that caused the SSLEngine to abort. Will try to properly close connection...");
engine.closeOutbound();
handshakeStatus = engine.getHandshakeStatus();
break;
}
switch (result.getStatus()) {
case OK:
break;
case CLOSED:
if (engine.isOutboundDone()) {
return false;
} else {
engine.closeOutbound();
handshakeStatus = engine.getHandshakeStatus();
break;
}
default:
throw new IllegalStateException("Invalid SSL status: " + result.getStatus());
}
break;
case NEED_WRAP:
myNetData.clear();
try {
result = engine.wrap(myAppData, myNetData);
handshakeStatus = result.getHandshakeStatus();
} catch (SSLException sslException) {
sslException.printStackTrace();
log.error("A problem was encountered while processing the data that caused the SSLEngine to abort. Will try to properly close connection...");
engine.closeOutbound();
handshakeStatus = engine.getHandshakeStatus();
break;
}
switch (result.getStatus()) {
case OK :
myNetData.flip();
while (myNetData.hasRemaining()) {
socketChannel.write(myNetData);
}
break;
case CLOSED:
try {
myNetData.flip();
while (myNetData.hasRemaining()) {
socketChannel.write(myNetData);
}
// At this point the handshake status will probably be NEED_UNWRAP so we make sure that peerNetData is clear to read.
peerNetData.clear();
} catch (Exception e) {
log.error("Failed to send server's CLOSE message due to socket channel's failure.");
handshakeStatus = engine.getHandshakeStatus();
}
break;
default:
throw new IllegalStateException("Invalid SSL status: " + result.getStatus());
}
break;
case NEED_TASK:
Runnable task;
while ((task = engine.getDelegatedTask()) != null) {
executor.execute(task);
}
handshakeStatus = engine.getHandshakeStatus();
break;
case FINISHED:
break;
case NOT_HANDSHAKING:
break;
default:
throw new IllegalStateException("Invalid SSL status: " + handshakeStatus);
}
}
return true;
}
private void loop(){
while(!Thread.interrupted()){
try {
selector.select();
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
Set<SelectionKey> selected = selector.selectedKeys();
for (SelectionKey key : selected) {
if (key.isValid()) {
int read = 0;
try {
read = upstreamSocketChannel.read(receiveBuffer);
} catch (IOException e) {
e.printStackTrace();
}
if (read == -1) {
// end-of-stream (disconnect)
key.cancel();
System.out.println("end of stream");
try {
upstreamSocketChannel.close();
} catch (IOException e) {
e.printStackTrace();
}
return;
} else if (read > 0) {
receiveBuffer.flip();
System.out.println(Arrays.toString(new ByteBuffer[]{receiveBuffer}));
try {
decryptBuffer(receiveBuffer);
System.out.println("decrypt successfull!");
} catch (SSLException e) {
e.printStackTrace();
}
receiveBuffer.compact();
}
}
}
selected.clear();
}
}
private ByteBuffer decryptBuffer(ByteBuffer buffer) throws SSLException {
peerAppData.clear();
while (buffer.hasRemaining()) {
SSLEngineResult result = upstreamSslEngine.unwrap(buffer, peerAppData);
log.debug(result.toString());
switch (result.getStatus()) {
case OK:
break;
case CLOSED:
log.debug("Client wants to close connection...");
return null;
default:
throw new IllegalStateException("Invalid SSL status: " + result.getStatus());
}
}
peerAppData.flip();
return peerAppData;
}
}
Output:
handshake suceeded
[java.nio.HeapByteBuffer[pos=0 lim=37 cap=16384]]
javax.net.ssl.SSLException: Tag mismatch!
at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:129)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:321)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:264)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:259)
at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:129)
at java.base/sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:672)
at java.base/sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:627)
at java.base/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:443)
at java.base/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:422)
at java.base/javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:634)
at Client.decryptBuffer(Client.java:262)
at Client.loop(Client.java:244)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: javax.crypto.AEADBadTagException: Tag mismatch!
at java.base/com.sun.crypto.provider.GaloisCounterMode.decryptFinal(GaloisCounterMode.java:580)
at java.base/com.sun.crypto.provider.CipherCore.finalNoPadding(CipherCore.java:1116)
at java.base/com.sun.crypto.provider.CipherCore.fillOutputBuffer(CipherCore.java:1053)
at java.base/com.sun.crypto.provider.CipherCore.doFinal(CipherCore.java:941)
at java.base/com.sun.crypto.provider.AESCipher.engineDoFinal(AESCipher.java:491)
at java.base/javax.crypto.CipherSpi.bufferCrypt(CipherSpi.java:779)
at java.base/javax.crypto.CipherSpi.engineDoFinal(CipherSpi.java:730)
at java.base/javax.crypto.Cipher.doFinal(Cipher.java:2497)
at java.base/sun.security.ssl.SSLCipher$T12GcmReadCipherGenerator$GcmReadCipher.decrypt(SSLCipher.java:1613)
at java.base/sun.security.ssl.SSLEngineInputRecord.decodeInputRecord(SSLEngineInputRecord.java:240)
at java.base/sun.security.ssl.SSLEngineInputRecord.decode(SSLEngineInputRecord.java:197)
at java.base/sun.security.ssl.SSLEngineInputRecord.decode(SSLEngineInputRecord.java:160)
at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:108)
... 8 more
[java.nio.HeapByteBuffer[pos=0 lim=37 cap=16384]]
decrypt successfull!
[java.nio.HeapByteBuffer[pos=0 lim=74 cap=16384]]
decrypt successfull!
[java.nio.HeapByteBuffer[pos=0 lim=105 cap=16384]]
decrypt successfull!
end of stream
Edit 1: None of the received messages are decrypted successfully. It is just the exception that is only thrown for the first message because the server tries to close the connection afterwards. Maybe there is a problem with the handshake, although I took the handshake implementation from here https://docs.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#SSLENG
Edit 2: When changing the cipher suite by setting the supported cipher suites of my client for example to TLS_RSA_WITH_AES_128_CBC_SHA (before TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 was used) the handshake still succeeds, but a different error is thrown afterwards:
javax.net.ssl.SSLException: bad record MAC
at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:129)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:321)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:264)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:259)
at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:129)
at java.base/sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:672)
at java.base/sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:627)
at java.base/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:443)
at java.base/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:422)
at java.base/javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:634)
at Client.decryptBuffer(Client.java:248)
at Client.loop(Client.java:230)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: javax.crypto.BadPaddingException: bad record MAC
at java.base/sun.security.ssl.SSLCipher.checkCBCMac(SSLCipher.java:2151)
at java.base/sun.security.ssl.SSLCipher$T11BlockReadCipherGenerator$BlockReadCipher.decrypt(SSLCipher.java:1337)
at java.base/sun.security.ssl.SSLEngineInputRecord.decodeInputRecord(SSLEngineInputRecord.java:240)
at java.base/sun.security.ssl.SSLEngineInputRecord.decode(SSLEngineInputRecord.java:197)
at java.base/sun.security.ssl.SSLEngineInputRecord.decode(SSLEngineInputRecord.java:160)
at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:108)
... 8 more
