crypto/openssl-wrapper: fix SSL error code mapping

Map mbedtls error codes to OpenSSL standard return codes in
SSL_connect/SSL_do_handshake:
- Return 1 on success
- Return 0 on controlled shutdown
- Return -1 on fatal error (was returning mbedtls error codes)

This aligns the return values with OpenSSL specification where
SSL_get_error() should be called to get the actual error reason.

Signed-off-by: makejian <makejian@xiaomi.com>
This commit is contained in:
makejian 2025-08-13 19:30:14 +08:00 committed by Xiang Xiao
parent 825007aafb
commit 690d8bddce

View file

@ -368,17 +368,21 @@ int ssl_pm_handshake(SSL *ssl)
}
/* OpenSSL return codes:
* 0 = did not complete, but may be retried
* 0 = The TLS/SSL handshake was not successful but was shut down
* controlled and by the specifications of the TLS/SSL protocol.
* 1 = successfully completed
* <0 = death
* <0 = The TLS/SSL handshake was not successful because a fatal error
* occurred either at the protocol level or a connection failure
* occurred.
*/
if (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE)
{
ssl->err = ret;
ssl->err = (ret == MBEDTLS_ERR_SSL_WANT_READ) ? SSL_ERROR_WANT_READ :
SSL_ERROR_WANT_WRITE;
SSL_DEBUG(SSL_PLATFORM_ERROR_LEVEL,
"mbedtls_ssl_handshake() return -0x%x", -ret);
return 0; /* OpenSSL: did not complete but may be retried */
return -1;
}
if (ret == 0)
@ -397,7 +401,7 @@ int ssl_pm_handshake(SSL *ssl)
{
ssl->err = ret == MBEDTLS_ERR_SSL_WANT_READ;
return 0;
return -1;
}
SSL_DEBUG(SSL_PLATFORM_ERROR_LEVEL,