SSL_connect on epoll f-stack socket in C++
I'm building an (EPOLL client) application in C++ that connects to a server via f-stack (and dpdk). My f-stack and dpdk is set up correctly and my code works if I connect a socket to google.com on port 80 and send a GET request. I mainly used this repo for accomplishing this: https://github.com/Luo-Liang/F-stack-bw2/blob/master/main_client.cpp
But, I need a TLS connection, so I'm trying to turn the (ff_)socket into a TLS socket. Here is (part of) the code:
int loop(void *arg)
{
int nevents = PLinkEpollWait(epfd, events, MAX_EVENTS, 0);
int i;
for (i = 0; i < nevents; ++i)
{
// std::cout << "event? " << events[i].events << std::endl;
if (events[i].events & EPOLLERR)
{
/* Simply close socket */
PLinkEpollCtrl(epfd, EPOLL_CTL_DEL, events[i].data.fd, NULL);
PLinkClose(events[i].data.fd);
printf("EPOLLERR. %s=?\n", strerror(errno));
exit(-1);
}
if (events[i].events & EPOLLIN)
{
ssize_t readlen = PLinkRead(events[i].data.fd, buf, sizeof(buf));
if (readlen > 0)
{
bytesRecv += readlen;
PLinkWrite(events[i].data.fd, req, sizeof(req));
}
else
{
PLinkEpollCtrl(epfd, EPOLL_CTL_DEL, events[i].data.fd, NULL);
PLinkClose(events[i].data.fd);
}
}
if (events[i].events & EPOLLOUT)
{
std::cout << "ready to send " << std::endl;
SSL *ssl = SSL_new(ssl_ctx);
if (!ssl)
{
std::cerr << "SSL_new failed on fd: " << events[i].data.fd << std::endl;
PLinkEpollCtrl(epfd, EPOLL_CTL_DEL, events[i].data.fd, NULL);
PLinkClose(events[i].data.fd);
exit(EXIT_FAILURE);
}
SSL_set_fd(ssl, events[i].data.fd);
SSL_set_connect_state(ssl);
SSL_set_info_callback(ssl, [](const SSL *ssl, int where, int ret)
{
std::cout << where << " callback " << ret << std::endl;
if (ret == 0) {
printf("SSL error: %s\n", ERR_error_string(ERR_get_error(), nullptr));
return;
}
const char *str = (where & SSL_ST_CONNECT) ? "SSL_connect" : "undefined";
int w = where & ~SSL_ST_MASK;
if (w & SSL_CB_LOOP) {
printf("%s: %s\n", str, SSL_state_string_long(ssl));
} else if (w & SSL_CB_ALERT) {
const char *alertType = (where & SSL_CB_READ) ? "read" : "write";
printf("SSL Alert [%s]: %s: %s\n", alertType,
SSL_alert_type_string_long(ret), SSL_alert_desc_string_long(ret));
} else if (w & SSL_CB_EXIT) {
if (ret == 0) {
printf("%s: failed in %s\n", str, SSL_state_string_long(ssl));
} else if (ret < 0) {
printf("%s: error in %s\n", str, SSL_state_string_long(ssl));
}
} });
for (;;)
{
int success = SSL_connect(ssl);
if (success < 0)
{
int ssl_error_code = SSL_get_error(ssl, success);
if (ssl_error_code == SSL_ERROR_WANT_READ || ssl_error_code == SSL_ERROR_WANT_WRITE)
{
printf("SSL_connect waiting for more data, error code: %d\n", ssl_error_code);
// Wait for epoll to notify the appropriate event
int nevents = PLinkEpollWait(epfd, events, 1, 5000); // 1000ms timeout
if (nevents < 0)
{
perror("PLinkEpollWait failed");
SSL_free(ssl);
PLinkClose(events[i].data.fd);
exit(EXIT_FAILURE);
}
// Continue to let the loop retry SSL_connect based on the appropriate event
continue;
}
else if (ssl_error_code == SSL_ERROR_ZERO_RETURN)
{
printf("SSL_connect: close notify received from peer\n");
exit(EXIT_FAILURE);
}
else
{
printf("Error SSL_connect: %d\n", ssl_error_code);
ERR_print_errors_fp(stderr);
SSL_free(ssl);
PLinkClose(events[i].data.fd);
exit(EXIT_FAILURE);
}
}
else
{
dump_cert_info(ssl, false);
printf("SSL handshake completed successfully.\n");
break;
}
}
}
}
}
int main(int argc, char *argv[])
{
std::string serverIp = "172.217.174.110";
int port = 443;
initializeSSL();
PLinkInit(argc, argv);
printf("f-stack initialized.\n");
epfd = PLinkEpollCreate();
if (epfd < 0)
{
std::cerr << "PLinkEpollCreate failed, errno: " << errno << " (" << strerror(errno) << ")" << std::endl;
exit(EXIT_FAILURE);
}
printf("Epoll instance created with fd: %d\n", epfd);
int sockfd = PLinkSocket(AF_INET, SOCK_STREAM, 0);
if (sockfd < 0)
{
std::cerr << "PLinkSocket failed, errno: " << errno << " (" << strerror(errno) << ")" << std::endl;
exit(EXIT_FAILURE);
}
printf("Socket created with fd: %d\n", sockfd);
PLinkSetNonBlock(sockfd);
auto CLIENT_PORT = (uint16_t)(1024 + rand() % (49000));
struct sockaddr_in serv_addr;
bzero(&serv_addr, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_port = htons(CLIENT_PORT);
// bind to any of my address.
serv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
std::string sip = serverIp;
int ret = PLinkBind(sockfd, (const sockaddr *)&serv_addr, sizeof(serv_addr));
if (ret < 0)
{
printf("PLinkBind failed, errno = %s\n", strerror(errno));
for (auto sock : sockfdVec)
{
PLinkClose(sock);
}
exit(1);
}
struct epoll_event ev;
ev.data.fd = sockfd;
ev.events = EPOLLIN | EPOLLOUT;
PLinkEpollCtrl(epfd, EPOLL_CTL_ADD, sockfd, &ev);
sockaddr_in remote_addr;
remote_addr.sin_family = AF_INET;
remote_addr.sin_port = htons(80);
inet_pton(AF_INET, sip.c_str(), &(remote_addr.sin_addr));
ret = PLinkConnect(sockfd, (const sockaddr *)&remote_addr, sizeof(sockaddr_in));
if (ret < 0 && errno != EINPROGRESS)
{
// ff_connect can return EINPROGRESS as it is always nb
printf("ff_connect failed %d: %s\n", errno, strerror(errno));
exit(1);
}
// ssl_map[sockfd] = ssl;
PLinkRun(loop, NULL);
SSL_CTX_free(ssl_ctx);
return 0;
}
What I'm getting is during SSL_connect the program just hangs, it doesn't get past SSL_connect. I get this output:
f-stack initialized.
Epoll instance created with fd: 0
Socket created with fd: 1
ready to send
16 callback 1
4097 callback 1
a;��.D��HT�`��^����ՠ��U�bѫ ʔ��"qۃ���.�������\�ӌ҉c��>�,�0�̨̩̪�+�/��$�(k�#�'g�
�9� �3��=<5/��
*(
+-3&$ O��+�7�Y�=���m�����{��Vѕt1�h4097 callback 1
The where variable in the callback is 4097, which maybe is a combination of SSL_ST_CONNECT (0x1000) and SSL_CB_LOOP (0x001) flags.
I'm on RHEL 7.5, OpenSSL 3.1.3, g++ (GCC) 7.3.1.
Is there someone who can help me out?