diff --git a/src/core/servers.c b/src/core/servers.c
index 143158ba..77b88fa9 100644
--- a/src/core/servers.c
+++ b/src/core/servers.c
@@ -208,7 +208,7 @@ static void server_real_connect(SERVER_REC *server, IPADDR *ip,
 	const char *errmsg;
 	char *errmsg2;
 	char ipaddr[MAX_IP_LEN];
-        int port;
+	int port = 0;
 
 	g_return_if_fail(ip != NULL || unix_socket != NULL);
 
@@ -222,12 +222,21 @@ static void server_real_connect(SERVER_REC *server, IPADDR *ip,
 		own_ip = IPADDR_IS_V6(ip) ? server->connrec->own_ip6 : server->connrec->own_ip4;
 		port = server->connrec->proxy != NULL ?
 			server->connrec->proxy_port : server->connrec->port;
-		handle = server->connrec->use_tls ?
-			net_connect_ip_ssl(ip, port, own_ip, server) : net_connect_ip(ip, port, own_ip);
+		handle = net_connect_ip(ip, port, own_ip);
 	} else {
 		handle = net_connect_unix(unix_socket);
 	}
 
+	if (server->connrec->use_tls && handle != NULL) {
+		server->handle = net_sendbuffer_create(handle, 0);
+		handle = net_start_ssl(server);
+		if (handle == NULL) {
+			net_sendbuffer_destroy(server->handle, TRUE);
+		} else {
+			server->handle->handle = handle;
+		}
+	}
+
 	if (handle == NULL) {
 		/* failed */
 		errmsg = g_strerror(errno);
@@ -251,7 +260,8 @@ static void server_real_connect(SERVER_REC *server, IPADDR *ip,
 		g_free(errmsg2);
 	} else {
 		server->connrec->last_failed_family = 0;
-		server->handle = net_sendbuffer_create(handle, 0);
+		if (!server->connrec->use_tls)
+			server->handle = net_sendbuffer_create(handle, 0);
 		if (server->connrec->use_tls)
 			server_connect_callback_init_ssl(server, handle);
 		else