diff --git a/src/fe-common/core/fe-server.c b/src/fe-common/core/fe-server.c
index 73c77e77..46d25ae9 100644
--- a/src/fe-common/core/fe-server.c
+++ b/src/fe-common/core/fe-server.c
@@ -120,7 +120,8 @@ static void cmd_server_add_modify(const char *data, gboolean add)
 
 	value = g_hash_table_lookup(optlist, "port");
 	port = *portstr == '\0' ?
-		(value != NULL && *value != '\0' ? atoi(value) : DEFAULT_SERVER_ADD_PORT)
+			(value != NULL && *value != '\0' ?
+			atoi(value) : DEFAULT_SERVER_ADD_PORT)
 		: atoi(portstr);
 
 	chatnet = g_hash_table_lookup(optlist, "network");
@@ -141,7 +142,11 @@ static void cmd_server_add_modify(const char *data, gboolean add)
 			return;
 		}
 		rec->address = g_strdup(addr);
+		rec->port = port;
 	} else {
+		if (*portstr != '\0' || g_hash_table_lookup(optlist, "port"))
+			rec->port = port;
+
 		if (*password != '\0') g_free_and_null(rec->password);
 		if (g_hash_table_lookup(optlist, "host")) {
 			g_free_and_null(rec->own_host);
@@ -149,8 +154,6 @@ static void cmd_server_add_modify(const char *data, gboolean add)
 		}
 	}
 
-	rec->port = port;
-
 	if (g_hash_table_lookup(optlist, "6"))
 		rec->family = AF_INET6;
         else if (g_hash_table_lookup(optlist, "4"))