diff --git a/src/server.c b/src/server.c
index ec812d6..4642e37 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1437,7 +1437,8 @@ static int handle_rr(patty_ax25_server *server,
                 patty_ax25_sock_send_rnr(sock, PATTY_AX25_FRAME_RESPONSE, 1);
 
         case PATTY_AX25_FRAME_RESPONSE:
-            sock->vs = frame->nr;
+            sock->vs      = frame->nr;
+            sock->retries = sock->n_retry;
 
             fd_watch(server, sock->fd);
 
@@ -1470,8 +1471,9 @@ static int handle_rnr(patty_ax25_server *server,
                 patty_ax25_sock_send_rnr(sock, PATTY_AX25_FRAME_RESPONSE, 1);
 
         case PATTY_AX25_FRAME_RESPONSE:
-            sock->flow = PATTY_AX25_SOCK_WAIT;
-            sock->vs   = frame->nr;
+            sock->flow    = PATTY_AX25_SOCK_WAIT;
+            sock->vs      = frame->nr;
+            sock->retries = sock->n_retry;
 
             fd_clear(server, sock->fd);