diff --git a/peer.go b/peer.go
index 6bbf80b8..a491a2f1 100644
--- a/peer.go
+++ b/peer.go
@@ -77,7 +77,6 @@ type peer struct {
 
 	// MUST be used atomically.
 	started    int32
-	connected  int32
 	disconnect int32
 
 	connReq *connmgr.ConnReq
@@ -270,8 +269,6 @@ func (p *peer) Start() error {
 		return nil
 	}
 
-	atomic.AddInt32(&p.connected, 1)
-
 	peerLog.Tracef("peer %v starting", p)
 
 	p.wg.Add(5)
@@ -289,14 +286,12 @@ func (p *peer) Start() error {
 // all goroutines have exited.
 func (p *peer) Stop() error {
 	// If we're already disconnecting, just exit.
-	if atomic.AddInt32(&p.disconnect, 1) != 1 {
+	if !atomic.CompareAndSwapInt32(&p.disconnect, 0, 1) {
 		return nil
 	}
 
-	// Otherwise, close the connection if we're currently connected.
-	if atomic.LoadInt32(&p.connected) != 0 {
-		p.conn.Close()
-	}
+	// Ensure that the TCP connection is properly closed before continuing.
+	p.conn.Close()
 
 	// Signal all worker goroutines to gracefully exit.
 	close(p.quit)
@@ -314,12 +309,14 @@ func (p *peer) Disconnect() {
 	}
 
 	peerLog.Tracef("Disconnecting %s", p)
-	if atomic.LoadInt32(&p.connected) != 0 {
-		p.conn.Close()
-	}
+
+	// Ensure that the TCP connection is properly closed before continuing.
+	p.conn.Close()
 
 	close(p.quit)
 
+	// If this connection was established persistently, then notify the
+	// connection manager that the peer has been disconnected.
 	if p.connReq != nil {
 		p.server.connMgr.Disconnect(p.connReq.ID())
 	}
diff --git a/server.go b/server.go
index eea70275..d49daa5b 100644
--- a/server.go
+++ b/server.go
@@ -518,9 +518,14 @@ func (s *server) removePeer(p *peer) {
 		return
 	}
 
+	// As the peer is now finished, ensure that the TCP connection is
+	// closed and all of its related goroutines have exited.
+	if err := p.Stop(); err != nil {
+		peerLog.Errorf("unable to stop peer: %v", err)
+	}
+
 	// Ignore deleting peers if we're shutting down.
 	if atomic.LoadInt32(&s.shutdown) != 0 {
-		p.Stop()
 		return
 	}