Add error code handling

client/internal/peer/conn.go

@@ -2,6 +2,7 @@ package peer
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math/rand"
 	"net"
@@ -209,7 +210,11 @@ func (conn *Conn) Open(engineCtx context.Context) error {
 	// both peer send offer
 	if err := conn.handshaker.SendOffer(); err != nil {
 		conn.Log.Errorf("failed to send offer: %v", err)
-		conn.guard.FailedToSendOffer()
+		// if remote peer is offline, no need to try to reconnect.
+		// The remote peer when online will send an offer to us
+		if !errors.Is(err, ErrPeerNotAvailable) {
+			conn.guard.FailedToSendOffer()
+		}
 	}
 
 	conn.opened = true

client/internal/peer/signaler.go

@@ -1,6 +1,8 @@
 package peer
 
 import (
+	"errors"
+
 	"github.com/pion/ice/v4"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
@@ -9,6 +11,8 @@ import (
 	sProto "github.com/netbirdio/netbird/shared/signal/proto"
 )
 
+var ErrPeerNotAvailable = signal.ErrPeerNotAvailable
+
 type Signaler struct {
 	signal       signal.Client
 	wgPrivateKey wgtypes.Key
@@ -68,6 +72,9 @@ func (s *Signaler) signalOfferAnswer(offerAnswer OfferAnswer, remoteKey string,
 	}
 
 	if err = s.signal.SendWithDeliveryCheck(msg); err != nil {
+		if errors.Is(err, signal.ErrPeerNotAvailable) {
+			return ErrPeerNotAvailable
+		}
 		return err
 	}
 

shared/signal/client/grpc.go

@@ -2,6 +2,7 @@ package client
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"sync"
@@ -22,6 +23,10 @@ import (
 	"github.com/netbirdio/netbird/shared/signal/proto"
 )
 
+var (
+	ErrPeerNotAvailable = errors.New("peer not available")
+)
+
 // ConnStateNotifier is a wrapper interface of the status recorder
 type ConnStateNotifier interface {
 	MarkSignalDisconnected(error)
@@ -410,6 +415,17 @@ func (c *GrpcClient) SendWithDeliveryCheck(msg *proto.Message) error {
 	defer cancel()
 
 	_, err = c.realClient.SendWithDeliveryCheck(ctx, encryptedMessage)
+	if err != nil {
+		if st, ok := status.FromError(err); ok {
+			switch st.Code() {
+			case codes.NotFound:
+				return ErrPeerNotAvailable
+			default:
+				return fmt.Errorf("grpc error %s: %w", st.Code(), err)
+			}
+		}
+		return err // Not a gRPC status error
+	}
 	return err
 }
 

signal/server/signal.go

@@ -105,6 +105,11 @@ func (s *Server) Send(ctx context.Context, msg *proto.EncryptedMessage) (*proto.
 }
 
 // SendWithDeliveryCheck forwards a message to the signal peer with error handling
+// When the remote peer is not connected it returns codes.NotFound error, otherwise it returns other types of errors
+// that can be retried. In case codes.NotFound is returned the caller should not retry sending the message. The remote
+// peer should send a new offer to re-establish the connection when it comes back online.
+// Todo: double check the thread safe registry management. When both peer come online at the same time then both peers
+// might not be registered yet when the first message is sent.
 func (s *Server) SendWithDeliveryCheck(ctx context.Context, msg *proto.EncryptedMessage) (*emptypb.Empty, error) {
 	log.Tracef("received a new message to send from peer [%s] to peer [%s]", msg.Key, msg.RemoteKey)
 
@@ -113,8 +118,7 @@ func (s *Server) SendWithDeliveryCheck(ctx context.Context, msg *proto.Encrypted
 		s.forwardMessageToPeer(ctx, msg)
 		return &emptypb.Empty{}, nil
 	}
-
-	return nil, status.Errorf(codes.FailedPrecondition, "remote peer not connected")
+	return nil, status.Errorf(codes.NotFound, "remote peer not connected")
 	//return s.dispatcher.SendMessage(ctx, msg)
 }