diff --git a/src/timesync/timesyncd-manager.c b/src/timesync/timesyncd-manager.c
index 637a3b81c6a..0fab37d7834 100644
--- a/src/timesync/timesyncd-manager.c
+++ b/src/timesync/timesyncd-manager.c
@@ -647,7 +647,8 @@ static int manager_listen_setup(Manager *m) {
         if (r < 0)
                 return r;
 
-        (void) setsockopt_int(m->server_socket, IPPROTO_IP, IP_TOS, IPTOS_LOWDELAY);
+        if (addr.sa.sa_family == AF_INET)
+                (void) setsockopt_int(m->server_socket, IPPROTO_IP, IP_TOS, IPTOS_LOWDELAY);
 
         return sd_event_add_io(m->event, &m->event_receive, m->server_socket, EPOLLIN, manager_receive_response, m);
 }