diff --git a/src/core/load-fragment.c b/src/core/load-fragment.c
index 7bcce9b1314..7fa1bafaef7 100644
--- a/src/core/load-fragment.c
+++ b/src/core/load-fragment.c
@@ -1174,14 +1174,16 @@ int config_parse_capability_set(
                 return 0;
         }
 
-        sum = invert ? ~sum : sum;
-
         if (sum == 0 || *capability_set == initial)
-                /* "" or uninitialized data -> replace */
-                *capability_set = sum;
-        else
+                /* "", "~" or uninitialized data -> replace */
+                *capability_set = invert ? ~sum : sum;
+        else {
                 /* previous data -> merge */
-                *capability_set |= sum;
+                if (invert)
+                        *capability_set &= ~sum;
+                else
+                        *capability_set |= sum;
+        }
 
         return 0;
 }