Sort dropped caps when caps are added (#3973)

As sets are unordered, not sorting the resulting list built from set
operations means that we're constantly changing the order of metadata in
the final podspec, leading to bounces almost every time the s_k_j runs

paasta_tools/kubernetes_tools.py

@@ -1403,7 +1403,9 @@ def get_security_context(self) -> Optional[V1SecurityContext]:
                     # NOTE: this is necessary as containerd differs in behavior from dockershim: in dockershim
                     # dropped capabilities were overriden if the same capability was added - but in containerd
                     # the dropped capabilities appear to have higher priority.
-                    drop=list(set(CAPS_DROP) - set(cap_add)),
+                    # WARNING: this must be sorted - otherwise the order of the capabilities will be different
+                    # on every setup_kubernetes_job run and cause unnecessary redeployments
+                    drop=sorted(list(set(CAPS_DROP) - set(cap_add))),
                 )
             )
 

tests/test_kubernetes_tools.py

@@ -1067,7 +1067,7 @@ def test_get_security_context_without_cap_add(self):
 
     def test_get_security_context_with_cap_add(self):
         self.deployment.config_dict["cap_add"] = ["SETGID"]
-        expected_dropped_caps = list(set(CAPS_DROP) - {"SETGID"})
+        expected_dropped_caps = sorted(list(set(CAPS_DROP) - {"SETGID"}))
         expected_security_context = V1SecurityContext(
             capabilities=V1Capabilities(add=["SETGID"], drop=expected_dropped_caps)
         )