diff --git a/mercurial/exchange.py b/mercurial/exchange.py
index 773852b7ff0cd25d137c8f3d3a2b46fe1ddf868e_bWVyY3VyaWFsL2V4Y2hhbmdlLnB5..fbf3cf982d303cb054d3fc6de17f154a90975826_bWVyY3VyaWFsL2V4Y2hhbmdlLnB5 100644
--- a/mercurial/exchange.py
+++ b/mercurial/exchange.py
@@ -597,7 +597,8 @@
 
     (computed for both success and failure case for changesets push)"""
     outgoing = pushop.outgoing
-    unfi = pushop.repo.unfiltered()
+    repo = pushop.repo
+    unfi = repo.unfiltered()
     to_rev = unfi.changelog.index.rev
     remotephases = listkeys(pushop.remote, b'phases')
 
@@ -621,10 +622,8 @@
         return
 
     fallbackheads_rev = [to_rev(n) for n in pushop.fallbackheads]
-    futureheads_rev = [to_rev(n) for n in pushop.futureheads]
-
     pushop.remotephases = phases.RemotePhasesSummary(
         pushop.repo,
         fallbackheads_rev,
         remotephases,
     )
@@ -626,20 +625,39 @@
     pushop.remotephases = phases.RemotePhasesSummary(
         pushop.repo,
         fallbackheads_rev,
         remotephases,
     )
-    droots = pushop.remotephases.draft_roots
-
-    extracond = b''
-    if not pushop.remotephases.publishing:
-        extracond = b' and public()'
-    revset = b'heads((%%ld::%%ld) %s)' % extracond
-    # Get the list of all revs draft on remote by public here.
-    # XXX Beware that revset break if droots is not strictly
-    # XXX root we may want to ensure it is but it is costly
-    fallback = list(unfi.set(revset, droots, fallbackheads_rev))
-    if not pushop.remotephases.publishing and pushop.publish:
-        future = list(
-            unfi.set(
-                b'%ld and (not public() or %ld::)', futureheads_rev, droots
+    droots = set(pushop.remotephases.draft_roots)
+
+    fallback_publishing = pushop.remotephases.publishing
+    push_publishing = pushop.remotephases.publishing and pushop.publish
+    explicit_publishing = not pushop.remotephases.publishing and pushop.publish
+    missing_revs = {to_rev(n) for n in outgoing.missing}
+    drafts = unfi._phasecache.get_raw_set(unfi, phases.draft)
+
+    if fallback_publishing:
+        fallback_roots = droots - missing_revs
+        revset = b'heads(%ld::%ld)'
+    else:
+        fallback_roots = droots - drafts
+        fallback_roots -= missing_revs
+        # Get the list of all revs draft on remote but public here.
+        revset = b'heads((%ld::%ld) and public())'
+    if not fallback_roots:
+        fallback = []
+    else:
+        fallback = unfi.revs(revset, fallback_roots, fallbackheads_rev)
+        fallback = [repo[r] for r in fallback]
+
+    if explicit_publishing:
+        future_heads = {to_rev(n) for n in pushop.futureheads}
+        need_publish = future_heads & drafts
+        if fallback_roots:
+            need_publish.extend(
+                unfi.revs(
+                    b'%ld and (%ld::%ld)',
+                    future_heads,
+                    fallback_roots,
+                    future_heads,
+                )
             )
@@ -645,5 +663,5 @@
             )
-        )
-    elif not outgoing.missing:
-        future = fallback
+        future = [repo[r] for r in need_publish]
+    elif push_publishing:
+        future = pushop.future_heads
     else:
@@ -649,14 +667,20 @@
     else:
-        # adds changeset we are going to push as draft
-        #
-        # should not be necessary for publishing server, but because of an
-        # issue fixed in xxxxx we have to do it anyway.
-        missing_rev = [to_rev(n) for n in outgoing.missing]
-        fdroots = list(
-            unfi.set(b'roots(%ld  + %ld::)', missing_rev, droots)
-        )
-        fdroots = [f.rev() for f in fdroots]
-        future = list(unfi.set(revset, fdroots, futureheads_rev))
+        # adds changeset we are going to push as public
+        public_missing = missing_revs - drafts
+        if not public_missing:
+            future = fallback
+        else:
+            # We also needs to add the outdatedphases we computed earlier
+            #
+            # On some topological branches, we might be pushing only drafts,
+            # but some of the common part might need publishing. So we still
+            # need to push that format.
+            #
+            # We add them to public_missing to still ensure we only have heads
+            # here.
+            public_missing.update(c.rev() for c in fallback)
+            public_missing = unfi.revs('heads(%ld)', public_missing)
+            future = [repo[r] for r in public_missing]
     pushop.outdatedphases = future
     pushop.fallbackoutdatedphases = fallback