diff --git a/benchmarks/utils.py b/benchmarks/utils.py
index a5e03453eae88f5f643413f893d3da99d00cdff2_YmVuY2htYXJrcy91dGlscy5weQ==..74ab07f919955b70e16354d929bf5a279314030c_YmVuY2htYXJrcy91dGlscy5weQ== 100644
--- a/benchmarks/utils.py
+++ b/benchmarks/utils.py
@@ -289,7 +289,8 @@
             )
 
         self.repo_name = repo_name
-        self.repo_path = os.path.join(REPOS_DIR, self.repo_name)
+        reference_basename = "{}-reference".format(self.repo_name)
+        self.repo_path = os.path.join(REPOS_DIR, reference_basename)
 
         # Use a clean environ to run command
         self.environ = {'HGRCPATH': ''}
diff --git a/lib/before-check.sh b/lib/before-check.sh
index a5e03453eae88f5f643413f893d3da99d00cdff2_bGliL2JlZm9yZS1jaGVjay5zaA==..74ab07f919955b70e16354d929bf5a279314030c_bGliL2JlZm9yZS1jaGVjay5zaA== 100755
--- a/lib/before-check.sh
+++ b/lib/before-check.sh
@@ -26,6 +26,4 @@
 mkdir -p repos/partial-references
 make MAKE_VERBOSE=1 -f repos.make
 
-./repo-scripts/rename-reference-directories repos
-
 python create_skip_file.py
diff --git a/repo-scripts/rename-reference-directories b/repo-scripts/rename-reference-directories
deleted file mode 100755
index a5e03453eae88f5f643413f893d3da99d00cdff2_cmVwby1zY3JpcHRzL3JlbmFtZS1yZWZlcmVuY2UtZGlyZWN0b3JpZXM=..0000000000000000000000000000000000000000
--- a/repo-scripts/rename-reference-directories
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-
-import sys
-import os
-
-SUFFIX = "-reference"
-
-
-def main(repo_dir):
-    for file in sorted(os.listdir(repo_dir)):
-        file_path = os.path.join(repo_dir, file)
-        if not os.path.isdir(file_path):
-            continue
-
-        if not file_path.endswith(SUFFIX):
-            continue
-
-        new_dir_path = file_path[: -len(SUFFIX)]
-
-        print("Renaming %s into %s" % (file_path, new_dir_path))
-        os.rename(file_path, new_dir_path)
-
-
-if __name__ == "__main__":
-    main(sys.argv[1])
diff --git a/tests/test-smoke-test.t b/tests/test-smoke-test.t
index a5e03453eae88f5f643413f893d3da99d00cdff2_dGVzdHMvdGVzdC1zbW9rZS10ZXN0LnQ=..74ab07f919955b70e16354d929bf5a279314030c_dGVzdHMvdGVzdC1zbW9rZS10ZXN0LnQ= 100644
--- a/tests/test-smoke-test.t
+++ b/tests/test-smoke-test.t
@@ -146,9 +146,6 @@
   (cd repos/partial-references/; tar xf "$TESTTMP/references"/smoke-test-B-4324509a-partial-last-ten.tar --warning=no-timestamp )
   (cd repos/partial-references/; tar xf "$TESTTMP/references"/smoke-test-B-4324509a-partial-last-thousand.tar --warning=no-timestamp )
   (cd repos/partial-references/; tar xf "$TESTTMP/references"/smoke-test-B-4324509a-partial-same.tar --warning=no-timestamp )
-  + ./repo-scripts/rename-reference-directories repos
-  Renaming repos/smoke-test-A-11eb1fef-reference into repos/smoke-test-A-11eb1fef
-  Renaming repos/smoke-test-B-4324509a-reference into repos/smoke-test-B-4324509a
   + python create_skip_file.py
   - Fetching recent changes
   - Creating environments
@@ -189,5 +186,5 @@
   $ ls repos/
   partial-references
   skip.json
-  smoke-test-A-11eb1fef
+  smoke-test-A-11eb1fef-reference
   smoke-test-A-11eb1fef.benchrepo
@@ -193,5 +190,5 @@
   smoke-test-A-11eb1fef.benchrepo
-  smoke-test-B-4324509a
+  smoke-test-B-4324509a-reference
   smoke-test-B-4324509a.benchrepo
 
   $ ls repos/partial-references/