diff --git a/config/software/chef-acme.rb b/config/software/chef-acme.rb
index d7ecc0c1c0ac5a59a6394c7dcc0aee37b61182a4_Y29uZmlnL3NvZnR3YXJlL2NoZWYtYWNtZS5yYg==..0ee1457ebb58f0cd8afcd2106c11eb06a59ad650_Y29uZmlnL3NvZnR3YXJlL2NoZWYtYWNtZS5yYg== 100644
--- a/config/software/chef-acme.rb
+++ b/config/software/chef-acme.rb
@@ -15,7 +15,7 @@
 #
 
 name 'chef-acme'
-version = Gitlab::Version.new(name, 'v4.1.3')
+version = Gitlab::Version.new(name, 'v4.1.5')
 default_version version.print(false)
 
 license 'Apache-2.0'
diff --git a/files/gitlab-cookbooks/letsencrypt/recipes/enable.rb b/files/gitlab-cookbooks/letsencrypt/recipes/enable.rb
index d7ecc0c1c0ac5a59a6394c7dcc0aee37b61182a4_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9sZXRzZW5jcnlwdC9yZWNpcGVzL2VuYWJsZS5yYg==..0ee1457ebb58f0cd8afcd2106c11eb06a59ad650_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9sZXRzZW5jcnlwdC9yZWNpcGVzL2VuYWJsZS5yYg== 100644
--- a/files/gitlab-cookbooks/letsencrypt/recipes/enable.rb
+++ b/files/gitlab-cookbooks/letsencrypt/recipes/enable.rb
@@ -14,6 +14,7 @@
 
 # We assume that the certificate and key will be stored in the same directory
 ssl_dir = File.dirname(node['gitlab']['nginx']['ssl_certificate'])
+node.default['acme']['private_key_file'] = File.join(ssl_dir, 'letsencrypt_account_private_key.pem')
 
 directory ssl_dir do
   owner 'root'
diff --git a/files/gitlab-cookbooks/letsencrypt/resources/certificate.rb b/files/gitlab-cookbooks/letsencrypt/resources/certificate.rb
index d7ecc0c1c0ac5a59a6394c7dcc0aee37b61182a4_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9sZXRzZW5jcnlwdC9yZXNvdXJjZXMvY2VydGlmaWNhdGUucmI=..0ee1457ebb58f0cd8afcd2106c11eb06a59ad650_ZmlsZXMvZ2l0bGFiLWNvb2tib29rcy9sZXRzZW5jcnlwdC9yZXNvdXJjZXMvY2VydGlmaWNhdGUucmI= 100644
--- a/files/gitlab-cookbooks/letsencrypt/resources/certificate.rb
+++ b/files/gitlab-cookbooks/letsencrypt/resources/certificate.rb
@@ -59,6 +59,11 @@
     end
   end
 
+  # Delete the private key_file
+  file node['acme']['private_key_file'] do
+    action :delete
+  end
+
   if ::File.file?(new_resource.key)
     production_key = OpenSSL::PKey::RSA.new ::File.read new_resource.key
     production_key_size = production_key.n.num_bits
@@ -88,4 +93,9 @@
     notifies :run, 'execute[reload nginx]'
     sensitive true
   end
+
+  # Delete the private key_file
+  file node['acme']['private_key_file'] do
+    action :delete
+  end
 end
diff --git a/spec/chef/cookbooks/letsencrypt/resources/letsencrypt_certificate_spec.rb b/spec/chef/cookbooks/letsencrypt/resources/letsencrypt_certificate_spec.rb
index d7ecc0c1c0ac5a59a6394c7dcc0aee37b61182a4_c3BlYy9jaGVmL2Nvb2tib29rcy9sZXRzZW5jcnlwdC9yZXNvdXJjZXMvbGV0c2VuY3J5cHRfY2VydGlmaWNhdGVfc3BlYy5yYg==..0ee1457ebb58f0cd8afcd2106c11eb06a59ad650_c3BlYy9jaGVmL2Nvb2tib29rcy9sZXRzZW5jcnlwdC9yZXNvdXJjZXMvbGV0c2VuY3J5cHRfY2VydGlmaWNhdGVfc3BlYy5yYg== 100644
--- a/spec/chef/cookbooks/letsencrypt/resources/letsencrypt_certificate_spec.rb
+++ b/spec/chef/cookbooks/letsencrypt/resources/letsencrypt_certificate_spec.rb
@@ -44,6 +44,10 @@
       )
     end
 
+    it "deletes the private_key_file" do
+      expect(chef_run).to delete_file('/etc/gitlab/ssl/letsencrypt_account_private_key.pem')
+    end
+
     context 'specifying a different key_size' do
       before do
         allow(File).to receive(:file?).and_call_original