diff --git a/hosting/forms.py b/hosting/forms.py
index 36d05431..8aff59d4 100644
--- a/hosting/forms.py
+++ b/hosting/forms.py
@@ -108,8 +108,10 @@ class UserHostingKeyForm(forms.ModelForm):
                 raise forms.ValidationError(KEY_ERROR_MESSAGE)
             try:
                 os.remove(tmp_public_key_file.name)
-            except OSError:
-                pass
+            except FileNotFoundError:
+                logger.debug(
+                    "{} could not be deleted because it doesn't exist".format(
+                        tmp_public_key_file.name))
         return openssh_pubkey_str
 
     def clean_name(self):
diff --git a/utils/tasks.py b/utils/tasks.py
index 21d2c9b3..ecad5c71 100644
--- a/utils/tasks.py
+++ b/utils/tasks.py
@@ -1,5 +1,5 @@
-import tempfile
 import os
+import tempfile
 
 import cdist
 from cdist.integration import configure_hosts_simple
@@ -70,8 +70,10 @@ def save_ssh_key(self, hosts, keys):
             return_value = False
     try:
         os.remove(tmp_manifest.name)
-    except OSError:
-        pass
+    except FileNotFoundError:
+        logger.debug(
+            "{} could not be deleted because it doesn't exist".format(
+                tmp_manifest.name))
     return return_value