diff --git a/lib/GHCup.hs b/lib/GHCup.hs
index 7605d65..00d4339 100644
--- a/lib/GHCup.hs
+++ b/lib/GHCup.hs
@@ -376,6 +376,10 @@ rmGHCVer ver = do
   if exists
     then do
       -- this isn't atomic, order matters
+      when isSetGHC $ do
+        lift $ $(logInfo) [i|Removing ghc symlinks|]
+        liftE $ rmPlain ver
+
       lift $ $(logInfo) [i|Removing directory recursively: #{d'}|]
       liftIO $ deleteDirRecursive dir
 
@@ -389,11 +393,6 @@ rmGHCVer ver = do
       (mj, mi) <- getGHCMajor ver
       getGHCForMajor mj mi >>= mapM_ (\v -> liftE $ setGHC v SetGHC_XY)
 
-
-      when isSetGHC $ do
-        lift $ $(logInfo) [i|Removing ghc symlinks|]
-        liftE $ rmPlain ver
-
       liftIO
         $   ghcupBaseDir
         >>= hideError doesNotExistErrorType