Allow building newer GHCs from git

The user will have to pass --overwrite-version=<ver> because
we can't discover the GHC version from git anymore.

https://gitlab.haskell.org/ghc/ghc/-/issues/22322

.github/scripts/common.sh

@@ -10,6 +10,16 @@ ecabal() {
 	cabal "$@"
 }
 
+sync_from_retry() {
+	if [ "${RUNNER_OS}" != "Windows" ] ; then
+		cabal_store_path="$(dirname "$(cabal help user-config | tail -n 1 | xargs)")/store"
+	else
+		cabal_store_path="${CABAL_DIR}/store"
+	fi
+
+    sync_from || { sleep 9 ; rm -rf "${cabal_store_path:?}"/* ; sync_from || { sleep 20 ; rm -rf "${cabal_store_path:?}"/* ; sync_from ; } }
+}
+
 sync_from() {
 	if [ "${RUNNER_OS}" != "Windows" ] ; then
 		cabal_store_path="$(dirname "$(cabal help user-config | tail -n 1 | xargs)")/store"
@@ -24,6 +34,10 @@ sync_from() {
 		--archive-uri "s3://ghcup-hs/${RUNNER_OS}-${ARCH}-${DISTRO}"
 }
 
+sync_to_retry() {
+    sync_to || { sleep 9 ; sync_to || { sleep 20 ; sync_to ; } }
+}
+
 sync_to() {
 	if [ "${RUNNER_OS}" != "Windows" ] ; then
 		cabal_store_path="$(dirname "$(cabal help user-config | tail -n 1 | xargs)")/store"
@@ -129,11 +143,11 @@ download_cabal_cache() {
 build_with_cache() {
 	ecabal configure "$@"
 	ecabal build --dependencies-only "$@" --dry-run
-	sync_from
-	ecabal build --dependencies-only "$@" || sync_to
-	sync_to
+	sync_from_retry
+	ecabal build --dependencies-only "$@" || sync_to_retry
+	sync_to_retry
 	ecabal build "$@"
-	sync_to
+	sync_to_retry
 }
 
 install_ghcup() {

.github/scripts/hls.sh

@@ -57,9 +57,9 @@ eghcup debug-info
 	cd "haskell-language-server-${HLS_TARGET_VERSION}/"
 	ecabal configure -w "ghc-${GHC_VERSION}" --disable-profiling --disable-tests --jobs="$(nproc)"
 	ecabal build --dependencies-only -w "ghc-${GHC_VERSION}" --disable-profiling --disable-tests --jobs="$(nproc)" --dry-run
-	sync_from
+	sync_from_retry
 	ecabal build --dependencies-only -w "ghc-${GHC_VERSION}" --disable-profiling --disable-tests --jobs="$(nproc)" || sync_to
-	sync_to
+	sync_to_retry
 )
 
 eghcup -v compile hls -j "$(nproc)" -g "${HLS_TARGET_VERSION}" --ghc "${GHC_VERSION}"

.github/workflows/cache.yaml

@@ -0,0 +1,37 @@
+name: Cache eviction
+
+on:
+  workflow_dispatch:
+    inputs:
+      key:
+        description: Which cache to evict
+        required: true
+        default: '/'
+        type: choice
+        options:
+        - FreeBSD-64-na
+        - Linux-32-Alpine
+        - Linux-64-Alpine
+        - Linux-64-Ubuntu
+        - Linux-ARM-Ubuntu
+        - Linux-ARM64-Ubuntu
+        - Windows-64-na
+        - macOS-64-na
+        - macOS-ARM64-na
+        - /
+jobs:
+  evict:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Remove from S3
+      uses: vitorsgomes/s3-rm-action@master
+      with:
+        args: --recursive
+      env:
+        AWS_S3_ENDPOINT: ${{ secrets.S3_HOST }}
+        AWS_S3_BUCKET: ghcup-hs
+        AWS_REGION: us-west-2
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        PATH_TO_DELETE: ${{ github.event.inputs.key }}

docs/guide.md

@@ -97,6 +97,10 @@ This is the complete list of env variables that change GHCup behavior:
 * `GHCUP_SKIP_UPDATE_CHECK`: Skip the (possibly annoying) update check when you run a command
 * `CC`/`LD` etc.: full environment is passed to the build system when compiling GHC via GHCup
 
+On windows, there's additionally:
+
+* `GHCUP_MSYS2`: Has to point to the root of an existing MSYS2 installation (when installed by GHCup, that's e.g. `C:\ghcup\msys64`). GHCup bootstrap takes care of this usually.
+
 ### XDG support
 
 To enable XDG style directories, set the environment variable `GHCUP_USE_XDG_DIRS` to anything.
@@ -198,6 +202,50 @@ url-source:
   - "https://raw.githubusercontent.com/haskell/ghcup-metadata/master/ghcup-prereleases-0.0.7.yaml"
 ```
 
+## Stack integration
+
+Stack manages GHC versions internally by default. In order to make it use ghcup installed
+GHC versions there are two strategies.
+
+### Strategy 1: System GHC (works on all stack versions)
+
+You can instruct stack to use "system" GHC versions (whatever is in PATH). To do so,
+run the following commands:
+
+```sh
+stack config set install-ghc false --global
+stack config set system-ghc  true  --global
+```
+
+### Strategy 2: Stack hooks (new, recommended)
+
+Since stack 2.9.1 you can customize the installation logic of GHC completely, see [https://docs.haskellstack.org/en/stable/yaml_configuration/#ghc-installation-customisation](https://docs.haskellstack.org/en/stable/yaml_configuration/#ghc-installation-customisation).
+
+We can use this to simply invoke ghcup whenever stack is trying to install/discover a GHC versions. This
+is done via placing a shell script at `~/.stack/hooks/ghc-install.sh` and making it executable.
+
+The ghcup bootstrap script asks you during installation whether you want to install this shell script. You can also
+install/update it manually like so:
+
+```sh
+mkdir -p ~/.stack/hooks/
+curl https://raw.githubusercontent.com/haskell/ghcup-hs/master/scripts/hooks/stack/ghc-install.sh \
+  > ~/.stack/hooks/ghc-install.sh
+chmod +x ~/.stack/hooks/ghc-install.sh
+# hooks are only run when 'system-ghc: false'
+stack config set system-ghc false --global
+```
+
+By default, when the hook fails for whatever reason, stack will fall back to its own installation logic. To disable
+this, run `stack config set install-ghc false --global`.
+
+### Windows
+
+On windows, you may find the following config options useful too:
+`skip-msys`, `extra-path`, `extra-include-dirs`, `extra-lib-dirs`.
+
+Also check out: [https://docs.haskellstack.org/en/stable/yaml_configuration](https://docs.haskellstack.org/en/stable/yaml_configuration)
+
 # More on installation
 
 ## Customisation of the installation scripts

lib/GHCup/GHC.hs

@@ -676,7 +676,7 @@ compileGHC targetGhc ov bstrap jobs mbuildConfig patches aargs buildFlavour hadr
                          (view dlSubdir dlInfo)
         liftE $ applyAnyPatch patches (fromGHCupPath workdir)
 
-        pure (workdir, tmpUnpack, tver)
+        pure (workdir, tmpUnpack, Just tver)
 
       RemoteDist uri -> do
         lift $ logDebug $ "Requested to compile (from uri): " <> T.pack (show uri)
@@ -694,12 +694,13 @@ compileGHC targetGhc ov bstrap jobs mbuildConfig patches aargs buildFlavour hadr
                            execBlank
                            regex
             )
-          tver <- liftE $ getGHCVer (appendGHCupPath tmpUnpack (takeDirectory bootFile))
+          tver <- liftE $ catchAllE @_ @'[ProcessError, ParseError] @'[] (\_ -> pure Nothing) $ fmap Just $ getGHCVer
+            (appendGHCupPath tmpUnpack (takeDirectory bootFile))
           pure (bootFile, tver)
 
         let workdir = appendGHCupPath tmpUnpack (takeDirectory bf)
 
-        pure (workdir, tmpUnpack, mkTVer tver)
+        pure (workdir, tmpUnpack, mkTVer <$> tver)
 
       -- clone from git
       GitDist GitBranch{..} -> do
@@ -745,20 +746,23 @@ compileGHC targetGhc ov bstrap jobs mbuildConfig patches aargs buildFlavour hadr
           liftE $ applyAnyPatch patches (fromGHCupPath tmpUnpack)
 
           -- bootstrap
-          tver <- liftE $ getGHCVer tmpUnpack
+          tver <- liftE $ catchAllE @_ @'[ProcessError, ParseError] @'[] (\_ -> pure Nothing) $ fmap Just $ getGHCVer
+            tmpUnpack
           liftE $ catchWarn $ lEM @_ @'[ProcessError] $ darwinNotarization _rPlatform (fromGHCupPath tmpUnpack)
           lift $ logInfo $ "Examining git ref " <> T.pack ref <> "\n  " <>
-                           "GHC version (from Makefile): " <> prettyVer tver <>
+                           "GHC version (from Makefile): " <> T.pack (show (prettyVer <$> tver)) <>
                            (if not shallow_clone then "\n  " <> "'git describe' output: " <> fromJust git_describe else mempty) <>
                            (if isCommitHash ref then mempty else "\n  " <> "commit hash: " <> chash)
           liftIO $ threadDelay 5000000 -- give the user a sec to intervene
 
           pure tver
 
-        pure (tmpUnpack, tmpUnpack, GHCTargetVersion Nothing tver)
+        pure (tmpUnpack, tmpUnpack, mkTVer <$> tver)
     -- the version that's installed may differ from the
     -- compiled version, so the user can overwrite it
-    let installVer = maybe tver (\ov' -> tver { _tvVersion = ov' }) ov
+    installVer <- if | Just ov'   <- ov   -> pure (mkTVer ov')
+                     | Just tver' <- tver -> pure tver'
+                     | otherwise          -> fail "Newer GHCs don't support discovering the version in git. Complain to GHC devs: https://gitlab.haskell.org/ghc/ghc/-/issues/22322"
 
     alreadyInstalled <- lift $ ghcInstalled installVer
     alreadySet <- fmap (== Just installVer) $ lift $ ghcSet (_tvTarget installVer)
@@ -781,8 +785,10 @@ compileGHC targetGhc ov bstrap jobs mbuildConfig patches aargs buildFlavour hadr
       tmpUnpack
       (do
         b <- if hadrian
-             then compileHadrianBindist tver (fromGHCupPath workdir) ghcdir
-             else compileMakeBindist tver (fromGHCupPath workdir) ghcdir
+             -- prefer 'tver', because the real version carries out compatibility checks
+             -- we don't want the user to do funny things with it
+             then compileHadrianBindist (fromMaybe installVer tver) (fromGHCupPath workdir) ghcdir
+             else compileMakeBindist (fromMaybe installVer tver) (fromGHCupPath workdir) ghcdir
         bmk <- liftIO $ handleIO (\_ -> pure "") $ B.readFile (build_mk $ fromGHCupPath workdir)
         pure (b, bmk)
       )
@@ -826,14 +832,14 @@ compileGHC targetGhc ov bstrap jobs mbuildConfig patches aargs buildFlavour hadr
                , MonadThrow m
                )
             => GHCupPath
-            -> Excepts '[ProcessError] m Version
+            -> Excepts '[ProcessError, ParseError] m Version
   getGHCVer tmpUnpack = do
     lEM $ execWithGhcEnv "python3" ["./boot"] (Just $ fromGHCupPath tmpUnpack) "ghc-bootstrap"
     lEM $ execWithGhcEnv "sh" ["./configure"] (Just $ fromGHCupPath tmpUnpack) "ghc-bootstrap"
     CapturedProcess {..} <- lift $ makeOut
       ["show!", "--quiet", "VALUE=ProjectVersion" ] (Just $ fromGHCupPath tmpUnpack)
     case _exitCode of
-      ExitSuccess -> throwEither . MP.parse ghcProjectVersion "" . T.pack . stripNewlineEnd . T.unpack . decUTF8Safe' $ _stdOut
+      ExitSuccess -> either (throwE . ParseError . show) pure . MP.parse ghcProjectVersion "" . T.pack . stripNewlineEnd . T.unpack . decUTF8Safe' $ _stdOut
       ExitFailure c -> throwE $ NonZeroExit c "make" ["show!", "--quiet", "VALUE=ProjectVersion" ]
 
   defaultConf =