Fix CI

.gitlab-ci.yml

@@ -13,7 +13,7 @@ variables:
 
   # Sequential version number of all cached things.
   # Bump to invalidate GitLab CI cache.
-  CACHE_REV: 0
+  CACHE_REV: 1
 
   GIT_SUBMODULE_STRATEGY: recursive
 
@@ -125,6 +125,10 @@ variables:
       - test/golden
       - dist-newstyle/cache/
     when: on_failure
+  cache:
+    key: ghcup-test-$CACHE_REV
+    paths:
+      - cabal-cache
 
 # .test_ghcup_scoop:
   # script:
@@ -134,72 +138,77 @@ variables:
   extends:
     - .test_ghcup_version
     - .debian
-    - .root_cleanup
   before_script:
     - ./.gitlab/before_script/linux/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh
 
 .test_ghcup_version:linux32:
   extends:
     - .test_ghcup_version
     - .alpine:32bit
-    - .root_cleanup
   before_script:
     - ./.gitlab/before_script/linux/alpine/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh
 
 .test_ghcup_version:armv7:
   extends:
     - .test_ghcup_version
     - .linux:armv7
-    - .root_cleanup
   before_script:
     - ./.gitlab/before_script/linux/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh
 
 .test_ghcup_version:aarch64:
   extends:
     - .test_ghcup_version
     - .linux:aarch64
-    - .root_cleanup
   before_script:
     - ./.gitlab/before_script/linux/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh
 
 .test_ghcup_version:darwin:
   extends:
     - .test_ghcup_version
     - .darwin
-    - .root_cleanup
   before_script:
     - ./.gitlab/before_script/darwin/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh
 
 .test_ghcup_version:darwin:aarch64:
   extends:
     - .test_ghcup_version
     - .darwin:aarch64
-    - .root_cleanup
   cache:
     key: darwin-brew-$CACHE_REV
     paths:
-      - .brew
+      - brew_cache
-      - .brew_cache
+    key: ghcup-test-$CACHE_REV
+    paths:
+      - cabal-cache
   before_script:
-    # Install brew locally in the project dir. Packages will also be installed here.
+    # extract brew cache
-    - '[ -e "$CI_PROJECT_DIR/.brew" ] || git clone --depth=1 https://github.com/Homebrew/brew $CI_PROJECT_DIR/.brew'
+    - ./.gitlab/script/ci.sh extract_brew_cache
-    - export PATH="$CI_PROJECT_DIR/.brew/bin:$CI_PROJECT_DIR/.brew/sbin:$PATH"
-
     # otherwise we seem to get intel binaries
     - export HOMEBREW_CHANGE_ARCH_TO_ARM=1
-
-    # make sure to not pollute the machine with temp files etc
-    - mkdir -p $CI_PROJECT_DIR/.brew_cache
-    - export HOMEBREW_CACHE=$CI_PROJECT_DIR/.brew_cache
-    - mkdir -p $CI_PROJECT_DIR/.brew_logs
-    - export HOMEBREW_LOGS=$CI_PROJECT_DIR/.brew_logs
-    - mkdir -p /private/tmp/.brew_tmp
-    - export HOMEBREW_TEMP=/private/tmp/.brew_tmp
-
     # update and install packages
-    - brew update
+    - /bin/bash ./.gitlab/script/brew.sh llvm autoconf automake
-    - brew install llvm
+    # extract cabal cache
-    - brew install autoconf automake coreutils
+    - ./.gitlab/script/ci.sh extract_cabal_cache
   script: |
     export PATH="$CI_PROJECT_DIR/.brew/opt/llvm/bin:$CI_PROJECT_DIR/.brew/bin:$CI_PROJECT_DIR/.brew/sbin:$PATH"
     export CC=$CI_PROJECT_DIR/.brew/opt/llvm/bin/clang
@@ -209,40 +218,51 @@ variables:
     export RANLIB=$CI_PROJECT_DIR/.brew/opt/llvm/bin/llvm-ranlib
     ./.gitlab/before_script/darwin/install_deps.sh
     ./.gitlab/script/ghcup_version.sh
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - ./.gitlab/script/ci.sh save_brew_cache
+    - bash ./.gitlab/after_script.sh
 
 .test_ghcup_version:freebsd12:
   extends:
     - .test_ghcup_version
     - .freebsd12
-    - .root_cleanup
   before_script:
     - ./.gitlab/before_script/freebsd/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh
 
 .test_ghcup_version:freebsd13:
   extends:
     - .test_ghcup_version
     - .freebsd13
-    - .root_cleanup
   before_script:
     - sudo pkg update
     - sudo pkg install --yes compat12x-amd64
     - sudo ln -s libncurses.so.6 /usr/local/lib/libncurses.so.6.2
     - ./.gitlab/before_script/freebsd/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh
 
 .test_ghcup_version:windows:
   extends:
     - .test_ghcup_version
     - .windows
-    - .root_cleanup
   before_script:
-    - set CABAL_DIR="$CI_PROJECT_DIR/cabal"
     - bash ./.gitlab/before_script/windows/install_deps.sh
+    - bash ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - bash ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh
 
 # .test_ghcup_scoop:windows:
   # extends:
     # - .windows
     # - .test_ghcup_scoop
-    # - .root_cleanup
 
 .release_ghcup:
   script:
@@ -262,9 +282,12 @@ variables:
 test:linux:stack:
   stage: test
   before_script:
+    - ./.gitlab/script/ci.sh extract_stack_cache
     - ./.gitlab/before_script/linux/install_deps_minimal.sh
   script:
     - ./.gitlab/script/ghcup_stack.sh
+  after_script:
+    - ./.gitlab/script/ci.sh save_stack_cache
   extends:
     - .debian
   needs: []
@@ -294,6 +317,7 @@ test:windows:bootstrap_powershell_script:
     - "[Environment]::SetEnvironmentVariable('GHCUP_MSYS2', $null, [System.EnvironmentVariableTarget]::User)"
     - "[Environment]::SetEnvironmentVariable('CABAL_DIR', $null, [System.EnvironmentVariableTarget]::User)"
     - bash ./.gitlab/after_script.sh
+    - bash ./.gitlab/script/ci.sh save_cabal_cache
   variables:
     GHC_VERSION: "8.10.7"
     CABAL_VERSION: "3.6.2.0"
@@ -540,28 +564,17 @@ release:darwin:aarch64:
   cache:
     key: darwin-brew-$CACHE_REV
     paths:
-      - .brew
+      - brew_cache
-      - .brew_cache
+    key: ghcup-test-$CACHE_REV
+    paths:
+      - cabal-cache
   before_script:
-    # Install brew locally in the project dir. Packages will also be installed here.
+    - ./.gitlab/script/ci.sh extract_brew_cache
-    - '[ -e "$CI_PROJECT_DIR/.brew" ] || git clone --depth=1 https://github.com/Homebrew/brew $CI_PROJECT_DIR/.brew'
+    - ./.gitlab/script/ci.sh extract_cabal_cache
-    - export PATH="$CI_PROJECT_DIR/.brew/bin:$CI_PROJECT_DIR/.brew/sbin:$PATH"
-
     # otherwise we seem to get intel binaries
     - export HOMEBREW_CHANGE_ARCH_TO_ARM=1
-
-    # make sure to not pollute the machine with temp files etc
-    - mkdir -p $CI_PROJECT_DIR/.brew_cache
-    - export HOMEBREW_CACHE=$CI_PROJECT_DIR/.brew_cache
-    - mkdir -p $CI_PROJECT_DIR/.brew_logs
-    - export HOMEBREW_LOGS=$CI_PROJECT_DIR/.brew_logs
-    - mkdir -p /private/tmp/.brew_tmp
-    - export HOMEBREW_TEMP=/private/tmp/.brew_tmp
-
     # update and install packages
-    - brew update
+    - /bin/bash ./.gitlab/script/brew.sh llvm autoconf automake
-    - brew install llvm
-    - brew install autoconf automake
   script: |
     export PATH="$CI_PROJECT_DIR/.brew/opt/llvm/bin:$CI_PROJECT_DIR/.brew/bin:$CI_PROJECT_DIR/.brew/sbin:$PATH"
     export CC=$CI_PROJECT_DIR/.brew/opt/llvm/bin/clang
@@ -571,6 +584,9 @@ release:darwin:aarch64:
     export RANLIB=$CI_PROJECT_DIR/.brew/opt/llvm/bin/llvm-ranlib
     ./.gitlab/before_script/darwin/install_deps.sh
     ./.gitlab/script/ghcup_release.sh
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - ./.gitlab/script/ci.sh save_brew_cache
   variables:
     ARTIFACT: "aarch64-apple-darwin-ghcup"
     GHC_VERSION: "8.10.7"

.gitlab/ghcup_env

@@ -1,11 +1,23 @@
 if [ "${OS}" = "WINDOWS" ] ; then
-	export GHCUP_INSTALL_BASE_PREFIX="$CI_PROJECT_DIR"
+    export GHCUP_INSTALL_BASE_PREFIX="$CI_PROJECT_DIR"
-	export GHCUP_BIN="$CI_PROJECT_DIR/ghcup/bin"
+    export GHCUP_BIN="$CI_PROJECT_DIR/ghcup/bin"
-	export PATH="$GHCUP_BIN:$CI_PROJECT_DIR/.local/bin:$PATH"
+    export PATH="$GHCUP_BIN:$CI_PROJECT_DIR/.local/bin:$PATH"
-	export TMPDIR="$CI_PROJECT_DIR/tmp"
+    export TMPDIR="$CI_PROJECT_DIR/tmp"
+    export CABAL_DIR="$CI_PROJECT_DIR/cabal"
+    export CABAL_CACHE="$CI_PROJECT_DIR/cabal-cache"
+    export STACK_ROOT="$CI_PROJECT_DIR/stack"
+    export STACK_CACHE="$CI_PROJECT_DIR/stack-cache"
+    export BREW_DIR="$CI_PROJECT_DIR/.brew_cache"
+    export BREW_CACHE="$CI_PROJECT_DIR/brew-cache"
 else
-	export GHCUP_INSTALL_BASE_PREFIX="$CI_PROJECT_DIR"
+    export GHCUP_INSTALL_BASE_PREFIX="$CI_PROJECT_DIR"
-	export GHCUP_BIN="$CI_PROJECT_DIR/.ghcup/bin"
+    export GHCUP_BIN="$CI_PROJECT_DIR/.ghcup/bin"
-	export PATH="$GHCUP_BIN:$CI_PROJECT_DIR/.local/bin:$PATH"
+    export PATH="$GHCUP_BIN:$CI_PROJECT_DIR/.local/bin:$PATH"
-	export TMPDIR="$CI_PROJECT_DIR/tmp"
+    export TMPDIR="$CI_PROJECT_DIR/tmp"
+    export CABAL_DIR="$CI_PROJECT_DIR/cabal"
+    export CABAL_CACHE="$CI_PROJECT_DIR/cabal-cache"
+    export STACK_ROOT="$CI_PROJECT_DIR/stack"
+    export STACK_CACHE="$CI_PROJECT_DIR/stack-cache"
+    export BREW_DIR="$CI_PROJECT_DIR/.brew_cache"
+    export BREW_CACHE="$CI_PROJECT_DIR/brew-cache"
 fi

.gitlab/script/brew.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -Eeuxo pipefail
+
+# Install brew locally in the project dir. Packages will also be installed here.
+[ -e "$CI_PROJECT_DIR/.brew" ] || git clone --depth=1 https://github.com/Homebrew/brew $CI_PROJECT_DIR/.brew
+export PATH="$CI_PROJECT_DIR/.brew/bin:$CI_PROJECT_DIR/.brew/sbin:$PATH"
+
+# make sure to not pollute the machine with temp files etc
+mkdir -p $CI_PROJECT_DIR/.brew_cache
+export HOMEBREW_CACHE=$CI_PROJECT_DIR/.brew_cache
+mkdir -p $CI_PROJECT_DIR/.brew_logs
+export HOMEBREW_LOGS=$CI_PROJECT_DIR/.brew_logs
+mkdir -p /private/tmp/.brew_tmp
+export HOMEBREW_TEMP=/private/tmp/.brew_tmp
+
+# update and install packages
+brew update
+brew install ${1+"$@"}

.gitlab/script/ci.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+TOP="$( cd "$(dirname "$0")" ; pwd -P )"
+. "${TOP}/../ghcup_env"
+
+function save_cabal_cache () {
+  echo "Storing cabal cache from $CABAL_DIR to $CABAL_CACHE..."
+  rm -Rf "$CABAL_CACHE"
+  mkdir -p "$CABAL_CACHE"
+  if [ -d "$CABAL_DIR" ]; then
+	  cp -Rf "$CABAL_DIR" "$CABAL_CACHE/"
+  fi
+}
+
+
+function extract_cabal_cache () {
+  if [ -d "$CABAL_CACHE" ]; then
+      echo "Extracting cabal cache from $CABAL_CACHE to $CABAL_DIR..."
+      mkdir -p "$CABAL_DIR"
+      cp -Rf "$CABAL_CACHE"/* "$CABAL_DIR"
+  fi
+}
+
+function save_stack_cache () {
+  echo "Storing stack cache from $STACK_ROOT to $STACK_CACHE..."
+  rm -Rf "$STACK_CACHE"
+  mkdir -p "$STACK_CACHE"
+  if [ -d "$STACK_ROOT" ]; then
+	  cp -Rf "$STACK_DIR" "$STACK_CACHE"
+  fi
+}
+
+
+function extract_stack_cache () {
+  if [ -d "$STACK_CACHE" ]; then
+      echo "Extracting stack cache from $STACK_CACHE to $STACK_ROOT..."
+      mkdir -p "$STACK_ROOT"
+      cp -Rf "$STACK_CACHE"/* "$STACK_ROOT"
+  fi
+}
+
+function save_brew_cache () {
+  echo "Storing brew cache from $BREW_DIR to $BREW_CACHE..."
+  rm -Rf "$BREW_CACHE"
+  mkdir -p "$BREW_CACHE"
+  if [ -d "$BREW_DIR" ]; then
+	  cp -Rf "$BREW_DIR" "$BREW_CACHE"
+  fi
+}
+
+
+function extract_brew_cache () {
+  if [ -d "$BREW_CACHE" ]; then
+      echo "Extracting stack cache from $BREW_CACHE to $BREW_DIR..."
+      mkdir -p "$BREW_DIR"
+      cp -Rf "$BREW_CACHE"/* "$BREW_DIR"
+  fi
+}
+
+case $1 in
+  extract_cabal_cache) extract_cabal_cache ;;
+  save_cabal_cache) save_cabal_cache ;;
+  extract_stack_cache) extract_stack_cache ;;
+  save_stack_cache) save_stack_cache ;;
+  extract_brew_cache) extract_brew_cache ;;
+  save_brew_cache) save_brew_cache ;;
+  *) echo "unknown mode $1" ; exit 11 ;;
+esac

.gitlab/script/ghcup_version.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 set -eux
 
@@ -8,6 +8,7 @@ mkdir -p "$CI_PROJECT_DIR"/.local/bin
 
 CI_PROJECT_DIR=$(pwd)
 
+
 ecabal() {
 	cabal "$@"
 }
@@ -94,16 +95,17 @@ rm -rf "${GHCUP_DIR}"
 eghcup --numeric-version
 
 eghcup install ghc ${GHC_VERSION}
-[ `$(eghcup whereis ghc ${GHC_VERSION}) --numeric-version` = "${GHC_VERSION}" ]
+ls -lah "$(eghcup whereis -d ghc ${GHC_VERSION})"
-[ `eghcup run --ghc ${GHC_VERSION} -- ghc --numeric-version` = "${GHC_VERSION}" ]
+[ "`$(eghcup whereis ghc ${GHC_VERSION}) --numeric-version`" = "${GHC_VERSION}" ]
+[ "`eghcup run --ghc ${GHC_VERSION} -- ghc --numeric-version`" = "${GHC_VERSION}" ]
 eghcup set ghc ${GHC_VERSION}
 eghcup install cabal ${CABAL_VERSION}
-[ `$(eghcup whereis cabal ${CABAL_VERSION}) --numeric-version` = "${CABAL_VERSION}" ]
+[ "`$(eghcup whereis cabal ${CABAL_VERSION}) --numeric-version`" = "${CABAL_VERSION}" ]
 eghcup unset cabal
 "$GHCUP_BIN"/cabal --version && exit 1 || echo yes
 eghcup set cabal ${CABAL_VERSION}
-[ `$(eghcup whereis cabal ${CABAL_VERSION}) --numeric-version` = "${CABAL_VERSION}" ]
+[ "`$(eghcup whereis cabal ${CABAL_VERSION}) --numeric-version`" = "${CABAL_VERSION}" ]
-[ `eghcup run --cabal ${CABAL_VERSION} -- cabal --numeric-version` = "${CABAL_VERSION}" ]
+[ "`eghcup run --cabal ${CABAL_VERSION} -- cabal --numeric-version`" = "${CABAL_VERSION}" ]
 
 if [ "${OS}" != "FREEBSD" ] ; then
 	if [ "${ARCH}" = "64" ] ; then