Fix CI

2022-05-13 10:32:44 +02:00 · 2022-05-13 10:32:44 +02:00 · fa924eac15
commit fa924eac15
parent db4e411dfd
5 changed files with 181 additions and 62 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -13,7 +13,7 @@ variables:

  # Sequential version number of all cached things.
  # Bump to invalidate GitLab CI cache.
-  CACHE_REV: 0
+  CACHE_REV: 1

  GIT_SUBMODULE_STRATEGY: recursive

@ -125,6 +125,10 @@ variables:
      - test/golden
      - dist-newstyle/cache/
    when: on_failure
+  cache:
+    key: ghcup-test-$CACHE_REV
+    paths:
+      - cabal-cache

 # .test_ghcup_scoop:
  # script:
@ -134,72 +138,77 @@ variables:
  extends:
    - .test_ghcup_version
    - .debian
-    - .root_cleanup
  before_script:
    - ./.gitlab/before_script/linux/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh

 .test_ghcup_version:linux32:
  extends:
    - .test_ghcup_version
    - .alpine:32bit
-    - .root_cleanup
  before_script:
    - ./.gitlab/before_script/linux/alpine/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh

 .test_ghcup_version:armv7:
  extends:
    - .test_ghcup_version
    - .linux:armv7
-    - .root_cleanup
  before_script:
    - ./.gitlab/before_script/linux/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh

 .test_ghcup_version:aarch64:
  extends:
    - .test_ghcup_version
    - .linux:aarch64
-    - .root_cleanup
  before_script:
    - ./.gitlab/before_script/linux/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh

 .test_ghcup_version:darwin:
  extends:
    - .test_ghcup_version
    - .darwin
-    - .root_cleanup
  before_script:
    - ./.gitlab/before_script/darwin/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh

 .test_ghcup_version:darwin:aarch64:
  extends:
    - .test_ghcup_version
    - .darwin:aarch64
-    - .root_cleanup
  cache:
    key: darwin-brew-$CACHE_REV
    paths:
-      - .brew
-      - .brew_cache
+      - brew_cache
+    key: ghcup-test-$CACHE_REV
+    paths:
+      - cabal-cache
  before_script:
-    # Install brew locally in the project dir. Packages will also be installed here.
-    - '[ -e "$CI_PROJECT_DIR/.brew" ] || git clone --depth=1 https://github.com/Homebrew/brew $CI_PROJECT_DIR/.brew'
-    - export PATH="$CI_PROJECT_DIR/.brew/bin:$CI_PROJECT_DIR/.brew/sbin:$PATH"
-
+    # extract brew cache
+    - ./.gitlab/script/ci.sh extract_brew_cache
    # otherwise we seem to get intel binaries
    - export HOMEBREW_CHANGE_ARCH_TO_ARM=1
-
-    # make sure to not pollute the machine with temp files etc
-    - mkdir -p $CI_PROJECT_DIR/.brew_cache
-    - export HOMEBREW_CACHE=$CI_PROJECT_DIR/.brew_cache
-    - mkdir -p $CI_PROJECT_DIR/.brew_logs
-    - export HOMEBREW_LOGS=$CI_PROJECT_DIR/.brew_logs
-    - mkdir -p /private/tmp/.brew_tmp
-    - export HOMEBREW_TEMP=/private/tmp/.brew_tmp
-
    # update and install packages
-    - brew update
-    - brew install llvm
-    - brew install autoconf automake coreutils
+    - /bin/bash ./.gitlab/script/brew.sh llvm autoconf automake
+    # extract cabal cache
+    - ./.gitlab/script/ci.sh extract_cabal_cache
  script: |
    export PATH="$CI_PROJECT_DIR/.brew/opt/llvm/bin:$CI_PROJECT_DIR/.brew/bin:$CI_PROJECT_DIR/.brew/sbin:$PATH"
    export CC=$CI_PROJECT_DIR/.brew/opt/llvm/bin/clang
@ -209,40 +218,51 @@ variables:
    export RANLIB=$CI_PROJECT_DIR/.brew/opt/llvm/bin/llvm-ranlib
    ./.gitlab/before_script/darwin/install_deps.sh
    ./.gitlab/script/ghcup_version.sh
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - ./.gitlab/script/ci.sh save_brew_cache
+    - bash ./.gitlab/after_script.sh

 .test_ghcup_version:freebsd12:
  extends:
    - .test_ghcup_version
    - .freebsd12
-    - .root_cleanup
  before_script:
    - ./.gitlab/before_script/freebsd/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh

 .test_ghcup_version:freebsd13:
  extends:
    - .test_ghcup_version
    - .freebsd13
-    - .root_cleanup
  before_script:
    - sudo pkg update
    - sudo pkg install --yes compat12x-amd64
    - sudo ln -s libncurses.so.6 /usr/local/lib/libncurses.so.6.2
    - ./.gitlab/before_script/freebsd/install_deps.sh
+    - ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh

 .test_ghcup_version:windows:
  extends:
    - .test_ghcup_version
    - .windows
-    - .root_cleanup
  before_script:
-    - set CABAL_DIR="$CI_PROJECT_DIR/cabal"
    - bash ./.gitlab/before_script/windows/install_deps.sh
+    - bash ./.gitlab/script/ci.sh extract_cabal_cache
+  after_script:
+    - bash ./.gitlab/script/ci.sh save_cabal_cache
+    - bash ./.gitlab/after_script.sh

 # .test_ghcup_scoop:windows:
  # extends:
    # - .windows
    # - .test_ghcup_scoop
-    # - .root_cleanup

 .release_ghcup:
  script:
@ -262,9 +282,12 @@ variables:
 test:linux:stack:
  stage: test
  before_script:
+    - ./.gitlab/script/ci.sh extract_stack_cache
    - ./.gitlab/before_script/linux/install_deps_minimal.sh
  script:
    - ./.gitlab/script/ghcup_stack.sh
+  after_script:
+    - ./.gitlab/script/ci.sh save_stack_cache
  extends:
    - .debian
  needs: []
@ -294,6 +317,7 @@ test:windows:bootstrap_powershell_script:
    - "[Environment]::SetEnvironmentVariable('GHCUP_MSYS2', $null, [System.EnvironmentVariableTarget]::User)"
    - "[Environment]::SetEnvironmentVariable('CABAL_DIR', $null, [System.EnvironmentVariableTarget]::User)"
    - bash ./.gitlab/after_script.sh
+    - bash ./.gitlab/script/ci.sh save_cabal_cache
  variables:
    GHC_VERSION: "8.10.7"
    CABAL_VERSION: "3.6.2.0"
@ -540,28 +564,17 @@ release:darwin:aarch64:
  cache:
    key: darwin-brew-$CACHE_REV
    paths:
-      - .brew
-      - .brew_cache
+      - brew_cache
+    key: ghcup-test-$CACHE_REV
+    paths:
+      - cabal-cache
  before_script:
-    # Install brew locally in the project dir. Packages will also be installed here.
-    - '[ -e "$CI_PROJECT_DIR/.brew" ] || git clone --depth=1 https://github.com/Homebrew/brew $CI_PROJECT_DIR/.brew'
-    - export PATH="$CI_PROJECT_DIR/.brew/bin:$CI_PROJECT_DIR/.brew/sbin:$PATH"
-
+    - ./.gitlab/script/ci.sh extract_brew_cache
+    - ./.gitlab/script/ci.sh extract_cabal_cache
    # otherwise we seem to get intel binaries
    - export HOMEBREW_CHANGE_ARCH_TO_ARM=1
-
-    # make sure to not pollute the machine with temp files etc
-    - mkdir -p $CI_PROJECT_DIR/.brew_cache
-    - export HOMEBREW_CACHE=$CI_PROJECT_DIR/.brew_cache
-    - mkdir -p $CI_PROJECT_DIR/.brew_logs
-    - export HOMEBREW_LOGS=$CI_PROJECT_DIR/.brew_logs
-    - mkdir -p /private/tmp/.brew_tmp
-    - export HOMEBREW_TEMP=/private/tmp/.brew_tmp
-
    # update and install packages
-    - brew update
-    - brew install llvm
-    - brew install autoconf automake
+    - /bin/bash ./.gitlab/script/brew.sh llvm autoconf automake
  script: |
    export PATH="$CI_PROJECT_DIR/.brew/opt/llvm/bin:$CI_PROJECT_DIR/.brew/bin:$CI_PROJECT_DIR/.brew/sbin:$PATH"
    export CC=$CI_PROJECT_DIR/.brew/opt/llvm/bin/clang
@ -571,6 +584,9 @@ release:darwin:aarch64:
    export RANLIB=$CI_PROJECT_DIR/.brew/opt/llvm/bin/llvm-ranlib
    ./.gitlab/before_script/darwin/install_deps.sh
    ./.gitlab/script/ghcup_release.sh
+  after_script:
+    - ./.gitlab/script/ci.sh save_cabal_cache
+    - ./.gitlab/script/ci.sh save_brew_cache
  variables:
    ARTIFACT: "aarch64-apple-darwin-ghcup"
    GHC_VERSION: "8.10.7"
--- a/.gitlab/ghcup_env
+++ b/.gitlab/ghcup_env
@ -1,11 +1,23 @@
 if [ "${OS}" = "WINDOWS" ] ; then
-	export GHCUP_INSTALL_BASE_PREFIX="$CI_PROJECT_DIR"
-	export GHCUP_BIN="$CI_PROJECT_DIR/ghcup/bin"
-	export PATH="$GHCUP_BIN:$CI_PROJECT_DIR/.local/bin:$PATH"
-	export TMPDIR="$CI_PROJECT_DIR/tmp"
+    export GHCUP_INSTALL_BASE_PREFIX="$CI_PROJECT_DIR"
+    export GHCUP_BIN="$CI_PROJECT_DIR/ghcup/bin"
+    export PATH="$GHCUP_BIN:$CI_PROJECT_DIR/.local/bin:$PATH"
+    export TMPDIR="$CI_PROJECT_DIR/tmp"
+    export CABAL_DIR="$CI_PROJECT_DIR/cabal"
+    export CABAL_CACHE="$CI_PROJECT_DIR/cabal-cache"
+    export STACK_ROOT="$CI_PROJECT_DIR/stack"
+    export STACK_CACHE="$CI_PROJECT_DIR/stack-cache"
+    export BREW_DIR="$CI_PROJECT_DIR/.brew_cache"
+    export BREW_CACHE="$CI_PROJECT_DIR/brew-cache"
 else
-	export GHCUP_INSTALL_BASE_PREFIX="$CI_PROJECT_DIR"
-	export GHCUP_BIN="$CI_PROJECT_DIR/.ghcup/bin"
-	export PATH="$GHCUP_BIN:$CI_PROJECT_DIR/.local/bin:$PATH"
-	export TMPDIR="$CI_PROJECT_DIR/tmp"
+    export GHCUP_INSTALL_BASE_PREFIX="$CI_PROJECT_DIR"
+    export GHCUP_BIN="$CI_PROJECT_DIR/.ghcup/bin"
+    export PATH="$GHCUP_BIN:$CI_PROJECT_DIR/.local/bin:$PATH"
+    export TMPDIR="$CI_PROJECT_DIR/tmp"
+    export CABAL_DIR="$CI_PROJECT_DIR/cabal"
+    export CABAL_CACHE="$CI_PROJECT_DIR/cabal-cache"
+    export STACK_ROOT="$CI_PROJECT_DIR/stack"
+    export STACK_CACHE="$CI_PROJECT_DIR/stack-cache"
+    export BREW_DIR="$CI_PROJECT_DIR/.brew_cache"
+    export BREW_CACHE="$CI_PROJECT_DIR/brew-cache"
 fi
--- a/.gitlab/script/brew.sh
+++ b/.gitlab/script/brew.sh
@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -Eeuxo pipefail
+
+# Install brew locally in the project dir. Packages will also be installed here.
+[ -e "$CI_PROJECT_DIR/.brew" ] || git clone --depth=1 https://github.com/Homebrew/brew $CI_PROJECT_DIR/.brew
+export PATH="$CI_PROJECT_DIR/.brew/bin:$CI_PROJECT_DIR/.brew/sbin:$PATH"
+
+# make sure to not pollute the machine with temp files etc
+mkdir -p $CI_PROJECT_DIR/.brew_cache
+export HOMEBREW_CACHE=$CI_PROJECT_DIR/.brew_cache
+mkdir -p $CI_PROJECT_DIR/.brew_logs
+export HOMEBREW_LOGS=$CI_PROJECT_DIR/.brew_logs
+mkdir -p /private/tmp/.brew_tmp
+export HOMEBREW_TEMP=/private/tmp/.brew_tmp
+
+# update and install packages
+brew update
+brew install ${1+"$@"}
--- a/.gitlab/script/ci.sh
+++ b/.gitlab/script/ci.sh
@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+TOP="$( cd "$(dirname "$0")" ; pwd -P )"
+. "${TOP}/../ghcup_env"
+
+function save_cabal_cache () {
+  echo "Storing cabal cache from $CABAL_DIR to $CABAL_CACHE..."
+  rm -Rf "$CABAL_CACHE"
+  mkdir -p "$CABAL_CACHE"
+  if [ -d "$CABAL_DIR" ]; then
+	  cp -Rf "$CABAL_DIR" "$CABAL_CACHE/"
+  fi
+}
+
+
+function extract_cabal_cache () {
+  if [ -d "$CABAL_CACHE" ]; then
+      echo "Extracting cabal cache from $CABAL_CACHE to $CABAL_DIR..."
+      mkdir -p "$CABAL_DIR"
+      cp -Rf "$CABAL_CACHE"/* "$CABAL_DIR"
+  fi
+}
+
+function save_stack_cache () {
+  echo "Storing stack cache from $STACK_ROOT to $STACK_CACHE..."
+  rm -Rf "$STACK_CACHE"
+  mkdir -p "$STACK_CACHE"
+  if [ -d "$STACK_ROOT" ]; then
+	  cp -Rf "$STACK_DIR" "$STACK_CACHE"
+  fi
+}
+
+
+function extract_stack_cache () {
+  if [ -d "$STACK_CACHE" ]; then
+      echo "Extracting stack cache from $STACK_CACHE to $STACK_ROOT..."
+      mkdir -p "$STACK_ROOT"
+      cp -Rf "$STACK_CACHE"/* "$STACK_ROOT"
+  fi
+}
+
+function save_brew_cache () {
+  echo "Storing brew cache from $BREW_DIR to $BREW_CACHE..."
+  rm -Rf "$BREW_CACHE"
+  mkdir -p "$BREW_CACHE"
+  if [ -d "$BREW_DIR" ]; then
+	  cp -Rf "$BREW_DIR" "$BREW_CACHE"
+  fi
+}
+
+
+function extract_brew_cache () {
+  if [ -d "$BREW_CACHE" ]; then
+      echo "Extracting stack cache from $BREW_CACHE to $BREW_DIR..."
+      mkdir -p "$BREW_DIR"
+      cp -Rf "$BREW_CACHE"/* "$BREW_DIR"
+  fi
+}
+
+case $1 in
+  extract_cabal_cache) extract_cabal_cache ;;
+  save_cabal_cache) save_cabal_cache ;;
+  extract_stack_cache) extract_stack_cache ;;
+  save_stack_cache) save_stack_cache ;;
+  extract_brew_cache) extract_brew_cache ;;
+  save_brew_cache) save_brew_cache ;;
+  *) echo "unknown mode $1" ; exit 11 ;;
+esac
--- a/.gitlab/script/ghcup_version.sh
+++ b/.gitlab/script/ghcup_version.sh
@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash

 set -eux

@ -8,6 +8,7 @@ mkdir -p "$CI_PROJECT_DIR"/.local/bin

 CI_PROJECT_DIR=$(pwd)

+
 ecabal() {
 	cabal "$@"
 }
@ -94,16 +95,17 @@ rm -rf "${GHCUP_DIR}"
 eghcup --numeric-version

 eghcup install ghc ${GHC_VERSION}
-[ `$(eghcup whereis ghc ${GHC_VERSION}) --numeric-version` = "${GHC_VERSION}" ]
-[ `eghcup run --ghc ${GHC_VERSION} -- ghc --numeric-version` = "${GHC_VERSION}" ]
+ls -lah "$(eghcup whereis -d ghc ${GHC_VERSION})"
+[ "`$(eghcup whereis ghc ${GHC_VERSION}) --numeric-version`" = "${GHC_VERSION}" ]
+[ "`eghcup run --ghc ${GHC_VERSION} -- ghc --numeric-version`" = "${GHC_VERSION}" ]
 eghcup set ghc ${GHC_VERSION}
 eghcup install cabal ${CABAL_VERSION}
-[ `$(eghcup whereis cabal ${CABAL_VERSION}) --numeric-version` = "${CABAL_VERSION}" ]
+[ "`$(eghcup whereis cabal ${CABAL_VERSION}) --numeric-version`" = "${CABAL_VERSION}" ]
 eghcup unset cabal
 "$GHCUP_BIN"/cabal --version && exit 1 || echo yes
 eghcup set cabal ${CABAL_VERSION}
-[ `$(eghcup whereis cabal ${CABAL_VERSION}) --numeric-version` = "${CABAL_VERSION}" ]
-[ `eghcup run --cabal ${CABAL_VERSION} -- cabal --numeric-version` = "${CABAL_VERSION}" ]
+[ "`$(eghcup whereis cabal ${CABAL_VERSION}) --numeric-version`" = "${CABAL_VERSION}" ]
+[ "`eghcup run --cabal ${CABAL_VERSION} -- cabal --numeric-version`" = "${CABAL_VERSION}" ]

 if [ "${OS}" != "FREEBSD" ] ; then
 	if [ "${ARCH}" = "64" ] ; then