Fixup

.gitlab-ci.yml

@@ -103,7 +103,7 @@ variables:
   artifacts:
     expire_in: 2 week
     paths:
-      - golden
+      - test/golden
       - dist-newstyle/cache/
     when: on_failure
 
@@ -240,7 +240,7 @@ test:linux:bootstrap_script:
 test:windows:bootstrap_powershell_script:
   stage: test
   script:
-    - ./bootstrap-haskell.ps1 -InstallDir $CI_PROJECT_DIR -BootstrapUrl $CI_PROJECT_DIR/bootstrap-haskell -InBash
+    - ./scripts/bootstrap/bootstrap-haskell.ps1 -InstallDir $CI_PROJECT_DIR -BootstrapUrl $CI_PROJECT_DIR/bootstrap-haskell -InBash
   after_script:
     - "[Environment]::SetEnvironmentVariable('GHCUP_INSTALL_BASE_PREFIX', $null, [System.EnvironmentVariableTarget]::User)"
     - "[Environment]::SetEnvironmentVariable('GHCUP_MSYS2', $null, [System.EnvironmentVariableTarget]::User)"

.gitlab/script/ghcup_bootstrap.sh

@@ -24,7 +24,7 @@ export BOOTSTRAP_HASKELL_NONINTERACTIVE=yes
 export BOOTSTRAP_HASKELL_GHC_VERSION=$GHC_VERSION
 export BOOTSTRAP_HASKELL_CABAL_VERSION=$CABAL_VERSION
 
-./bootstrap-haskell
+./scripts/bootstrap/bootstrap-haskell
 
 [ "$(ghc --numeric-version)" = "${GHC_VERSION}" ]
 

.gitlab/script/ghcup_cross.sh

@@ -13,7 +13,7 @@ ecabal() {
 }
 
 eghcup() {
-	ghcup -v -c -s file://$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml "$@"
+	ghcup -v -c -s file://$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml "$@"
 }
 
 git describe --always

.gitlab/script/ghcup_git.sh

@@ -13,7 +13,7 @@ ecabal() {
 }
 
 eghcup() {
-	ghcup -v -c -s file://$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml "$@"
+	ghcup -v -c -s file://$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml "$@"
 }
 
 git describe --always

.gitlab/script/ghcup_version.sh

@@ -18,9 +18,9 @@ raw_eghcup() {
 
 eghcup() {
 	if [ "${OS}" = "WINDOWS" ] ; then
-		ghcup -v -c -s file:/$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml "$@"
+		ghcup -v -c -s file:/$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml "$@"
 	else
-		ghcup -v -c -s file://$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml "$@"
+		ghcup -v -c -s file://$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml "$@"
 	fi
 }
 
@@ -92,7 +92,7 @@ rm -rf "${GHCUP_DIR}"
 ### manual cli based testing
 
 
-ghcup-gen check -f ghcup-${JSON_VERSION}.yaml
+ghcup-gen check -f data/metadata/ghcup-${JSON_VERSION}.yaml
 
 eghcup --numeric-version
 
@@ -172,10 +172,10 @@ else
 fi
 
 # check that lazy loading works for 'whereis'
-cp "$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml" "$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml.bak"
-echo '**' > "$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml"
+cp "$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml" "$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml.bak"
+echo '**' > "$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml"
 eghcup whereis ghc $(ghc --numeric-version)
-mv -f "$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml.bak" "$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml"
+mv -f "$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml.bak" "$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml"
 
 eghcup rm $(ghc --numeric-version)
 

README.md

@@ -19,7 +19,7 @@ Similar in scope to [rustup](https://github.com/rust-lang-nursery/rustup.rs), [p
      * [Configuration](#configuration)
      * [Manpages](#manpages)
      * [Shell-completion](#shell-completion)
-     * [Cross support](#cross-support)
+     * [Compiling from sourc](#compiling-from-source)
      * [XDG support](#xdg-support)
      * [Env variables](#env-variables)
      * [Installing custom bindists](#installing-custom-bindists)
@@ -91,7 +91,7 @@ handles your haskell packages and can demand that [a specific version](https://c
 ### Configuration
 
 A configuration file can be put in `~/.ghcup/config.yaml`. The default config file
-explaining all possible configurations can be found in this repo: [config.yaml](./config.yaml).
+explaining all possible configurations can be found in this repo: [config.yaml](./data/config.yaml).
 
 Partial configuration is fine. Command line options always override the config file settings.
 
@@ -102,14 +102,26 @@ For man pages to work you need [man-db](http://man-db.nongnu.org/) as your `man`
 
 ### Shell-completion
 
-Shell completions are in `shell-completions`.
+Shell completions are in [scripts/shell-completions](./scripts/shell-completions) directory of this repository.
 
 For bash: install `shell-completions/bash`
 as e.g. `/etc/bash_completion.d/ghcup` (depending on distro)
 and make sure your bashrc sources the startup script
 (`/usr/share/bash-completion/bash_completion` on some distros).
 
-### Cross support
+### Compiling from source
+
+Compiling from source is supported for both source tarballs and arbitrary git refs. See `ghcup compile ghc --help`
+for a list of all available options.
+
+If you need to overwrite the existing `build.mk`, check the default files
+in [data/build_mk](./data/build_mk), copy them somewhere, adjust them and
+pass `--config path/to/build.mk` to `ghcup compile ghc`.
+Common `build.mk` options are explained [here](https://gitlab.haskell.org/ghc/ghc/-/wikis/building/using#build-configuration).
+
+Make sure your system meets all the [prerequisites](https://gitlab.haskell.org/ghc/ghc/-/wikis/building/preparation).
+
+#### Cross support
 
 ghcup can compile and install a cross GHC for any target. However, this
 requires that the build host has a complete cross toolchain and various
@@ -241,14 +253,15 @@ jobs:
       name: Install ghcup on windows
       run: Set-ExecutionPolicy Bypass -Scope Process -Force;[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072;Invoke-Command -ScriptBlock ([ScriptBlock]::Create((Invoke-WebRequest https://www.haskell.org/ghcup/sh/bootstrap-haskell.ps1 -UseBasicParsing))) -ArgumentList $false,$true,$true,$false,$false,$false,$false,"C:\"
 
+    - if: matrix.os == 'windows-latest'
+      name: Add ghcup to PATH
+      run: echo "/c/ghcup/bin" >> $GITHUB_PATH
+      shell: bash
+
     - if: matrix.os != 'windows-latest'
       name: Install ghcup on non-windows
       run: curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 BOOTSTRAP_HASKELL_MINIMAL=1 sh
 
-    - if: matrix.os == 'windows-latest'
-      run: echo "/c/ghcup/bin" >> $GITHUB_PATH
-      shell: bash
-
     - name: Install ghc/cabal
       run: |
         ghcup install ghc ${{ matrix.ghc }}
@@ -266,50 +279,6 @@ jobs:
     - name: Run tests
       run: cabal test
       shell: bash
-
-    - name: Run benches
-      run: cabal bench
-      shell: bash
-
-  build-stack:
-    name: Stack ${{ matrix.stack }} ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macOS-latest, windows-latest]
-        stack: ['latest']
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - if: matrix.os == 'windows-latest'
-      name: Install ghcup on windows
-      run: Set-ExecutionPolicy Bypass -Scope Process -Force;[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072;Invoke-Command -ScriptBlock ([ScriptBlock]::Create((Invoke-WebRequest https://www.haskell.org/ghcup/sh/bootstrap-haskell.ps1 -UseBasicParsing))) -ArgumentList $false,$true,$true,$false,$false,$false,$false,"C:\"
-
-    - if: matrix.os != 'windows-latest'
-      name: Install ghcup on non-windows
-      run: curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 BOOTSTRAP_HASKELL_MINIMAL=1 sh
-
-    - if: matrix.os == 'windows-latest'
-      run: echo "/c/ghcup/bin" >> $GITHUB_PATH
-      shell: bash
-
-    - name: Install stack
-      run: ghcup install stack ${{ matrix.stack }}
-      shell: bash
-
-    - name: Build
-      run: stack build
-      shell: bash
-
-    - name: Run tests
-      run: stack test
-      shell: bash
-
-    - name: Run benches
-      run: stack bench
-      shell: bash
 ```
    
 ### Tips and tricks
@@ -387,8 +356,13 @@ In addition this script can also install `cabal-install`.
 
 ## Known users
 
-* Github action [haskell/actions/setup](https://github.com/haskell/actions/tree/main/setup)
-* [vabal](https://github.com/Franciman/vabal)
+* Github actions:
+	- [actions/virtual-environments](https://github.com/actions/virtual-environments)
+	- [haskell/actions/setup](https://github.com/haskell/actions/tree/main/setup)
+* mirrors:
+	- [sjtug](https://mirror.sjtu.edu.cn/docs/ghcup)
+* tools:
+	- [vabal](https://github.com/Franciman/vabal)
 
 ## Known problems
 

RELEASING.md

@@ -1,19 +0,0 @@
-# RELEASING
-
-1. update `GHCup.Version` module. `ghcupURL` must only be updated if we change the `_toolRequirements` type or the YAML representation of it. The version of the YAML represents the change increments. `ghcUpVer` is the current application version.
-
-2. Update version in ghcup.cabal
-
-3. Add ChangeLog entry
-
-4. Add/fix downloads in `ghcup-<ver>.yaml`, then verify with `ghcup-gen check -f ghcup-<ver>.yaml`
-
-5. Commit and git push with tag. Wait for tests to succeed and release artifacts to build.
-
-6. Download release artifacts and upload them `downloads.haskell.org/ghcup`
-
-7. Add release artifacts to yaml file (see point 4.)
-
-8. Upload the final `ghcup-<ver>.yaml` to `webhost.haskell.org/ghcup/data/`.
-
-9. Update bootstrap-haskell and symlinks on `downloads.haskell.org/ghcup`

app/ghcup/Main.hs

@@ -57,6 +57,7 @@ import           Data.Void
 import           GHC.IO.Encoding
 import           Haskus.Utils.Variant.Excepts
 import           Language.Haskell.TH
+import           Language.Haskell.TH.Syntax     ( Quasi(qAddDependentFile) )
 import           Options.Applicative     hiding ( style )
 import           Options.Applicative.Help.Pretty ( text )
 import           Prelude                 hiding ( appendFile )
@@ -1045,7 +1046,7 @@ ghcCompileOpts =
           (option
             str
             (short 'p' <> long "patchdir" <> metavar "PATCH_DIR" <> help
-              "Absolute path to patch directory (applied in order, uses -p1)"
+              "Absolute path to patch directory (applies all .patch and .diff files in order using -p1)"
             )
           )
     <*> optional
@@ -1402,14 +1403,14 @@ describe_result = $( LitE . StringL <$>
                    )
 
 plan_json :: String
-plan_json = $( LitE . StringL <$>
-                     runIO (handleIO (\_ -> pure "") $ do
+plan_json = $( do
+                (fp, c) <- runIO (handleIO (\_ -> pure ("", "")) $ do
                              fp <- findPlanJson (ProjectRelativeToDir ".")
                              c <- B.readFile fp
                              (Just res) <- pure $ decodeStrict' @Value c
-                             pure $ T.unpack $ decUTF8Safe' $ encodePretty res
-                     )
-                   )
+                             pure (fp, T.unpack $ decUTF8Safe' $ encodePretty res))
+                when (not . null $ fp ) $ qAddDependentFile fp
+                pure . LitE . StringL $ c)
 
 formatConfig :: UserSettings -> String
 formatConfig settings

data/build_mk/cross

@@ -0,0 +1,9 @@
+V=0
+BUILD_MAN = NO
+BUILD_SPHINX_HTML = NO
+BUILD_SPHINX_PDF = NO
+HADDOCK_DOCS = NO
+ifneq "$(BuildFlavour)" ""
+include mk/flavours/$(BuildFlavour).mk
+endif
+Stage1Only = YES

data/build_mk/default

@@ -0,0 +1,8 @@
+V=0
+BUILD_MAN = NO
+BUILD_SPHINX_HTML = NO
+BUILD_SPHINX_PDF = NO
+HADDOCK_DOCS = YES
+ifneq "$(BuildFlavour)" ""
+include mk/flavours/$(BuildFlavour).mk
+endif

docs/RELEASING.md

@@ -0,0 +1,21 @@
+# RELEASING
+
+1. Update version in `ghcup.cabal` and `boostrap-haskell` (`ghver` variable at the top of the script)
+
+2. Update `GHCup.Version` module. `ghcupURL` must only be updated if we change the `GHCupInfo` type or the YAML representation of it. The version of the YAML represents the change increments. `ghcUpVer` is the current application version, read from `ghcup.cabal`.
+
+3. Add ChangeLog entry
+
+4. Add/fix downloads in `ghcup-<ver>.yaml` (under `data/metadata`), then verify with `ghcup-gen check -f ghcup-<ver>.yaml` and possibly (example only) `ghcup-gen check-tarballs -f ghcup-<ver>.yaml -u 'ghc-8.10.7'`. Generally, new GHC/cabal/stack/hls versions are only added to the latest yaml file. New GHCup versions are added to all (great care must be taken here to not break the parser... e.g. ARM platforms don't parse in all older formats).
+
+5. Commit and git push with tag. Wait for tests to succeed and release artifacts to build.
+
+6. Download release artifacts and upload them `downloads.haskell.org/ghcup` along with checksum files (`sha256sum --tag * > SHA256SUMS && gpg --detach-sign -u <your-email> SHA256SUMS`)
+
+7. Add ghcup release artifacts to ALL yaml files (see point 4.)
+
+8. Upload the final `ghcup-<ver>.yaml` to `webhost.haskell.org/ghcup/data/`.
+
+9. Update `bootstrap-haskell` and `bootstrap-haskell.ps1` to `webhost.haskell.org/ghcup/sh/`
+
+10. Update the ghcup symlinks at `downloads.haskell.org/ghcup`

ghcup.cabal

@@ -16,15 +16,19 @@ description:
 category:        System
 build-type:      Simple
 extra-doc-files:
-  CHANGELOG.md
-  config.yaml
-  ghcup-0.0.4.yaml
-  ghcup-0.0.5.yaml
-  ghcup-0.0.6.yaml
-  ghcup-0.0.7.yaml
-  HACKING.md
   README.md
-  RELEASING.md
+  docs/CHANGELOG.md
+  docs/HACKING.md
+  docs/RELEASING.md
+  data/config.yaml
+  data/metadata/ghcup-0.0.4.yaml
+  data/metadata/ghcup-0.0.5.yaml
+  data/metadata/ghcup-0.0.6.yaml
+  data/metadata/ghcup-0.0.7.yaml
+
+extra-source-files:
+  data/build_mk/default
+  data/build_mk/cross
 
 source-repository head
   type:     git

lib/GHCup.hs

@@ -65,9 +65,10 @@ import           Data.Time.Format.ISO8601
 import           Data.Versions
 import           GHC.IO.Exception
 import           Haskus.Utils.Variant.Excepts
+import           Language.Haskell.TH
+import           Language.Haskell.TH.Syntax     ( Quasi(qAddDependentFile) )
 import           Optics
 import           Prelude                 hiding ( abs
-                                                , readFile
                                                 , writeFile
                                                 )
 import           Safe                    hiding ( at )
@@ -1922,26 +1923,12 @@ compileGHC targetGhc ov bstrap jobs mbuildConfig patchdir aargs buildFlavour had
     pure tver
 
  where
-  defaultConf = case targetGhc of
-    Left (GHCTargetVersion (Just _) _) -> [s|
-V=0
-BUILD_MAN = NO
-BUILD_SPHINX_HTML = NO
-BUILD_SPHINX_PDF = NO
-HADDOCK_DOCS = NO
-ifneq "$(BuildFlavour)" ""
-include mk/flavours/$(BuildFlavour).mk
-endif
-Stage1Only = YES|]
-    _ -> [s|
-V=0
-BUILD_MAN = NO
-BUILD_SPHINX_HTML = NO
-BUILD_SPHINX_PDF = NO
-HADDOCK_DOCS = YES
-ifneq "$(BuildFlavour)" ""
-include mk/flavours/$(BuildFlavour).mk
-endif|]
+  defaultConf = 
+    let cross_mk = $(LitE . StringL <$> (qAddDependentFile "data/build_mk/cross" >> runIO (readFile "data/build_mk/cross")))
+        default_mk = $(LitE . StringL <$> (qAddDependentFile "data/build_mk/default" >> runIO (readFile "data/build_mk/default")))
+    in case targetGhc of
+         Left (GHCTargetVersion (Just _) _) -> cross_mk
+         _ -> default_mk
 
   compileHadrianBindist :: ( MonadReader env m
                            , HasDirs env

lib/GHCup/Utils.hs

@@ -176,7 +176,7 @@ rmMajorSymlinks tv@GHCTargetVersion{..} = do
   forM_ files $ \f -> do
     let f_xy = f <> "-" <> T.unpack v' <> exeExt
     let fullF = binDir </> f_xy
-    lift $ logDebug "rm -f #{fullF}"
+    lift $ logDebug ("rm -f " <> T.pack fullF)
     lift $ hideError doesNotExistErrorType $ rmLink fullF
 
 
@@ -791,7 +791,12 @@ applyPatches :: (MonadReader env m, HasDirs env, HasLog env, MonadIO m)
              -> FilePath   -- ^ dir to apply patches in
              -> Excepts '[PatchFailed] m ()
 applyPatches pdir ddir = do
-  patches <- (fmap . fmap) (pdir </>) $ liftIO $ listDirectory pdir
+  patches <- (fmap . fmap) (pdir </>) $ liftIO $ findFiles
+      pdir
+      (makeRegexOpts compExtended
+                     execBlank
+                     ([s|.+\.(patch|diff)$|] :: ByteString)
+      )
   forM_ (sort patches) $ \patch' -> do
     lift $ logInfo $ "Applying patch " <> T.pack patch'
     fmap (either (const Nothing) Just)
@@ -1035,7 +1040,7 @@ ensureGlobalTools = do
   let dl = downloadCached' shimDownload (Just "gs.exe") Nothing
   void $ (\(DigestError _ _) -> do
       lift $ logWarn "Digest doesn't match, redownloading gs.exe..."
-      lift $ logDebug "rm -f #{shimDownload}"
+      lift $ logDebug ("rm -f " <> T.pack (cacheDir dirs </> "gs.exe"))
       lift $ hideError doesNotExistErrorType $ recycleFile (cacheDir dirs </> "gs.exe")
       liftE @'[DigestError , DownloadFailed] $ dl
     ) `catchE` (liftE @'[DigestError , DownloadFailed] dl)

lib/GHCup/Version.hs

@@ -24,6 +24,9 @@ import qualified Data.List.NonEmpty            as NE
 import qualified Data.Text                     as T
 
 -- | This reflects the API version of the YAML.
+--
+-- Note that when updating this, CI requires that the file exsists AND the same file exists at
+-- 'https://www.haskell.org/ghcup/exp/ghcup-<ver>.yaml' with some newlines added.
 ghcupURL :: URI
 ghcupURL = [uri|https://www.haskell.org/ghcup/data/ghcup-0.0.7.yaml|]
 

test/GHCup/Types/JSONSpec.hs

@@ -13,4 +13,5 @@ import           Test.Hspec
 
 spec :: Spec
 spec = do
-  roundtripAndGoldenSpecs (Proxy @GHCupInfo)
+  roundtripAndGoldenSpecsWithSettings (defaultSettings { goldenDirectoryOption = CustomDirectoryName "test/golden" }) (Proxy @GHCupInfo)
+