Merge branch 'ghc-compile-patch'

.gitlab-ci.yml

@@ -103,7 +103,7 @@ variables:
   artifacts:
     expire_in: 2 week
     paths:
-      - golden
+      - test/golden
       - dist-newstyle/cache/
     when: on_failure
 
@@ -240,7 +240,7 @@ test:linux:bootstrap_script:
 test:windows:bootstrap_powershell_script:
   stage: test
   script:
-    - ./bootstrap-haskell.ps1 -InstallDir $CI_PROJECT_DIR -BootstrapUrl $CI_PROJECT_DIR/bootstrap-haskell -InBash
+    - ./scripts/bootstrap/bootstrap-haskell.ps1 -InstallDir $CI_PROJECT_DIR -BootstrapUrl $CI_PROJECT_DIR/bootstrap-haskell -InBash
   after_script:
     - "[Environment]::SetEnvironmentVariable('GHCUP_INSTALL_BASE_PREFIX', $null, [System.EnvironmentVariableTarget]::User)"
     - "[Environment]::SetEnvironmentVariable('GHCUP_MSYS2', $null, [System.EnvironmentVariableTarget]::User)"

.gitlab/script/ghcup_bootstrap.sh

@@ -24,7 +24,7 @@ export BOOTSTRAP_HASKELL_NONINTERACTIVE=yes
 export BOOTSTRAP_HASKELL_GHC_VERSION=$GHC_VERSION
 export BOOTSTRAP_HASKELL_CABAL_VERSION=$CABAL_VERSION
 
-./bootstrap-haskell
+./scripts/bootstrap/bootstrap-haskell
 
 [ "$(ghc --numeric-version)" = "${GHC_VERSION}" ]
 

.gitlab/script/ghcup_cross.sh

@@ -13,7 +13,7 @@ ecabal() {
 }
 
 eghcup() {
-	ghcup -v -c -s file://$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml "$@"
+	ghcup -v -c -s file://$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml "$@"
 }
 
 git describe --always

.gitlab/script/ghcup_git.sh

@@ -13,7 +13,7 @@ ecabal() {
 }
 
 eghcup() {
-	ghcup -v -c -s file://$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml "$@"
+	ghcup -v -c -s file://$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml "$@"
 }
 
 git describe --always

.gitlab/script/ghcup_version.sh

@@ -18,9 +18,9 @@ raw_eghcup() {
 
 eghcup() {
 	if [ "${OS}" = "WINDOWS" ] ; then
-		ghcup -v -c -s file:/$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml "$@"
+		ghcup -v -c -s file:/$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml "$@"
 	else
-		ghcup -v -c -s file://$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml "$@"
+		ghcup -v -c -s file://$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml "$@"
 	fi
 }
 
@@ -92,7 +92,7 @@ rm -rf "${GHCUP_DIR}"
 ### manual cli based testing
 
 
-ghcup-gen check -f ghcup-${JSON_VERSION}.yaml
+ghcup-gen check -f data/metadata/ghcup-${JSON_VERSION}.yaml
 
 eghcup --numeric-version
 
@@ -172,10 +172,10 @@ else
 fi
 
 # check that lazy loading works for 'whereis'
-cp "$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml" "$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml.bak"
+cp "$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml" "$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml.bak"
-echo '**' > "$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml"
+echo '**' > "$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml"
 eghcup whereis ghc $(ghc --numeric-version)
-mv -f "$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml.bak" "$CI_PROJECT_DIR/ghcup-${JSON_VERSION}.yaml"
+mv -f "$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml.bak" "$CI_PROJECT_DIR/data/metadata/ghcup-${JSON_VERSION}.yaml"
 
 eghcup rm $(ghc --numeric-version)
 

README.md

@@ -19,7 +19,7 @@ Similar in scope to [rustup](https://github.com/rust-lang-nursery/rustup.rs), [p
      * [Configuration](#configuration)
      * [Manpages](#manpages)
      * [Shell-completion](#shell-completion)
-     * [Cross support](#cross-support)
+     * [Compiling from sourc](#compiling-from-source)
      * [XDG support](#xdg-support)
      * [Env variables](#env-variables)
      * [Installing custom bindists](#installing-custom-bindists)
@@ -91,7 +91,7 @@ handles your haskell packages and can demand that [a specific version](https://c
 ### Configuration
 
 A configuration file can be put in `~/.ghcup/config.yaml`. The default config file
-explaining all possible configurations can be found in this repo: [config.yaml](./config.yaml).
+explaining all possible configurations can be found in this repo: [config.yaml](./data/config.yaml).
 
 Partial configuration is fine. Command line options always override the config file settings.
 
@@ -102,14 +102,26 @@ For man pages to work you need [man-db](http://man-db.nongnu.org/) as your `man`
 
 ### Shell-completion
 
-Shell completions are in `shell-completions`.
+Shell completions are in [scripts/shell-completions](./scripts/shell-completions) directory of this repository.
 
 For bash: install `shell-completions/bash`
 as e.g. `/etc/bash_completion.d/ghcup` (depending on distro)
 and make sure your bashrc sources the startup script
 (`/usr/share/bash-completion/bash_completion` on some distros).
 
-### Cross support
+### Compiling from source
+
+Compiling from source is supported for both source tarballs and arbitrary git refs. See `ghcup compile ghc --help`
+for a list of all available options.
+
+If you need to overwrite the existing `build.mk`, check the default files
+in [data/build_mk](./data/build_mk), copy them somewhere, adjust them and
+pass `--config path/to/build.mk` to `ghcup compile ghc`.
+Common `build.mk` options are explained [here](https://gitlab.haskell.org/ghc/ghc/-/wikis/building/using#build-configuration).
+
+Make sure your system meets all the [prerequisites](https://gitlab.haskell.org/ghc/ghc/-/wikis/building/preparation).
+
+#### Cross support
 
 ghcup can compile and install a cross GHC for any target. However, this
 requires that the build host has a complete cross toolchain and various

app/ghcup/Main.hs

@@ -57,6 +57,7 @@ import           Data.Void
 import           GHC.IO.Encoding
 import           Haskus.Utils.Variant.Excepts
 import           Language.Haskell.TH
+import           Language.Haskell.TH.Syntax     ( Quasi(qAddDependentFile) )
 import           Options.Applicative     hiding ( style )
 import           Options.Applicative.Help.Pretty ( text )
 import           Prelude                 hiding ( appendFile )
@@ -1045,7 +1046,7 @@ ghcCompileOpts =
           (option
             str
             (short 'p' <> long "patchdir" <> metavar "PATCH_DIR" <> help
-              "Absolute path to patch directory (applied in order, uses -p1)"
+              "Absolute path to patch directory (applies all .patch and .diff files in order using -p1)"
             )
           )
     <*> optional
@@ -1402,14 +1403,14 @@ describe_result = $( LitE . StringL <$>
                    )
 
 plan_json :: String
-plan_json = $( LitE . StringL <$>
+plan_json = $( do
-                     runIO (handleIO (\_ -> pure "") $ do
+                (fp, c) <- runIO (handleIO (\_ -> pure ("", "")) $ do
                              fp <- findPlanJson (ProjectRelativeToDir ".")
                              c <- B.readFile fp
                              (Just res) <- pure $ decodeStrict' @Value c
-                             pure $ T.unpack $ decUTF8Safe' $ encodePretty res
+                             pure (fp, T.unpack $ decUTF8Safe' $ encodePretty res))
-                     )
+                when (not . null $ fp ) $ qAddDependentFile fp
-                   )
+                pure . LitE . StringL $ c)
 
 formatConfig :: UserSettings -> String
 formatConfig settings

data/build_mk/cross

@@ -0,0 +1,9 @@
+V=0
+BUILD_MAN = NO
+BUILD_SPHINX_HTML = NO
+BUILD_SPHINX_PDF = NO
+HADDOCK_DOCS = NO
+ifneq "$(BuildFlavour)" ""
+include mk/flavours/$(BuildFlavour).mk
+endif
+Stage1Only = YES

data/build_mk/default

@@ -0,0 +1,8 @@
+V=0
+BUILD_MAN = NO
+BUILD_SPHINX_HTML = NO
+BUILD_SPHINX_PDF = NO
+HADDOCK_DOCS = YES
+ifneq "$(BuildFlavour)" ""
+include mk/flavours/$(BuildFlavour).mk
+endif

docs/RELEASING.md

@@ -6,7 +6,7 @@
 
 3. Add ChangeLog entry
 
-4. Add/fix downloads in `ghcup-<ver>.yaml`, then verify with `ghcup-gen check -f ghcup-<ver>.yaml` and possibly (example only) `ghcup-gen check-tarballs -f ghcup-<ver>.yaml -u 'ghc-8.10.7'`. Generally, new GHC/cabal/stack/hls versions are only added to the latest yaml file. New GHCup versions are added to all (great care must be taken here to not break the parser... e.g. ARM platforms don't parse in all older formats).
+4. Add/fix downloads in `ghcup-<ver>.yaml` (under `data/metadata`), then verify with `ghcup-gen check -f ghcup-<ver>.yaml` and possibly (example only) `ghcup-gen check-tarballs -f ghcup-<ver>.yaml -u 'ghc-8.10.7'`. Generally, new GHC/cabal/stack/hls versions are only added to the latest yaml file. New GHCup versions are added to all (great care must be taken here to not break the parser... e.g. ARM platforms don't parse in all older formats).
 
 5. Commit and git push with tag. Wait for tests to succeed and release artifacts to build.
 

ghcup.cabal

@@ -16,15 +16,19 @@ description:
 category:        System
 build-type:      Simple
 extra-doc-files:
-  CHANGELOG.md
-  config.yaml
-  ghcup-0.0.4.yaml
-  ghcup-0.0.5.yaml
-  ghcup-0.0.6.yaml
-  ghcup-0.0.7.yaml
-  HACKING.md
   README.md
-  RELEASING.md
+  docs/CHANGELOG.md
+  docs/HACKING.md
+  docs/RELEASING.md
+  data/config.yaml
+  data/metadata/ghcup-0.0.4.yaml
+  data/metadata/ghcup-0.0.5.yaml
+  data/metadata/ghcup-0.0.6.yaml
+  data/metadata/ghcup-0.0.7.yaml
+
+extra-source-files:
+  data/build_mk/default
+  data/build_mk/cross
 
 source-repository head
   type:     git

lib/GHCup.hs

@@ -65,9 +65,10 @@ import           Data.Time.Format.ISO8601
 import           Data.Versions
 import           GHC.IO.Exception
 import           Haskus.Utils.Variant.Excepts
+import           Language.Haskell.TH
+import           Language.Haskell.TH.Syntax     ( Quasi(qAddDependentFile) )
 import           Optics
 import           Prelude                 hiding ( abs
-                                                , readFile
                                                 , writeFile
                                                 )
 import           Safe                    hiding ( at )
@@ -1922,26 +1923,12 @@ compileGHC targetGhc ov bstrap jobs mbuildConfig patchdir aargs buildFlavour had
     pure tver
 
  where
-  defaultConf = case targetGhc of
+  defaultConf = 
-    Left (GHCTargetVersion (Just _) _) -> [s|
+    let cross_mk = $(LitE . StringL <$> (qAddDependentFile "data/build_mk/cross" >> runIO (readFile "data/build_mk/cross")))
-V=0
+        default_mk = $(LitE . StringL <$> (qAddDependentFile "data/build_mk/default" >> runIO (readFile "data/build_mk/default")))
-BUILD_MAN = NO
+    in case targetGhc of
-BUILD_SPHINX_HTML = NO
+         Left (GHCTargetVersion (Just _) _) -> cross_mk
-BUILD_SPHINX_PDF = NO
+         _ -> default_mk
-HADDOCK_DOCS = NO
-ifneq "$(BuildFlavour)" ""
-include mk/flavours/$(BuildFlavour).mk
-endif
-Stage1Only = YES|]
-    _ -> [s|
-V=0
-BUILD_MAN = NO
-BUILD_SPHINX_HTML = NO
-BUILD_SPHINX_PDF = NO
-HADDOCK_DOCS = YES
-ifneq "$(BuildFlavour)" ""
-include mk/flavours/$(BuildFlavour).mk
-endif|]
 
   compileHadrianBindist :: ( MonadReader env m
                            , HasDirs env

lib/GHCup/Utils.hs

@@ -791,7 +791,12 @@ applyPatches :: (MonadReader env m, HasDirs env, HasLog env, MonadIO m)
              -> FilePath   -- ^ dir to apply patches in
              -> Excepts '[PatchFailed] m ()
 applyPatches pdir ddir = do
-  patches <- (fmap . fmap) (pdir </>) $ liftIO $ listDirectory pdir
+  patches <- (fmap . fmap) (pdir </>) $ liftIO $ findFiles
+      pdir
+      (makeRegexOpts compExtended
+                     execBlank
+                     ([s|.+\.(patch|diff)$|] :: ByteString)
+      )
   forM_ (sort patches) $ \patch' -> do
     lift $ logInfo $ "Applying patch " <> T.pack patch'
     fmap (either (const Nothing) Just)
@@ -1035,7 +1040,7 @@ ensureGlobalTools = do
   let dl = downloadCached' shimDownload (Just "gs.exe") Nothing
   void $ (\(DigestError _ _) -> do
       lift $ logWarn "Digest doesn't match, redownloading gs.exe..."
-      lift $ logDebug ("rm -f " <> T.pack shimDownload)
+      lift $ logDebug ("rm -f " <> T.pack (cacheDir dirs </> "gs.exe"))
       lift $ hideError doesNotExistErrorType $ recycleFile (cacheDir dirs </> "gs.exe")
       liftE @'[DigestError , DownloadFailed] $ dl
     ) `catchE` (liftE @'[DigestError , DownloadFailed] dl)

test/GHCup/Types/JSONSpec.hs

@@ -13,4 +13,5 @@ import           Test.Hspec
 
 spec :: Spec
 spec = do
-  roundtripAndGoldenSpecs (Proxy @GHCupInfo)
+  roundtripAndGoldenSpecsWithSettings (defaultSettings { goldenDirectoryOption = CustomDirectoryName "test/golden" }) (Proxy @GHCupInfo)
+