From 63f22b28d78645d6dcf1147d48f66c9d0331d19b Mon Sep 17 00:00:00 2001 From: Julian Ospald Date: Sat, 9 Jul 2022 20:27:55 +0200 Subject: [PATCH 1/7] Allow to build HLS from hackage --- app/ghcup/GHCup/OptParse/Compile.hs | 45 +++++++++++--- app/ghcup/Main.hs | 4 +- lib/GHCup/HLS.hs | 95 ++++++++++++++++++++++------- lib/GHCup/Types.hs | 6 +- 4 files changed, 116 insertions(+), 34 deletions(-) diff --git a/app/ghcup/GHCup/OptParse/Compile.hs b/app/ghcup/GHCup/OptParse/Compile.hs index 42b6317..311bde0 100644 --- a/app/ghcup/GHCup/OptParse/Compile.hs +++ b/app/ghcup/GHCup/OptParse/Compile.hs @@ -78,8 +78,9 @@ data GHCCompileOptions = GHCCompileOptions , isolateDir :: Maybe FilePath } + data HLSCompileOptions = HLSCompileOptions - { targetHLS :: Either Version GitBranch + { targetHLS :: HLSVer , jobs :: Maybe Int , setCompile :: Bool , ovewrwiteVer :: Either Bool Version @@ -149,10 +150,12 @@ Examples: These need to be available in PATH prior to compilation. Examples: - # compile 1.7.0.0 for ghc 8.10.5 and 8.10.7, passing '--allow-newer' to cabal + # compile 1.7.0.0 from hackage for 8.10.7 + ghcup compile hls --hackage-version 1.7.0.0 --ghc 8.10.7 + # compile 1.7.0.0 from official source dist for ghc 8.10.5 and 8.10.7, passing '--allow-newer' to cabal ghcup compile hls -v 1.7.0.0 -j 12 --ghc 8.10.5 --ghc 8.10.7 -- --allow-newer - # compile from master for ghc 9.2.3 and use 'git describe' to name the binary - ghcup compile hls -g master --git-describe-version --ghc 9.2.3 + # compile from master for ghc 9.2.3 using 'git describe' to name the binary and ignore the pinned index state + ghcup compile hls -g master --git-describe-version --ghc 9.2.3 -- --index-state=@(date '+%s') # compile a specific commit for ghc 9.2.3 and set a specifc version for the binary name ghcup compile hls -g a32db0b -o 1.7.0.0-p1 --ghc 9.2.3|] @@ -270,7 +273,7 @@ ghcCompileOpts = hlsCompileOpts :: Parser HLSCompileOptions hlsCompileOpts = HLSCompileOptions - <$> ((Left <$> option + <$> ((SourceDist <$> option (eitherReader (first (const "Not a valid version") . version . T.pack) ) @@ -278,8 +281,9 @@ hlsCompileOpts = "The tool version to compile" <> (completer $ versionCompleter Nothing HLS) ) - ) <|> - (Right <$> (GitBranch <$> option + ) + <|> + (GitDist <$> (GitBranch <$> option str (short 'g' <> long "git-ref" <> metavar "GIT_REFERENCE" <> help "The git commit/branch/ref to build from (accepts anything 'git checkout' accepts)" @@ -287,7 +291,28 @@ hlsCompileOpts = optional (option str (short 'r' <> long "repository" <> metavar "GIT_REPOSITORY" <> help "The git repository to build from (defaults to HLS upstream)" <> completer (gitFileUri ["https://github.com/haskell/haskell-language-server.git"]) )) - ))) + )) + <|> + (HackageDist <$> (option + (eitherReader + (first (const "Not a valid version") . version . T.pack) + ) + (long "hackage-version" <> metavar "HACKAGE_VERSION" <> help + "The hackage version to compile" + <> (completer $ versionCompleter Nothing HLS) + ) + )) + <|> + ( + RemoteDist <$> (option + (eitherReader uriParser) + (long "remote-source-dist" <> metavar "URI" <> help + "URI (https/http/file) to a HLS source distribution" + <> completer fileUri + ) + ) + ) + ) <*> optional (option (eitherReader (readEither @Int)) @@ -468,7 +493,7 @@ compile compileCommand settings Dirs{..} runAppState runLogger = do (CompileHLS HLSCompileOptions { .. }) -> do runCompileHLS runAppState (do case targetHLS of - Left targetVer -> do + SourceDist targetVer -> do GHCupInfo { _ghcupDownloads = dls } <- lift getGHCupInfo let vi = getVersionInfo targetVer HLS dls forM_ (_viPreCompile =<< vi) $ \msg -> do @@ -476,7 +501,7 @@ compile compileCommand settings Dirs{..} runAppState runLogger = do lift $ logInfo "...waiting for 5 seconds, you can still abort..." liftIO $ threadDelay 5000000 -- for compilation, give the user a sec to intervene - Right _ -> pure () + _ -> pure () ghcs <- liftE $ forM targetGHCs (\ghc -> fmap (_tvVersion . fst) . fromVersion (Just ghc) $ GHC) targetVer <- liftE $ compileHLS targetHLS diff --git a/app/ghcup/Main.hs b/app/ghcup/Main.hs index 290aadb..7d55e57 100644 --- a/app/ghcup/Main.hs +++ b/app/ghcup/Main.hs @@ -342,7 +342,9 @@ Report bugs at |] (GHC, ver) = cmp' GHC (Just $ ToolVersion (mkTVer tver)) ver alreadyInstalling (Compile (CompileHLS HLSCompileOptions{ ovewrwiteVer = Right over })) (HLS, ver) = cmp' HLS (Just $ ToolVersion (mkTVer over)) ver - alreadyInstalling (Compile (CompileHLS HLSCompileOptions{ targetHLS = Left tver })) + alreadyInstalling (Compile (CompileHLS HLSCompileOptions{ targetHLS = SourceDist tver })) + (HLS, ver) = cmp' HLS (Just $ ToolVersion (mkTVer tver)) ver + alreadyInstalling (Compile (CompileHLS HLSCompileOptions{ targetHLS = HackageDist tver })) (HLS, ver) = cmp' HLS (Just $ ToolVersion (mkTVer tver)) ver alreadyInstalling (Upgrade _ _ _) (GHCup, _) = pure True alreadyInstalling _ _ = pure False diff --git a/lib/GHCup/HLS.hs b/lib/GHCup/HLS.hs index 9b99069..1f5779e 100644 --- a/lib/GHCup/HLS.hs +++ b/lib/GHCup/HLS.hs @@ -324,7 +324,7 @@ compileHLS :: ( MonadMask m , MonadUnliftIO m , MonadFail m ) - => Either Version GitBranch + => HLSVer -> [Version] -> Maybe Int -> Either Bool Version @@ -349,9 +349,9 @@ compileHLS targetHLS ghcs jobs ov installDir cabalProject cabalProjectLocal patc Dirs { .. } <- lift getDirs - (workdir, tver, git_describe) <- case targetHLS of + (workdir, tmpUnpack, tver, git_describe) <- case targetHLS of -- unpack from version tarball - Left tver -> do + SourceDist tver -> do lift $ logDebug $ "Requested to compile: " <> prettyVer tver -- download source tarball @@ -369,10 +369,47 @@ compileHLS targetHLS ghcs jobs ov installDir cabalProject cabalProjectLocal patc (liftE . intoSubdir tmpUnpack) (view dlSubdir dlInfo) - pure (workdir, tver, Nothing) + pure (workdir, tmpUnpack, tver, Nothing) + + HackageDist tver -> do + lift $ logDebug $ "Requested to compile (from hackage): " <> prettyVer tver + + -- download source tarball + tmpUnpack <- lift mkGhcupTmpDir + let hls = "haskell-language-server-" <> T.unpack (prettyVer tver) + reThrowAll @_ @'[ProcessError] DownloadFailed $ do + -- unpack + lEM $ exec "cabal" ["unpack", hls] (Just $ fromGHCupPath tmpUnpack) Nothing + + let workdir = appendGHCupPath tmpUnpack hls + + pure (workdir, tmpUnpack, tver, Nothing) + + RemoteDist uri -> do + lift $ logDebug $ "Requested to compile (from uri): " <> T.pack (show uri) + + -- download source tarball + tmpDownload <- lift withGHCupTmpDir + tmpUnpack <- lift mkGhcupTmpDir + tar <- liftE $ download uri Nothing Nothing (fromGHCupPath tmpDownload) Nothing False + (cf, tver) <- liftE $ cleanUpOnError tmpUnpack $ do + unpackToDir (fromGHCupPath tmpUnpack) tar + let regex = [s|haskell-language-server\.cabal$|] :: B.ByteString + [cabalFile] <- liftIO $ handleIO (\_ -> pure []) $ findFilesDeep + tmpUnpack + (makeRegexOpts compExtended + execBlank + regex + ) + tver <- getCabalVersion (fromGHCupPath tmpUnpack cabalFile) + pure (cabalFile, tver) + + let workdir = appendGHCupPath tmpUnpack (takeDirectory cf) + + pure (workdir, tmpUnpack, tver, Nothing) -- clone from git - Right GitBranch{..} -> do + GitDist GitBranch{..} -> do tmpUnpack <- lift mkGhcupTmpDir let git args = execLogged "git" ("--no-pager":args) (Just $ fromGHCupPath tmpUnpack) "git" Nothing reThrowAll @_ @'[ProcessError] DownloadFailed $ do @@ -409,15 +446,7 @@ compileHLS targetHLS ghcs jobs ov installDir cabalProject cabalProjectLocal patc then pure Nothing else fmap Just $ gitOut ["describe", "--tags"] (fromGHCupPath tmpUnpack) chash <- gitOut ["rev-parse", "HEAD" ] (fromGHCupPath tmpUnpack) - (Just gpd) <- parseGenericPackageDescriptionMaybe - <$> liftIO (B.readFile (fromGHCupPath tmpUnpack "haskell-language-server.cabal")) - let tver = (\c -> Version Nothing c [] Nothing) - . NE.fromList . fmap (NE.fromList . (:[]) . digits . fromIntegral) - . versionNumbers - . pkgVersion - . package - . packageDescription - $ gpd + tver <- getCabalVersion (fromGHCupPath tmpUnpack "haskell-language-server.cabal") liftE $ catchWarn $ lEM @_ @'[ProcessError] $ darwinNotarization _rPlatform (fromGHCupPath tmpUnpack) lift $ logInfo $ "Examining git ref " <> T.pack ref <> "\n " <> @@ -425,7 +454,7 @@ compileHLS targetHLS ghcs jobs ov installDir cabalProject cabalProjectLocal patc (if not shallow_clone then "\n " <> "'git describe' output: " <> fromJust git_describe else mempty) <> (if isCommitHash ref then mempty else "\n " <> "commit hash: " <> chash) - pure (tmpUnpack, tver, git_describe) + pure (tmpUnpack, tmpUnpack, tver, git_describe) -- the version that's installed may differ from the -- compiled version, so the user can overwrite it @@ -441,7 +470,7 @@ compileHLS targetHLS ghcs jobs ov installDir cabalProject cabalProjectLocal patc Right v -> pure v liftE $ runBuildAction - workdir + tmpUnpack (reThrowAll @_ @'[GPGError, DownloadFailed, DigestError, PatchFailed, ProcessError, FileAlreadyExistsError, CopyError] @'[BuildFailed] (BuildFailed $ fromGHCupPath workdir) $ do let tmpInstallDir = fromGHCupPath workdir "out" liftIO $ createDirRecursive' tmpInstallDir @@ -457,14 +486,22 @@ compileHLS targetHLS ghcs jobs ov installDir cabalProject cabalProjectLocal patc pure "cabal.project" | otherwise -> pure (takeFileName cp) Just (Right uri) -> do - tmpUnpack <- lift withGHCupTmpDir - cp <- liftE $ download uri Nothing Nothing (fromGHCupPath tmpUnpack) (Just "cabal.project") False + tmpUnpack' <- lift withGHCupTmpDir + cp <- liftE $ download uri Nothing Nothing (fromGHCupPath tmpUnpack') (Just "cabal.project") False copyFileE cp (fromGHCupPath workdir "cabal.project") False pure "cabal.project" - Nothing -> pure "cabal.project" + Nothing + | HackageDist _ <- targetHLS -> do + liftIO $ B.writeFile (fromGHCupPath workdir "cabal.project") "packages: ./" + pure "cabal.project" + | RemoteDist _ <- targetHLS -> do + let cabalFile = fromGHCupPath workdir "cabal.project" + liftIO $ whenM (not <$> doesFileExist cabalFile) $ B.writeFile cabalFile "packages: ./" + pure "cabal.project" + | otherwise -> pure "cabal.project" forM_ cabalProjectLocal $ \uri -> do - tmpUnpack <- lift withGHCupTmpDir - cpl <- liftE $ download uri Nothing Nothing (fromGHCupPath tmpUnpack) (Just (cp <.> "local")) False + tmpUnpack' <- lift withGHCupTmpDir + cpl <- liftE $ download uri Nothing Nothing (fromGHCupPath tmpUnpack') (Just (cp <.> "local")) False copyFileE cpl (fromGHCupPath workdir cp <.> "local") False artifacts <- forM (sort ghcs) $ \ghc -> do let ghcInstallDir = tmpInstallDir T.unpack (prettyVer ghc) @@ -658,3 +695,19 @@ rmHLSVer ver = do case headMay . reverse . sort $ hlsVers of Just latestver -> liftE $ setHLS latestver SetHLSOnly Nothing Nothing -> pure () + + +getCabalVersion :: (MonadIO m, MonadFail m) => FilePath -> m Version +getCabalVersion fp = do + contents <- liftIO $ B.readFile fp + gpd <- case parseGenericPackageDescriptionMaybe contents of + Nothing -> fail $ "could not parse cabal file: " <> fp + Just r -> pure r + let tver = (\c -> Version Nothing c [] Nothing) + . NE.fromList . fmap (NE.fromList . (:[]) . digits . fromIntegral) + . versionNumbers + . pkgVersion + . package + . packageDescription + $ gpd + pure tver diff --git a/lib/GHCup/Types.hs b/lib/GHCup/Types.hs index 63761d2..d59037d 100644 --- a/lib/GHCup/Types.hs +++ b/lib/GHCup/Types.hs @@ -657,7 +657,9 @@ isSafeDir (GHCupBinDir _) = False - - +data HLSVer = SourceDist Version + | GitDist GitBranch + | HackageDist Version + | RemoteDist URI From 9fb28896965922a443fd6c366d0a0266a8f04a00 Mon Sep 17 00:00:00 2001 From: Julian Ospald Date: Sat, 9 Jul 2022 23:12:00 +0200 Subject: [PATCH 2/7] Allow to build from arbitrary GHC source dists --- app/ghcup/GHCup/OptParse/Compile.hs | 42 ++++++++++++------ app/ghcup/Main.hs | 8 ++-- lib/GHCup.hs | 4 +- lib/GHCup/GHC.hs | 66 +++++++++++++++++++++++------ lib/GHCup/HLS.hs | 10 ++++- lib/GHCup/Types.hs | 9 ---- lib/GHCup/Utils.hs | 3 +- 7 files changed, 99 insertions(+), 43 deletions(-) diff --git a/app/ghcup/GHCup/OptParse/Compile.hs b/app/ghcup/GHCup/OptParse/Compile.hs index 311bde0..3ec5d32 100644 --- a/app/ghcup/GHCup/OptParse/Compile.hs +++ b/app/ghcup/GHCup/OptParse/Compile.hs @@ -12,6 +12,8 @@ module GHCup.OptParse.Compile where import GHCup +import qualified GHCup.GHC as GHC +import qualified GHCup.HLS as HLS import GHCup.Errors import GHCup.Types import GHCup.Types.Optics @@ -64,7 +66,7 @@ data CompileCommand = CompileGHC GHCCompileOptions data GHCCompileOptions = GHCCompileOptions - { targetGhc :: Either Version GitBranch + { targetGhc :: GHC.GHCVer Version , bootstrapGhc :: Either Version FilePath , jobs :: Maybe Int , buildConfig :: Maybe FilePath @@ -80,7 +82,7 @@ data GHCCompileOptions = GHCCompileOptions data HLSCompileOptions = HLSCompileOptions - { targetHLS :: HLSVer + { targetHLS :: HLS.HLSVer , jobs :: Maybe Int , setCompile :: Bool , ovewrwiteVer :: Either Bool Version @@ -163,7 +165,7 @@ Examples: ghcCompileOpts :: Parser GHCCompileOptions ghcCompileOpts = GHCCompileOptions - <$> ((Left <$> option + <$> ((GHC.SourceDist <$> option (eitherReader (first (const "Not a valid version") . version . T.pack) ) @@ -172,7 +174,7 @@ ghcCompileOpts = <> (completer $ versionCompleter Nothing GHC) ) ) <|> - (Right <$> (GitBranch <$> option + (GHC.GitDist <$> (GitBranch <$> option str (short 'g' <> long "git-ref" <> metavar "GIT_REFERENCE" <> help "The git commit/branch/ref to build from" @@ -181,7 +183,18 @@ ghcCompileOpts = short 'r' <> long "repository" <> metavar "GIT_REPOSITORY" <> help "The git repository to build from (defaults to GHC upstream)" <> completer (gitFileUri ["https://gitlab.haskell.org/ghc/ghc.git"]) )) - ))) + )) + <|> + ( + GHC.RemoteDist <$> (option + (eitherReader uriParser) + (long "remote-source-dist" <> metavar "URI" <> help + "URI (https/http/file) to a GHC source distribution" + <> completer fileUri + ) + ) + ) + ) <*> option (eitherReader (\x -> @@ -273,7 +286,7 @@ ghcCompileOpts = hlsCompileOpts :: Parser HLSCompileOptions hlsCompileOpts = HLSCompileOptions - <$> ((SourceDist <$> option + <$> ((HLS.SourceDist <$> option (eitherReader (first (const "Not a valid version") . version . T.pack) ) @@ -283,7 +296,7 @@ hlsCompileOpts = ) ) <|> - (GitDist <$> (GitBranch <$> option + (HLS.GitDist <$> (GitBranch <$> option str (short 'g' <> long "git-ref" <> metavar "GIT_REFERENCE" <> help "The git commit/branch/ref to build from (accepts anything 'git checkout' accepts)" @@ -293,7 +306,7 @@ hlsCompileOpts = )) )) <|> - (HackageDist <$> (option + (HLS.HackageDist <$> (option (eitherReader (first (const "Not a valid version") . version . T.pack) ) @@ -304,7 +317,7 @@ hlsCompileOpts = )) <|> ( - RemoteDist <$> (option + HLS.RemoteDist <$> (option (eitherReader uriParser) (long "remote-source-dist" <> metavar "URI" <> help "URI (https/http/file) to a HLS source distribution" @@ -493,7 +506,7 @@ compile compileCommand settings Dirs{..} runAppState runLogger = do (CompileHLS HLSCompileOptions { .. }) -> do runCompileHLS runAppState (do case targetHLS of - SourceDist targetVer -> do + HLS.SourceDist targetVer -> do GHCupInfo { _ghcupDownloads = dls } <- lift getGHCupInfo let vi = getVersionInfo targetVer HLS dls forM_ (_viPreCompile =<< vi) $ \msg -> do @@ -544,7 +557,7 @@ compile compileCommand settings Dirs{..} runAppState runLogger = do (CompileGHC GHCCompileOptions {..}) -> runCompileGHC runAppState (do case targetGhc of - Left targetVer -> do + GHC.SourceDist targetVer -> do GHCupInfo { _ghcupDownloads = dls } <- lift getGHCupInfo let vi = getVersionInfo targetVer GHC dls forM_ (_viPreCompile =<< vi) $ \msg -> do @@ -552,9 +565,12 @@ compile compileCommand settings Dirs{..} runAppState runLogger = do lift $ logInfo "...waiting for 5 seconds, you can still abort..." liftIO $ threadDelay 5000000 -- for compilation, give the user a sec to intervene - Right _ -> pure () + _ -> pure () targetVer <- liftE $ compileGHC - (first (GHCTargetVersion crossTarget) targetGhc) + ((\case + GHC.SourceDist v -> GHC.SourceDist $ GHCTargetVersion crossTarget v + GHC.GitDist g -> GHC.GitDist g + GHC.RemoteDist r -> GHC.RemoteDist r) targetGhc) ovewrwiteVer bootstrapGhc jobs diff --git a/app/ghcup/Main.hs b/app/ghcup/Main.hs index 7d55e57..5e48d25 100644 --- a/app/ghcup/Main.hs +++ b/app/ghcup/Main.hs @@ -14,6 +14,8 @@ module Main where import BrickMain ( brickMain ) #endif +import qualified GHCup.GHC as GHC +import qualified GHCup.HLS as HLS import GHCup.OptParse import GHCup.Download @@ -338,13 +340,13 @@ Report bugs at |] alreadyInstalling (Install (Left (InstallStack InstallOptions{..}))) (Stack, ver) = cmp' Stack instVer ver alreadyInstalling (Compile (CompileGHC GHCCompileOptions{ ovewrwiteVer = Just over })) (GHC, ver) = cmp' GHC (Just $ ToolVersion (mkTVer over)) ver - alreadyInstalling (Compile (CompileGHC GHCCompileOptions{ targetGhc = Left tver })) + alreadyInstalling (Compile (CompileGHC GHCCompileOptions{ targetGhc = GHC.SourceDist tver })) (GHC, ver) = cmp' GHC (Just $ ToolVersion (mkTVer tver)) ver alreadyInstalling (Compile (CompileHLS HLSCompileOptions{ ovewrwiteVer = Right over })) (HLS, ver) = cmp' HLS (Just $ ToolVersion (mkTVer over)) ver - alreadyInstalling (Compile (CompileHLS HLSCompileOptions{ targetHLS = SourceDist tver })) + alreadyInstalling (Compile (CompileHLS HLSCompileOptions{ targetHLS = HLS.SourceDist tver })) (HLS, ver) = cmp' HLS (Just $ ToolVersion (mkTVer tver)) ver - alreadyInstalling (Compile (CompileHLS HLSCompileOptions{ targetHLS = HackageDist tver })) + alreadyInstalling (Compile (CompileHLS HLSCompileOptions{ targetHLS = HLS.HackageDist tver })) (HLS, ver) = cmp' HLS (Just $ ToolVersion (mkTVer tver)) ver alreadyInstalling (Upgrade _ _ _) (GHCup, _) = pure True alreadyInstalling _ _ = pure False diff --git a/lib/GHCup.hs b/lib/GHCup.hs index 1d5c295..785b05a 100644 --- a/lib/GHCup.hs +++ b/lib/GHCup.hs @@ -33,8 +33,8 @@ module GHCup ( import GHCup.Cabal -import GHCup.GHC -import GHCup.HLS +import GHCup.GHC hiding ( GHCVer(..) ) +import GHCup.HLS hiding ( HLSVer(..) ) import GHCup.Stack import GHCup.List import GHCup.Download diff --git a/lib/GHCup/GHC.hs b/lib/GHCup/GHC.hs index 1bfe494..aa1aff5 100644 --- a/lib/GHCup/GHC.hs +++ b/lib/GHCup/GHC.hs @@ -80,6 +80,12 @@ import qualified Data.Text.Encoding as E import qualified Text.Megaparsec as MP +data GHCVer v = SourceDist v + | GitDist GitBranch + | RemoteDist URI + + + --------------------- --[ Tool fetching ]-- --------------------- @@ -607,7 +613,7 @@ compileGHC :: ( MonadMask m , MonadUnliftIO m , MonadFail m ) - => Either GHCTargetVersion GitBranch -- ^ version to install + => GHCVer GHCTargetVersion -> Maybe Version -- ^ overwrite version -> Either Version FilePath -- ^ version to bootstrap with -> Maybe Int -- ^ jobs @@ -650,7 +656,7 @@ compileGHC targetGhc ov bstrap jobs mbuildConfig patches aargs buildFlavour hadr (workdir, tmpUnpack, tver) <- case targetGhc of -- unpack from version tarball - Left tver -> do + SourceDist tver -> do lift $ logDebug $ "Requested to compile: " <> tVerToText tver <> " with " <> either prettyVer T.pack bstrap -- download source tarball @@ -671,8 +677,31 @@ compileGHC targetGhc ov bstrap jobs mbuildConfig patches aargs buildFlavour hadr pure (workdir, tmpUnpack, tver) + RemoteDist uri -> do + lift $ logDebug $ "Requested to compile (from uri): " <> T.pack (show uri) + + -- download source tarball + tmpDownload <- lift withGHCupTmpDir + tmpUnpack <- lift mkGhcupTmpDir + tar <- liftE $ download uri Nothing Nothing (fromGHCupPath tmpDownload) Nothing False + (bf, tver) <- liftE $ cleanUpOnError @'[UnknownArchive, ArchiveResult, ProcessError] tmpUnpack $ do + liftE $ unpackToDir (fromGHCupPath tmpUnpack) tar + let regex = [s|^(.*/)*boot$|] :: B.ByteString + [bootFile] <- liftIO $ findFilesDeep + tmpUnpack + (makeRegexOpts compExtended + execBlank + regex + ) + tver <- liftE $ getGHCVer (appendGHCupPath tmpUnpack (takeDirectory bootFile)) + pure (bootFile, tver) + + let workdir = appendGHCupPath tmpUnpack (takeDirectory bf) + + pure (workdir, tmpUnpack, mkTVer tver) + -- clone from git - Right GitBranch{..} -> do + GitDist GitBranch{..} -> do tmpUnpack <- lift mkGhcupTmpDir let git args = execLogged "git" ("--no-pager":args) (Just $ fromGHCupPath tmpUnpack) "git" Nothing tver <- reThrowAll @_ @'[PatchFailed, ProcessError, NotFoundInPATH, DigestError, DownloadFailed, GPGError] DownloadFailed $ do @@ -715,14 +744,7 @@ compileGHC targetGhc ov bstrap jobs mbuildConfig patches aargs buildFlavour hadr liftE $ applyAnyPatch patches (fromGHCupPath tmpUnpack) -- bootstrap - lEM $ execWithGhcEnv "python3" ["./boot"] (Just $ fromGHCupPath tmpUnpack) "ghc-bootstrap" - lEM $ execWithGhcEnv "sh" ["./configure"] (Just $ fromGHCupPath tmpUnpack) "ghc-bootstrap" - CapturedProcess {..} <- lift $ makeOut - ["show!", "--quiet", "VALUE=ProjectVersion" ] (Just $ fromGHCupPath tmpUnpack) - tver <- case _exitCode of - ExitSuccess -> throwEither . MP.parse ghcProjectVersion "" . T.pack . stripNewlineEnd . T.unpack . decUTF8Safe' $ _stdOut - ExitFailure c -> fail ("Could not figure out GHC project version. Exit code was: " <> show c <> ". Error was: " <> T.unpack (decUTF8Safe' _stdErr)) - + tver <- liftE $ getGHCVer tmpUnpack liftE $ catchWarn $ lEM @_ @'[ProcessError] $ darwinNotarization _rPlatform (fromGHCupPath tmpUnpack) lift $ logInfo $ "Examining git ref " <> T.pack ref <> "\n " <> "GHC version (from Makefile): " <> prettyVer tver <> @@ -795,11 +817,29 @@ compileGHC targetGhc ov bstrap jobs mbuildConfig patches aargs buildFlavour hadr pure installVer where + getGHCVer :: ( MonadReader env m + , HasSettings env + , HasDirs env + , HasLog env + , MonadIO m + , MonadThrow m + ) + => GHCupPath + -> Excepts '[ProcessError] m Version + getGHCVer tmpUnpack = do + lEM $ execWithGhcEnv "python3" ["./boot"] (Just $ fromGHCupPath tmpUnpack) "ghc-bootstrap" + lEM $ execWithGhcEnv "sh" ["./configure"] (Just $ fromGHCupPath tmpUnpack) "ghc-bootstrap" + CapturedProcess {..} <- lift $ makeOut + ["show!", "--quiet", "VALUE=ProjectVersion" ] (Just $ fromGHCupPath tmpUnpack) + case _exitCode of + ExitSuccess -> throwEither . MP.parse ghcProjectVersion "" . T.pack . stripNewlineEnd . T.unpack . decUTF8Safe' $ _stdOut + ExitFailure c -> throwE $ NonZeroExit c "make" ["show!", "--quiet", "VALUE=ProjectVersion" ] + defaultConf = let cross_mk = $(LitE . StringL <$> (qAddDependentFile "data/build_mk/cross" >> runIO (readFile "data/build_mk/cross"))) default_mk = $(LitE . StringL <$> (qAddDependentFile "data/build_mk/default" >> runIO (readFile "data/build_mk/default"))) in case targetGhc of - Left (GHCTargetVersion (Just _) _) -> cross_mk + SourceDist (GHCTargetVersion (Just _) _) -> cross_mk _ -> default_mk compileHadrianBindist :: ( MonadReader env m @@ -976,7 +1016,7 @@ compileGHC targetGhc ov bstrap jobs mbuildConfig patches aargs buildFlavour hadr -- for cross, we need Stage1Only case targetGhc of - Left (GHCTargetVersion (Just _) _) -> when ("Stage1Only = YES" `notElem` lines') $ throwE + SourceDist (GHCTargetVersion (Just _) _) -> when ("Stage1Only = YES" `notElem` lines') $ throwE (InvalidBuildConfig [s|Cross compiling needs to be a Stage1 build, add "Stage1Only = YES" to your config!|] ) diff --git a/lib/GHCup/HLS.hs b/lib/GHCup/HLS.hs index 1f5779e..8aeb218 100644 --- a/lib/GHCup/HLS.hs +++ b/lib/GHCup/HLS.hs @@ -71,6 +71,12 @@ import qualified Text.Megaparsec as MP import Text.PrettyPrint.HughesPJClass (prettyShow) +data HLSVer = SourceDist Version + | GitDist GitBranch + | HackageDist Version + | RemoteDist URI + + -------------------- --[ Installation ]-- @@ -394,8 +400,8 @@ compileHLS targetHLS ghcs jobs ov installDir cabalProject cabalProjectLocal patc tar <- liftE $ download uri Nothing Nothing (fromGHCupPath tmpDownload) Nothing False (cf, tver) <- liftE $ cleanUpOnError tmpUnpack $ do unpackToDir (fromGHCupPath tmpUnpack) tar - let regex = [s|haskell-language-server\.cabal$|] :: B.ByteString - [cabalFile] <- liftIO $ handleIO (\_ -> pure []) $ findFilesDeep + let regex = [s|^(.*/)*haskell-language-server\.cabal$|] :: B.ByteString + [cabalFile] <- liftIO $ findFilesDeep tmpUnpack (makeRegexOpts compExtended execBlank diff --git a/lib/GHCup/Types.hs b/lib/GHCup/Types.hs index d59037d..39af732 100644 --- a/lib/GHCup/Types.hs +++ b/lib/GHCup/Types.hs @@ -654,12 +654,3 @@ isSafeDir (IsolateDirResolved _) = False isSafeDir (GHCupDir _) = True isSafeDir (GHCupBinDir _) = False - - - -data HLSVer = SourceDist Version - | GitDist GitBranch - | HackageDist Version - | RemoteDist URI - - diff --git a/lib/GHCup/Utils.hs b/lib/GHCup/Utils.hs index 07817d6..687fb0d 100644 --- a/lib/GHCup/Utils.hs +++ b/lib/GHCup/Utils.hs @@ -1097,7 +1097,8 @@ runBuildAction bdir action = do -- | Clean up the given directory if the action fails, -- depending on the Settings. -cleanUpOnError :: ( MonadReader env m +cleanUpOnError :: forall e m a env . + ( MonadReader env m , HasDirs env , HasSettings env , MonadIO m From 974112016ea6d25d0efbc485fcedda635a446659 Mon Sep 17 00:00:00 2001 From: Julian Ospald Date: Sat, 9 Jul 2022 23:50:20 +0200 Subject: [PATCH 3/7] Allow to run 'cabal update' automatically before the HLS build --- app/ghcup/GHCup/OptParse/Compile.hs | 7 +++++-- lib/GHCup/HLS.hs | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/app/ghcup/GHCup/OptParse/Compile.hs b/app/ghcup/GHCup/OptParse/Compile.hs index 3ec5d32..efcefad 100644 --- a/app/ghcup/GHCup/OptParse/Compile.hs +++ b/app/ghcup/GHCup/OptParse/Compile.hs @@ -85,6 +85,7 @@ data HLSCompileOptions = HLSCompileOptions { targetHLS :: HLS.HLSVer , jobs :: Maybe Int , setCompile :: Bool + , updateCabal :: Bool , ovewrwiteVer :: Either Bool Version , isolateDir :: Maybe FilePath , cabalProject :: Maybe (Either FilePath URI) @@ -152,8 +153,8 @@ Examples: These need to be available in PATH prior to compilation. Examples: - # compile 1.7.0.0 from hackage for 8.10.7 - ghcup compile hls --hackage-version 1.7.0.0 --ghc 8.10.7 + # compile 1.7.0.0 from hackage for 8.10.7, running 'cabal update' before the build + ghcup compile hls --hackage-version 1.7.0.0 --ghc 8.10.7 --cabal-update # compile 1.7.0.0 from official source dist for ghc 8.10.5 and 8.10.7, passing '--allow-newer' to cabal ghcup compile hls -v 1.7.0.0 -j 12 --ghc 8.10.5 --ghc 8.10.7 -- --allow-newer # compile from master for ghc 9.2.3 using 'git describe' to name the binary and ignore the pinned index state @@ -335,6 +336,7 @@ hlsCompileOpts = ) ) <*> fmap (fromMaybe True) (invertableSwitch "set" Nothing True (help "Don't set as active version after install")) + <*> switch (long "cabal-update" <> help "Run 'cabal update' before the build") <*> ( (Right <$> option @@ -524,6 +526,7 @@ compile compileCommand settings Dirs{..} runAppState runLogger = do (maybe GHCupInternal IsolateDir isolateDir) cabalProject cabalProjectLocal + updateCabal patches cabalArgs GHCupInfo { _ghcupDownloads = dls } <- lift getGHCupInfo diff --git a/lib/GHCup/HLS.hs b/lib/GHCup/HLS.hs index 8aeb218..2c08864 100644 --- a/lib/GHCup/HLS.hs +++ b/lib/GHCup/HLS.hs @@ -337,6 +337,7 @@ compileHLS :: ( MonadMask m -> InstallDir -> Maybe (Either FilePath URI) -> Maybe URI + -> Bool -> Maybe (Either FilePath [URI]) -- ^ patches -> [Text] -- ^ additional args to cabal install -> Excepts '[ NoDownload @@ -349,11 +350,14 @@ compileHLS :: ( MonadMask m , BuildFailed , NotInstalled ] m Version -compileHLS targetHLS ghcs jobs ov installDir cabalProject cabalProjectLocal patches cabalArgs = do +compileHLS targetHLS ghcs jobs ov installDir cabalProject cabalProjectLocal updateCabal patches cabalArgs = do PlatformRequest { .. } <- lift getPlatformReq GHCupInfo { _ghcupDownloads = dls } <- lift getGHCupInfo Dirs { .. } <- lift getDirs + when updateCabal $ reThrowAll @_ @'[ProcessError] DownloadFailed $ do + lift $ logInfo "Updating cabal DB" + lEM $ exec "cabal" ["update"] (Just $ fromGHCupPath tmpDir) Nothing (workdir, tmpUnpack, tver, git_describe) <- case targetHLS of -- unpack from version tarball From 99a51d67a163e06f203934b9c67ed14caf163c9a Mon Sep 17 00:00:00 2001 From: Julian Ospald Date: Sun, 10 Jul 2022 21:58:03 +0200 Subject: [PATCH 4/7] Make compiling from hackage the default --- app/ghcup/GHCup/OptParse/Compile.hs | 14 ++++------ docs/guide.md | 43 ++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/app/ghcup/GHCup/OptParse/Compile.hs b/app/ghcup/GHCup/OptParse/Compile.hs index efcefad..82df4d1 100644 --- a/app/ghcup/GHCup/OptParse/Compile.hs +++ b/app/ghcup/GHCup/OptParse/Compile.hs @@ -154,9 +154,7 @@ Examples: Examples: # compile 1.7.0.0 from hackage for 8.10.7, running 'cabal update' before the build - ghcup compile hls --hackage-version 1.7.0.0 --ghc 8.10.7 --cabal-update - # compile 1.7.0.0 from official source dist for ghc 8.10.5 and 8.10.7, passing '--allow-newer' to cabal - ghcup compile hls -v 1.7.0.0 -j 12 --ghc 8.10.5 --ghc 8.10.7 -- --allow-newer + ghcup compile hls --version 1.7.0.0 --ghc 8.10.7 --cabal-update # compile from master for ghc 9.2.3 using 'git describe' to name the binary and ignore the pinned index state ghcup compile hls -g master --git-describe-version --ghc 9.2.3 -- --index-state=@(date '+%s') # compile a specific commit for ghc 9.2.3 and set a specifc version for the binary name @@ -287,12 +285,12 @@ ghcCompileOpts = hlsCompileOpts :: Parser HLSCompileOptions hlsCompileOpts = HLSCompileOptions - <$> ((HLS.SourceDist <$> option + <$> ((HLS.HackageDist <$> option (eitherReader (first (const "Not a valid version") . version . T.pack) ) (short 'v' <> long "version" <> metavar "VERSION" <> help - "The tool version to compile" + "The version to compile (pulled from hackage)" <> (completer $ versionCompleter Nothing HLS) ) ) @@ -307,12 +305,12 @@ hlsCompileOpts = )) )) <|> - (HLS.HackageDist <$> (option + (HLS.SourceDist <$> (option (eitherReader (first (const "Not a valid version") . version . T.pack) ) - (long "hackage-version" <> metavar "HACKAGE_VERSION" <> help - "The hackage version to compile" + (long "source-dist" <> metavar "VERSION" <> help + "The version to compile (pulled from packaged git sources)" <> (completer $ versionCompleter Nothing HLS) ) )) diff --git a/docs/guide.md b/docs/guide.md index 0b15fa5..3f3bd40 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -202,7 +202,9 @@ and produce the binaries `ghc-8.10.2-eff` and `ghc-head` respectively. GHCup always needs to know which version the bindist corresponds to (this is not automatically detected). -## Compiling GHC from source +## Compiling from source + +### GHC Compiling from source is supported for both source tarballs and arbitrary git refs. See `ghcup compile ghc --help` for a list of all available options. @@ -214,6 +216,45 @@ Common `build.mk` options are explained [here](https://gitlab.haskell.org/ghc/gh Make sure your system meets all the [prerequisites](https://gitlab.haskell.org/ghc/ghc/-/wikis/building/preparation). +### HLS + +There are 3 main ways to compile HLS from source. + +1. from hackage (should have up to date version bounds) + - `ghcup compile hls --version 1.7.0.0 --ghc 9.2.3` +2. from git (allows to build latest sources and PRs) + - `ghcup compile hls --git-ref master --ghc 9.2.3` + - `ghcup compile hls --git-ref a32db0b --ghc 9.2.3` + - `ghcup compile hls --git-ref 1.7.0.0 --ghc 9.2.3` +3. from source distribution that's packaged during release from the corresponding git sources + - `ghcup compile hls --source-dist 1.7.0.0 --ghc 9.2.3` + +All these use `cabal v2-install` under the hood, so all build components are cached. +You can pass arbitrary arguments to cabal, e.g. set the index state like so: + +```sh +ghcup compile hls --git-ref master --ghc 9.2.3 -- --index-state=2022-06-12T00:00:00Z --allow-newer +``` + +You can pass `--ghc ` multiple times to install for many GHCs at once. + +When building from git sources, ghcup will auto-detect the HLS version that the git commit corresponds to +from the `haskell-language-server.cabal` file. This version might not have been updated since the last release. +If you want to avoid overwriting the existing installed HLS version, you can instruct ghcup to use `git describe` +to set the HLS version instead: + +```sh +ghcup compile hls --git-ref master --ghc 9.2.3 --git-describe-version +``` + +You can also set the version explicitly: + +```sh +ghcup compile hls --git-ref master --ghc 9.2.3 --overwrite-version 1.7.0.0-p1 +``` + +As always, check `ghcup compile hls --help`. + ### Cross support ghcup can compile and install a cross GHC for any target. However, this From 9673d28d3ea4da121c94d308fdce50bcbfe8261d Mon Sep 17 00:00:00 2001 From: Julian Ospald Date: Mon, 11 Jul 2022 00:40:39 +0200 Subject: [PATCH 5/7] Docs --- docs/guide.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/guide.md b/docs/guide.md index 3f3bd40..ab8740a 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -253,6 +253,8 @@ You can also set the version explicitly: ghcup compile hls --git-ref master --ghc 9.2.3 --overwrite-version 1.7.0.0-p1 ``` +To instruct cabal to run `cabal update` before building, run `ghcup compile hls --version 1.7.0.0 --ghc 9.2.3 --cabal-update` + As always, check `ghcup compile hls --help`. ### Cross support From 35bda8d67aaa398f92fafd6366a39ee8ec251654 Mon Sep 17 00:00:00 2001 From: Julian Ospald Date: Mon, 11 Jul 2022 19:49:08 +0200 Subject: [PATCH 6/7] Fix hlint warnings --- lib/GHCup.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/GHCup.hs b/lib/GHCup.hs index 785b05a..2a3b248 100644 --- a/lib/GHCup.hs +++ b/lib/GHCup.hs @@ -206,9 +206,8 @@ rmGhcupDirs = do | isWindows = removeDirIfEmptyOrIsSymlink binDir | otherwise = do isXDGStyle <- liftIO useXDG - if not isXDGStyle - then removeDirIfEmptyOrIsSymlink binDir - else pure () + when (not isXDGStyle) $ + removeDirIfEmptyOrIsSymlink binDir reportRemainingFiles :: (MonadMask m, MonadIO m) => FilePath -> m [FilePath] reportRemainingFiles dir = do From 284fe1b3b6a487f20796e307f90902c2b4e96ccd Mon Sep 17 00:00:00 2001 From: Julian Ospald Date: Tue, 12 Jul 2022 00:05:08 +0200 Subject: [PATCH 7/7] Fix parser and completer for 'ghcup compile hls --version' --- app/ghcup/GHCup/OptParse/Common.hs | 8 +++++--- app/ghcup/GHCup/OptParse/Compile.hs | 9 +++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/app/ghcup/GHCup/OptParse/Common.hs b/app/ghcup/GHCup/OptParse/Common.hs index dba8664..95fde6b 100644 --- a/app/ghcup/GHCup/OptParse/Common.hs +++ b/app/ghcup/GHCup/OptParse/Common.hs @@ -440,9 +440,11 @@ tagCompleter tool add = listIOCompleter $ do pure $ nub $ (add ++) $ fmap tagToString allTags VLeft _ -> pure (nub $ ["recommended", "latest"] ++ add) - versionCompleter :: Maybe ListCriteria -> Tool -> Completer -versionCompleter criteria tool = listIOCompleter $ do +versionCompleter criteria tool = versionCompleter' criteria tool (const True) + +versionCompleter' :: Maybe ListCriteria -> Tool -> (Version -> Bool) -> Completer +versionCompleter' criteria tool filter' = listIOCompleter $ do dirs' <- liftIO getAllDirs let loggerConfig = LoggerConfig { lcPrintDebug = False @@ -471,7 +473,7 @@ versionCompleter criteria tool = listIOCompleter $ do runEnv = flip runReaderT appState installedVersions <- runEnv $ listVersions (Just tool) criteria - return $ T.unpack . prettyVer . lVer <$> installedVersions + return $ fmap (T.unpack . prettyVer) . filter filter' . fmap lVer $ installedVersions toolDlCompleter :: Tool -> Completer diff --git a/app/ghcup/GHCup/OptParse/Compile.hs b/app/ghcup/GHCup/OptParse/Compile.hs index 82df4d1..89b8157 100644 --- a/app/ghcup/GHCup/OptParse/Compile.hs +++ b/app/ghcup/GHCup/OptParse/Compile.hs @@ -32,7 +32,8 @@ import Control.Monad.Trans.Resource import Data.Bifunctor import Data.Functor import Data.Maybe -import Data.Versions ( Version, prettyVer, version ) +import Data.Versions ( Version, prettyVer, version, pvp ) +import qualified Data.Versions as V import Data.Text ( Text ) import Haskus.Utils.Variant.Excepts import Options.Applicative hiding ( style ) @@ -43,7 +44,7 @@ import Text.PrettyPrint.HughesPJClass ( prettyShow ) import URI.ByteString hiding ( uriParser ) import qualified Data.Text as T -import Control.Exception.Safe (MonadMask) +import Control.Exception.Safe (MonadMask, displayException) import System.FilePath (isPathSeparator) import Text.Read (readEither) @@ -287,11 +288,11 @@ hlsCompileOpts = HLSCompileOptions <$> ((HLS.HackageDist <$> option (eitherReader - (first (const "Not a valid version") . version . T.pack) + ((>>= first displayException . V.version . V.prettyPVP) . first (const "Not a valid PVP version") . pvp . T.pack) ) (short 'v' <> long "version" <> metavar "VERSION" <> help "The version to compile (pulled from hackage)" - <> (completer $ versionCompleter Nothing HLS) + <> (completer $ versionCompleter' Nothing HLS (either (const False) (const True) . V.pvp . V.prettyVer)) ) ) <|>