From 68731892ccf22e9120bf07ca94c46b405a980ff3 Mon Sep 17 00:00:00 2001 From: Julian Ospald Date: Sun, 1 Oct 2023 15:39:32 +0800 Subject: [PATCH] Validate all tags in CI, wrt #135 --- .github/workflows/bindists.yaml | 63 +++++++++++++++++++++++ README.md | 4 +- cabal.project | 2 +- ghcup-gen/Main.hs | 25 +++++++-- ghcup-gen/Validate.hs | 91 ++++++++++++++++++++------------- 5 files changed, 145 insertions(+), 40 deletions(-) diff --git a/.github/workflows/bindists.yaml b/.github/workflows/bindists.yaml index b72eaf3..21349ce 100644 --- a/.github/workflows/bindists.yaml +++ b/.github/workflows/bindists.yaml @@ -18,6 +18,11 @@ on: required: true default: ghcup-0.0.7.yaml type: string + channel: + description: Distribution channel (main|prerelease|nightly) + required: true + default: Main + type: string env: BOOTSTRAP_HASKELL_NONINTERACTIVE: 1 BOOTSTRAP_HASKELL_MINIMAL: 1 @@ -25,6 +30,7 @@ env: TOOL: ${{ github.event.inputs.tool }} VERSION: ${{ github.event.inputs.version }} METADATA_FILE: ${{ github.event.inputs.metadataFile }} + CHANNEL: ${{ github.event.inputs.channel }} jobs: bindist-install: name: linux-${{ matrix.image }} @@ -143,6 +149,63 @@ jobs: with: args: sh -c '.github/workflows/install-bindist.sh' + validate: + name: ghcup-gen check + runs-on: ubuntu-latest + env: + GHC: 9.2.8 + CABAL: 3.10.1.0 + steps: + - name: create ~/.local/bin + run: mkdir -p "$HOME/.local/bin" + shell: bash + + - name: Add ~/.local/bin to PATH + run: echo "$HOME/.local/bin" >> $GITHUB_PATH + shell: bash + + - name: install yamllint + run: pip install yamllint + + - name: Update cabal cache + run: cabal update + shell: bash + + - name: Install requirements + shell: sh + run: | + export DEBIAN_FRONTEND=noninteractive + export TZ=Asia/Singapore + sudo apt-get update && sudo apt-get install -y curl bash git gnupg libarchive-dev + + - uses: actions/checkout@v3 + + - name: Cache Cabal + uses: actions/cache@v2 + env: + cache-name: cache-cabal + with: + path: | + ~/.cabal/store + ~/.cabal/packages + key: v2-${{ runner.os }}-${{ env.GHC }}-${{ env.CABAL }}-build-${{ hashFiles('cabal.project') }} + restore-keys: | + v2-${{ runner.os }}-${{ env.GHC }}-${{ env.CABAL }}-build-${{ hashFiles('cabal.project') }} + v2-${{ runner.os }}-${{ env.GHC }}-${{ env.CABAL }}-build- + v2-${{ runner.os }}-${{ env.GHC }} + + - name: Install ghcup-gen + run: | + ghcup run --cabal 3.10.1.0 --ghc 9.2.8 --install -- cabal install --installdir="$HOME/.local/bin" --overwrite-policy=always --install-method=copy ghcup-gen + shell: bash + + - name: Check yaml + run: | + ghcup-gen -- check -f ${{ env.METADATA_FILE }} --channel ${{ env.CHANNEL }} + yamllint ${{ env.METADATA_FILE }} + python3 -c "import yaml ; stream = open('${{ env.METADATA_FILE }}', 'r') ; yaml.safe_load(stream)" + shell: bash + signature-test: name: Test signatures runs-on: ubuntu-latest diff --git a/README.md b/README.md index 74cab33..8c754cd 100644 --- a/README.md +++ b/README.md @@ -48,5 +48,7 @@ ghcup config add-release-channel https://raw.githubusercontent.com/haskell/ghcup ### Understanding tags Tags are documented [here](https://github.com/haskell/ghcup-hs/blob/master/lib/GHCup/Types.hs). Search for `data Tag`. -Some tags are unique. Uniqueness is checked by cabal run ghcup-gen -- check -f ghcup-.yaml`. +Some tags are unique. Uniqueness is checked by `cabal run ghcup-gen -- check -f ghcup-.yaml`. + +If you want to check prereleases, do: `cabal run ghcup-gen -- check -f ghcup-prereleases-.yaml --channel=prerelease` diff --git a/cabal.project b/cabal.project index 183896b..6a87d11 100644 --- a/cabal.project +++ b/cabal.project @@ -7,7 +7,7 @@ package ghcup source-repository-package type: git location: https://github.com/haskell/ghcup-hs.git - tag: fd6ff9f8ece147bb4527843822462c72824e8ba7 + tag: e27fed09f3eb4b0b72ce7825c65f16a4202a2399 constraints: http-io-streams -brotli, any.aeson >= 2.0.1.0 diff --git a/ghcup-gen/Main.hs b/ghcup-gen/Main.hs index 1781974..da92d7d 100644 --- a/ghcup-gen/Main.hs +++ b/ghcup-gen/Main.hs @@ -105,11 +105,30 @@ inputP :: Parser Input inputP = fileInput <|> stdInput data ValidateYAMLOpts = ValidateYAMLOpts - { vInput :: Maybe Input + { vChannel :: DistributionChannel + , vInput :: Maybe Input } validateYAMLOpts :: Parser ValidateYAMLOpts -validateYAMLOpts = ValidateYAMLOpts <$> optional inputP +validateYAMLOpts = ValidateYAMLOpts <$> channelParser <*> optional inputP + +channelParser :: Parser DistributionChannel +channelParser = + option + (eitherReader chanP) + (long "channel" <> metavar "CHANNEL" <> help + "Signal which distribution channel the YAML denotes: (main | prerelease | nightly). Main is defaul." + <> value MainChan + ) + where + chanP :: String -> Either String DistributionChannel + chanP s' | t == T.pack "main" = Right MainChan + | t == T.pack "prerelease" = Right PrereleaseChan + | t == T.pack "prereleases" = Right PrereleaseChan + | t == T.pack "nightly" = Right NightlyChan + | t == T.pack "nightlies" = Right NightlyChan + | otherwise = Left ("Unknown channel value: " <> s') + where t = T.toLower (T.pack s') tarballFilterP :: Parser TarballFilter tarballFilterP = option readm $ @@ -205,7 +224,7 @@ main = do _ <- customExecParser (prefs showHelpOnError) (info (opts <**> helper) idm) >>= \Options {..} -> case optCommand of - ValidateYAML vopts -> withValidateYamlOpts vopts validate + ValidateYAML vopts@ValidateYAMLOpts{ .. } -> withValidateYamlOpts vopts (validate vChannel) ValidateTarballs vopts tarballFilter -> withValidateYamlOpts vopts (validateTarballs tarballFilter) GenerateHlsGhc vopts format output -> withValidateYamlOpts vopts (generateHLSGhc format output) GenerateToolTable vopts output -> withValidateYamlOpts vopts (generateTable output) diff --git a/ghcup-gen/Validate.hs b/ghcup-gen/Validate.hs index 78d7cd0..949e888 100644 --- a/ghcup-gen/Validate.hs +++ b/ghcup-gen/Validate.hs @@ -51,6 +51,11 @@ data ValidationError = InternalError String instance Exception ValidationError +data DistributionChannel = MainChan + | PrereleaseChan + | NightlyChan + deriving (Show, Eq) + addError :: (MonadReader (IORef Int) m, MonadIO m, Monad m) => m () addError = do @@ -66,8 +71,9 @@ validate :: ( Monad m , MonadUnliftIO m , HasGHCupInfo env ) - => m ExitCode -validate = do + => DistributionChannel + -> m ExitCode +validate distroChannel = do GHCupInfo { _ghcupDownloads = dls } <- getGHCupInfo ref <- liftIO $ newIORef 0 @@ -95,33 +101,36 @@ validate = do lift $ logInfo "All good" pure ExitSuccess where - checkHasRequiredPlatforms t v tags arch pspecs = do - let v' = prettyVer v - arch' = prettyShow arch - when (Linux UnknownLinux `notElem` pspecs) $ do - lift $ logError $ - "Linux UnknownLinux missing for for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack arch' - addError - when ((Darwin `notElem` pspecs) && arch == A_64) $ do - lift $ logError $ "Darwin missing for for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack arch' - addError - when ((FreeBSD `notElem` pspecs) && arch == A_64) $ lift $ logWarn $ - "FreeBSD missing for for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack arch' - when (Windows `notElem` pspecs && arch == A_64) $ do - lift $ logError $ "Windows missing for for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack arch' - addError + checkHasRequiredPlatforms t v tags arch pspecs + -- relax requirements for prerelease and nightly channels + | distroChannel `elem` [PrereleaseChan, NightlyChan] = pure () + | otherwise = do + let v' = prettyVer v + arch' = prettyShow arch + when (Linux UnknownLinux `notElem` pspecs) $ do + lift $ logError $ + "Linux UnknownLinux missing for for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack arch' + addError + when ((Darwin `notElem` pspecs) && arch == A_64) $ do + lift $ logError $ "Darwin missing for for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack arch' + addError + when ((FreeBSD `notElem` pspecs) && arch == A_64) $ lift $ logWarn $ + "FreeBSD missing for for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack arch' + when (Windows `notElem` pspecs && arch == A_64) $ do + lift $ logError $ "Windows missing for for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack arch' + addError - -- alpine needs to be set explicitly, because - -- we cannot assume that "Linux UnknownLinux" runs on Alpine - -- (although it could be static) - when (Linux Alpine `notElem` pspecs) $ - case t of - GHCup | arch `elem` [A_64, A_32] -> lift (logError $ "Linux Alpine missing for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack (prettyShow arch)) >> addError - Cabal | v > [vver|2.4.1.0|] - , arch `elem` [A_64, A_32] -> lift (logError $ "Linux Alpine missing for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack (prettyShow arch)) >> addError - GHC | Latest `elem` tags || Recommended `elem` tags - , arch `elem` [A_64, A_32] -> lift (logError $ "Linux Alpine missing for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack (prettyShow arch)) - _ -> lift $ logWarn $ "Linux Alpine missing for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack (prettyShow arch) + -- alpine needs to be set explicitly, because + -- we cannot assume that "Linux UnknownLinux" runs on Alpine + -- (although it could be static) + when (Linux Alpine `notElem` pspecs) $ + case t of + GHCup | arch `elem` [A_64, A_32] -> lift (logError $ "Linux Alpine missing for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack (prettyShow arch)) >> addError + Cabal | v > [vver|2.4.1.0|] + , arch `elem` [A_64, A_32] -> lift (logError $ "Linux Alpine missing for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack (prettyShow arch)) >> addError + GHC | Latest `elem` tags || Recommended `elem` tags + , arch `elem` [A_64, A_32] -> lift (logError $ "Linux Alpine missing for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack (prettyShow arch)) + _ -> lift $ logWarn $ "Linux Alpine missing for " <> T.pack (prettyShow t) <> " " <> v' <> " " <> T.pack (prettyShow arch) checkUniqueTags tool = do GHCupInfo { _ghcupDownloads = dls } <- lift getGHCupInfo @@ -145,12 +154,15 @@ validate = do lift $ logError $ "Tags not unique for " <> T.pack (prettyShow tool) <> ": " <> T.pack (prettyShow xs) addError where - isUniqueTag Latest = True - isUniqueTag Recommended = True - isUniqueTag Old = False - isUniqueTag Prerelease = False - isUniqueTag (Base _) = False - isUniqueTag (UnknownTag _) = False + isUniqueTag Latest = True + isUniqueTag Recommended = True + isUniqueTag Old = False + isUniqueTag Prerelease = False + isUniqueTag LatestPrerelease = True + isUniqueTag Nightly = False + isUniqueTag LatestNightly = True + isUniqueTag (Base _) = False + isUniqueTag (UnknownTag _) = False checkGHCVerIsValid = do GHCupInfo { _ghcupDownloads = dls } <- lift getGHCupInfo @@ -166,12 +178,21 @@ validate = do checkMandatoryTags tool = do GHCupInfo { _ghcupDownloads = dls } <- lift getGHCupInfo let allTags = _viTags =<< M.elems (availableToolVersions dls tool) - forM_ [Latest, Recommended] $ \t -> case t `elem` allTags of + forM_ (mandatoryTags tool) $ \t -> case t `elem` allTags of False -> do lift $ logError $ "Tag " <> T.pack (prettyShow t) <> " missing from " <> T.pack (prettyShow tool) addError True -> pure () + mandatoryTags tool + -- due to a quirk, even for ghcup prereleases we need the 'latest' tag + -- https://github.com/haskell/ghcup-hs/issues/891 + | tool == GHCup = [Latest, Recommended] + | otherwise = case distroChannel of + MainChan -> [Latest, Recommended] + PrereleaseChan -> [LatestPrerelease] + NightlyChan -> [LatestNightly] + -- all GHC versions must have a base tag checkGHCHasBaseVersion = do GHCupInfo { _ghcupDownloads = dls } <- lift getGHCupInfo