Document distribution policies

Fixes #695

.github/workflows/release.yaml

@@ -319,7 +319,7 @@ jobs:
 
       - if: matrix.ARCH == 'ARM'
         uses: docker://hasufell/arm32v7-ubuntu-haskell:focal
-        name: Run build (armv7 linux)
+        name: Run test (armv7 linux)
         with:
           args: sh .github/scripts/test.sh
         env:
@@ -330,7 +330,7 @@ jobs:
 
       - if: matrix.ARCH == 'ARM64'
         uses: docker://hasufell/arm64v8-ubuntu-haskell:focal
-        name: Run build (aarch64 linux)
+        name: Run test (aarch64 linux)
         with:
           args: sh .github/scripts/test.sh
         env:

app/ghcup/GHCup/OptParse.hs

@@ -67,13 +67,13 @@ import           URI.ByteString
 import qualified Data.ByteString.UTF8          as UTF8
 
 
-
 data Options = Options
   {
   -- global options
     optVerbose     :: Maybe Bool
   , optCache       :: Maybe Bool
   , optMetaCache   :: Maybe Integer
+  , optMetaMode    :: Maybe MetaMode
   , optPlatform    :: Maybe PlatformRequest
   , optUrlSource   :: Maybe URI
   , optNoVerify    :: Maybe Bool
@@ -116,7 +116,8 @@ opts =
   Options
     <$> invertableSwitch "verbose" (Just 'v') False (help "Enable verbosity (default: disabled)")
     <*> invertableSwitch "cache" (Just 'c') False (help "Cache downloads in ~/.ghcup/cache (default: disabled)")
-    <*> optional (option auto (long "metadata-caching" <> help "How long the yaml metadata caching interval is (in seconds), 0 to disable" <> internal))
+    <*> optional (option auto (long "metadata-caching" <> metavar "SEC" <> help "How long the yaml metadata caching interval is (in seconds), 0 to disable"))
+    <*> optional (option auto (long "metadata-fetching-mode" <> metavar "<Strict|Lax>" <> help "Whether to fail on metadata download failure (Strict) or fall back to cached version (Lax (default))"))
     <*> optional
       (option
         (eitherReader platformParser)

app/ghcup/GHCup/OptParse/Config.hs

@@ -124,6 +124,7 @@ updateSettings :: UserSettings -> Settings -> Settings
 updateSettings UserSettings{..} Settings{..} =
    let cache'      = fromMaybe cache uCache
        metaCache'  = fromMaybe metaCache uMetaCache
+       metaMode'   = fromMaybe metaMode uMetaMode
        noVerify'   = fromMaybe noVerify uNoVerify
        keepDirs'   = fromMaybe keepDirs uKeepDirs
        downloader' = fromMaybe downloader uDownloader
@@ -132,7 +133,7 @@ updateSettings UserSettings{..} Settings{..} =
        noNetwork'  = fromMaybe noNetwork uNoNetwork
        gpgSetting' = fromMaybe gpgSetting uGPGSetting
        platformOverride' = uPlatformOverride <|> platformOverride
-   in Settings cache' metaCache' noVerify' keepDirs' downloader' verbose' urlSource' noNetwork' gpgSetting' noColor platformOverride'
+   in Settings cache' metaCache' metaMode' noVerify' keepDirs' downloader' verbose' urlSource' noNetwork' gpgSetting' noColor platformOverride'
 
 
 

app/ghcup/GHCup/OptParse/Install.hs

@@ -454,7 +454,7 @@ install installCommand settings getAppState' runLogger = case installCommand of
            (_tvVersion -> v, vi) <- liftE $ fromVersion instVer HLS
            -- TODO: support legacy
            liftE $ runBothE' (installHLSBindist
-                                      (DownloadInfo uri (Just $ RegexDir "haskell-language-server-*") "" Nothing)
+                                      (DownloadInfo uri (if isWindows then Nothing else Just (RegexDir "haskell-language-server-*")) "" Nothing)
                                       v
                                       (maybe GHCupInternal IsolateDir isolateDir)
                                       forceInstall

app/ghcup/Main.hs

@@ -79,6 +79,7 @@ toSettings options = do
    mergeConf Options{..} UserSettings{..} noColor =
      let cache       = fromMaybe (fromMaybe (Types.cache defaultSettings) uCache) optCache
          metaCache   = fromMaybe (fromMaybe (Types.metaCache defaultSettings) uMetaCache) optMetaCache
+         metaMode    = fromMaybe (fromMaybe (Types.metaMode defaultSettings) uMetaMode) optMetaMode
          noVerify    = fromMaybe (fromMaybe (Types.noVerify defaultSettings) uNoVerify) optNoVerify
          verbose     = fromMaybe (fromMaybe (Types.verbose defaultSettings) uVerbose) optVerbose
          keepDirs    = fromMaybe (fromMaybe (Types.keepDirs defaultSettings) uKeepDirs) optKeepDirs

data/config.yaml

@@ -40,6 +40,12 @@ key-bindings:
 # of the file. These usually are in '~/.ghcup/cache/ghcup-<ver>.yaml'.
 meta-cache: 300 # in seconds
 
+# When trying to download ghcup metadata, this option decides what to do
+# when the download fails:
+#   1. Lax: use existing ~/.ghcup/cache/ghcup-<ver>.yaml as fallback (default)
+#   2. Strict: fail hard
+meta-mode: Lax # Strict | Lax
+
 # Where to get GHC/cabal/hls download info/versions from. For more detailed explanation
 # check the 'URLSource' type in the code.
 url-source:

docs/about.md

@@ -60,6 +60,29 @@ All you wanted to know about GHCup.
 3. handling cabal projects
 4. being a stack alternative
 
+## Distribution policies
+
+Like most Linux distros and other distribution channels, GHCup also
+follows certain policies. These are as follows:
+
+1. The end-user experience is our primary concern
+    - ghcup in CI systems as a use case is a first class citizen
+2. We strive to collaborate with all maintainers of all the tools we support and maintain a good relationship
+3. We may fix build system or other distribution bugs in upstream bindists
+    - these are always communicated upstream
+4. We may even patch source code of supported tools in very rare cases if that is required to ensure that the end-user experience does not break
+    - we'll first try to upstream any such required patch and demand a new release to avoid downstream patching
+    - patches will be communicated to the maintainers either way and we'll strive to get their review
+    - they will also be communicated to the end-user
+    - they will be uploaded along with the bindist
+    - we will avoid maintaining long-running downstream patches (currently zero)
+5. We may add bindists for platforms that upstream does not support
+    - this is currently the case for GHC for e.g. Alpine and possibly FreeBSD in the future
+    - this is currently also the case for stack on darwin M1
+    - we don't guarantee for unofficial bindists that the test suite passes at the moment (this may change in the future)
+6. We GPG sign all the GHCup metadata as well as the unofficial bindists
+    - any trust issues relating to missing checksums or GPG signatures is a bug and given high priority
+
 ## How
 
 Installs a specified GHC version into `~/.ghcup/ghc/<ver>`, and places `ghc-<ver>` symlinks in `~/.ghcup/bin/`.
@@ -75,15 +98,15 @@ cabal-install/HLS/stack are installed in `~/.ghcup/bin/<tool>-<ver>` and have un
 ## Known users
 
 * CI:
-	- [Github actions/virtual-environments](https://github.com/actions/virtual-environments)
+    - [Github actions/virtual-environments](https://github.com/actions/virtual-environments)
-	- [Github haskell/actions/setup](https://github.com/haskell/actions/tree/main/setup)
+    - [Github haskell/actions/setup](https://github.com/haskell/actions/tree/main/setup)
-	- [haskell-ci](https://github.com/haskell-CI/haskell-ci)
+    - [haskell-ci](https://github.com/haskell-CI/haskell-ci)
 * mirrors:
-	- [sjtug](https://mirror.sjtu.edu.cn/docs/ghcup)
+    - [sjtug](https://mirror.sjtu.edu.cn/docs/ghcup)
 * tools:
-	- [vscode-haskell](https://github.com/haskell/vscode-haskell)
+    - [vscode-haskell](https://github.com/haskell/vscode-haskell)
-	- [nvim-lsp-installer](https://github.com/williamboman/nvim-lsp-installer)
+    - [nvim-lsp-installer](https://github.com/williamboman/nvim-lsp-installer)
-	- [vabal](https://github.com/Franciman/vabal)
+    - [vabal](https://github.com/Franciman/vabal)
 
 ## Known problems
 

lib/GHCup/Download.hs

@@ -162,17 +162,21 @@ getBase :: ( MonadReader env m
            , MonadMask m
            )
         => URI
-        -> Excepts '[GPGError, DigestError, ContentLengthError, JSONError, FileDoesNotExistError] m GHCupInfo
+        -> Excepts '[DownloadFailed, GPGError, DigestError, ContentLengthError, JSONError, FileDoesNotExistError] m GHCupInfo
 getBase uri = do
-  Settings { noNetwork, downloader } <- lift getSettings
+  Settings { noNetwork, downloader, metaMode } <- lift getSettings
 
   -- try to download yaml... usually this writes it into cache dir,
   -- but in some cases not (e.g. when using file://), so we honour
   -- the return filepath, if any
   mYaml <- if noNetwork && view (uriSchemeL' % schemeBSL') uri /= "file" -- for file://, let it fall through
            then pure Nothing
-           else handleIO (\e -> lift (warnCache (displayException e) downloader) >> pure Nothing)
+           else handleIO (\e -> case metaMode of
-               . catchE @_ @_ @'[] (\e@(DownloadFailed _) -> lift (warnCache (prettyShow e) downloader) >> pure Nothing)
+                                  Strict -> throwIO e
+                                  Lax -> lift (warnCache (displayException e) downloader) >> pure Nothing)
+               . catchE @_ @_ @'[DownloadFailed] (\e@(DownloadFailed _) -> case metaMode of
+                   Strict -> throwE e
+                   Lax -> lift (warnCache (prettyShow e) downloader) >> pure Nothing)
                . fmap Just
                . smartDl
                $ uri
@@ -336,7 +340,6 @@ download uri gpgUri eDigest eCSize dest mfn etags
   | scheme == "file"  = do
       let destFile' = T.unpack . decUTF8Safe $ view pathL' uri
       lift $ logDebug $ "using local file: " <> T.pack destFile'
-      forM_ eCSize  (liftE . flip checkCSize  destFile')
       forM_ eDigest (liftE . flip checkDigest destFile')
       pure destFile'
   | otherwise = throwE $ DownloadFailed (variantFromValue UnsupportedScheme)
@@ -444,7 +447,6 @@ download uri gpgUri eDigest eCSize dest mfn etags
         liftE $ lEM @_ @'[ProcessError] $ exec "curl"
             (o' ++ (if etags then ["--dump-header", dh] else [])
                 ++ maybe [] (\t -> ["-H", "If-None-Match: " <> T.unpack t]) metag
-                ++ maybe [] (\s -> ["--max-file-size", show s]) eCSize
                 ++ ["-fL", "-o", destFileTemp, T.unpack uri']) Nothing Nothing
         headers <- liftIO $ T.readFile dh
 

lib/GHCup/Prelude/File/Posix.hs

@@ -48,6 +48,7 @@ import           Streamly.Internal.Data.Unfold.Type
 import qualified Streamly.Internal.Data.Unfold as U
 import           Streamly.Internal.Control.Concurrent ( withRunInIO )
 import           Streamly.Internal.Data.IOFinalizer   ( newIOFinalizer, runIOFinalizer )
+import GHC.IO.Exception (IOException(ioe_type), IOErrorType (..))
 
 
 -- | On unix, we can use symlinks, so we just get the
@@ -116,8 +117,18 @@ copyFile from to fail' = do
         let dflags = [ FD.oNofollow
                      , if fail' then FD.oExcl else FD.oTrunc
                      ]
+        let openFdHandle' = openFdHandle to SPI.WriteOnly dflags $ Just sourceFileMode
         bracket
-          (openFdHandle to SPI.WriteOnly dflags $ Just sourceFileMode)
+          (handleIO (\e -> if
+                              -- if we copy from regular file to symlink, we need
+                              -- to delete the symlink
+                              | ioe_type e == InvalidArgument
+                              , not fail' -> do
+                                 removeLink to
+                                 openFdHandle'
+                              | otherwise -> throwIO e
+                    )
+            openFdHandle')
           (hClose . snd)
           $ \(_, tH) -> do
               hSetBinaryMode fH True

lib/GHCup/Types.hs

@@ -298,10 +298,16 @@ instance NFData URLSource
 instance NFData (URIRef Absolute) where
   rnf (URI !_ !_ !_ !_ !_) = ()
 
+data MetaMode = Strict
+              | Lax
+  deriving (Show, Read, Eq, GHC.Generic)
+
+instance NFData MetaMode
 
 data UserSettings = UserSettings
   { uCache       :: Maybe Bool
   , uMetaCache   :: Maybe Integer
+  , uMetaMode    :: Maybe MetaMode
   , uNoVerify    :: Maybe Bool
   , uVerbose     :: Maybe Bool
   , uKeepDirs    :: Maybe KeepDirs
@@ -315,13 +321,14 @@ data UserSettings = UserSettings
   deriving (Show, GHC.Generic)
 
 defaultUserSettings :: UserSettings
-defaultUserSettings = UserSettings Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing
+defaultUserSettings = UserSettings Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing
 
 fromSettings :: Settings -> Maybe KeyBindings -> UserSettings
 fromSettings Settings{..} Nothing =
   UserSettings {
       uCache = Just cache
     , uMetaCache = Just metaCache
+    , uMetaMode = Just metaMode
     , uNoVerify = Just noVerify
     , uVerbose = Just verbose
     , uKeepDirs = Just keepDirs
@@ -347,6 +354,7 @@ fromSettings Settings{..} (Just KeyBindings{..}) =
   in UserSettings {
       uCache = Just cache
     , uMetaCache = Just metaCache
+    , uMetaMode = Just metaMode
     , uNoVerify = Just noVerify
     , uVerbose = Just verbose
     , uKeepDirs = Just keepDirs
@@ -427,6 +435,7 @@ instance NFData LeanAppState
 data Settings = Settings
   { cache            :: Bool
   , metaCache        :: Integer
+  , metaMode         :: MetaMode
   , noVerify         :: Bool
   , keepDirs         :: KeepDirs
   , downloader       :: Downloader
@@ -443,7 +452,7 @@ defaultMetaCache :: Integer
 defaultMetaCache = 300 -- 5 minutes
 
 defaultSettings :: Settings
-defaultSettings = Settings False defaultMetaCache False Never Curl False GHCupURL False GPGNone False Nothing
+defaultSettings = Settings False defaultMetaCache Lax False Never Curl False GHCupURL False GPGNone False Nothing
 
 instance NFData Settings
 

lib/GHCup/Types/JSON.hs

@@ -43,6 +43,7 @@ import qualified Text.Megaparsec               as MP
 import qualified Text.Megaparsec.Char          as MPC
 
 
+deriveJSON defaultOptions { fieldLabelModifier = removeLensFieldLabel } ''MetaMode
 deriveJSON defaultOptions { fieldLabelModifier = removeLensFieldLabel } ''Architecture
 deriveJSON defaultOptions { fieldLabelModifier = removeLensFieldLabel } ''LinuxDistro
 deriveJSON defaultOptions { fieldLabelModifier = removeLensFieldLabel } ''VSep

lib/GHCup/Utils/Dirs.hs

@@ -465,15 +465,22 @@ withGHCupTmpDir :: ( MonadReader env m
                    , MonadMask m
                    , MonadIO m)
                 => m GHCupPath
-withGHCupTmpDir = snd <$> withRunInIO (\run ->
+withGHCupTmpDir = do
-  run
+  Settings{keepDirs} <- getSettings
-    $ allocate
+  snd <$> withRunInIO (\run ->
-        (run mkGhcupTmpDir)
+    run
-        (\fp ->
+      $ allocate
-            handleIO (\e -> run
+          (run mkGhcupTmpDir)
-                $ logDebug ("Resource cleanup failed for " <> T.pack (fromGHCupPath fp) <> ", error was: " <> T.pack (displayException e)))
+          (\fp -> if -- we don't know whether there was a failure, so can only
-            . removePathForcibly
+                     -- decide for 'Always'
-            $ fp))
+                     | keepDirs == Always -> pure ()
+                     | otherwise -> handleIO (\e -> run
+                        $ logDebug ("Resource cleanup failed for "
+                                   <> T.pack (fromGHCupPath fp)
+                                   <> ", error was: "
+                                   <> T.pack (displayException e)))
+                        . removePathForcibly
+                        $ fp))