Fix build on GHC-7.10.3

Also add getDirsFilesStream.

.travis.yml

@@ -9,18 +9,50 @@ matrix:
   include:
     - env: CABALVER=3.0 GHCVER=7.10.3 SKIP_DOCTESTS=yes
       addons: {apt: {packages: [cabal-install-3.0,ghc-7.10.3], sources: [hvr-ghc]}}
+      before_install:
+       - sudo apt-get install -y hscolour
+       - export PATH=~/.cabal/bin:/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$PATH
     - env: CABALVER=3.0 GHCVER=8.0.2 SKIP_DOCTESTS=yes
       addons: {apt: {packages: [cabal-install-3.0,ghc-8.0.2], sources: [hvr-ghc]}}
+      before_install:
+       - sudo apt-get install -y hscolour
+       - export PATH=~/.cabal/bin:/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$PATH
     - env: CABALVER=3.0 GHCVER=8.2.2
       addons: {apt: {packages: [cabal-install-3.0,ghc-8.2.2], sources: [hvr-ghc]}}
+      before_install:
+       - sudo apt-get install -y hscolour
+       - export PATH=~/.cabal/bin:/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$PATH
     - env: CABALVER=3.0 GHCVER=8.4.4
       addons: {apt: {packages: [cabal-install-3.0,ghc-8.4.4], sources: [hvr-ghc]}}
+      before_install:
+       - sudo apt-get install -y hscolour
+       - export PATH=~/.cabal/bin:/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$PATH
     - env: CABALVER=3.0 GHCVER=8.6.5
       addons: {apt: {packages: [cabal-install-3.0,ghc-8.6.5], sources: [hvr-ghc]}}
-    - env: CABALVER=3.0 GHCVER=8.8.1
+      before_install:
-      addons: {apt: {packages: [cabal-install-3.0,ghc-8.8.1], sources: [hvr-ghc]}}
+       - sudo apt-get install -y hscolour
+       - export PATH=~/.cabal/bin:/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$PATH
+    - env: CABALVER=3.0 GHCVER=8.8.3
+      addons: {apt: {packages: [cabal-install-3.0,ghc-8.8.3], sources: [hvr-ghc]}}
+      before_install:
+       - sudo apt-get install -y hscolour
+       - export PATH=~/.cabal/bin:/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$PATH
     - env: CABALVER=head GHCVER=head
       addons: {apt: {packages: [cabal-install-head,ghc-head],  sources: [hvr-ghc]}}
+      before_install:
+       - sudo apt-get install -y hscolour
+       - export PATH=~/.cabal/bin:/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$PATH
+    - os: osx
+      osx_image: xcode11.3
+      language: generic
+      before_install:
+       - mkdir -p ~/.ghcup/bin
+       - curl https://gitlab.haskell.org/haskell/ghcup/raw/master/ghcup > ~/.ghcup/bin/ghcup
+       - chmod +x ~/.ghcup/bin/ghcup
+       - export PATH=~/.cabal/bin:~/.ghcup/bin:$PATH
+       - ghcup install 8.6.5
+       - ghcup set 8.6.5
+       - ghcup install-cabal 3.2.0.0
 
   allow_failures:
     - env: CABALVER=head GHCVER=head
@@ -29,10 +61,6 @@ env:
   global:
     - secure: HPBARvNM85ea2U0Ynq5MMe6BRlnuwqXWuSn20VY3EYCAT2njkVPYnR3O7+bGE6aq0KHAV87zz5iUfGJontd86tE0sDVjcSuRY0hqjOeJTkQq5M8WXJZOpVqlBTwDP1Q3x/fwoRa0dt9Z0tZZdKMlrf2XdcKPDdhcP1QYP4aV/jO4ZCfAQr7zVCvTae+Lp/KmwFYcBbFo/pj0duF1M4Oqx/D388b/W4jVE3lgd/TK7Ja1xWP6g+Oyvo6iQK8yJVYGdm6E+cVsNueiisnTJ/rRA53lsaC9dmWtZaFGl41wPviSU5zPq03vOuZMiyE2WtCHoo46ONXrXJ9N2soqdQVfEkr9Nw5LQl+6C5lCPEejZ575YUkuO05H3wvHMk3YY4zWXNFA9eZ47PEH8tpoUk9LPBacCKQFtp5lfRk63crba5CiFtcMyFq++0mLpNthNvtto7ffHMZrt6fvK9axI+r21VPftf/3FiFY4mnCp/Bln+ijklfZSN71VqiT20EWuqxQHw8aCpT00KA/PKGl9iJfoN4OO3XzNRTtmM+L9Im4bc1ni9YQ6N3UYg3z0nEnCLwFcTmTH/tDMHRremE0dM6B++YfcnyIhen8w+hG4bcXk7jbMUizRhUhStN7TZQuC9S4wE5whhp9c03rJZMmH5E2rlXY3lwVgeyWm1TuMp1RYWI=
 
-before_install:
- - sudo apt-get install -y hscolour
- - export PATH=~/.cabal/bin:/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$PATH
-
 install:
  - cabal --version
  - travis_retry cabal update
@@ -48,7 +76,6 @@ script:
  - (cd hpath-filepath && cabal check)
  - (cd hpath-io && cabal check)
  - cabal sdist all
- - cabal haddock --haddock-hyperlink-source --haddock-html-location=https://hackage.haskell.org/package/\$pkg-\$version/docs/ all
  - cabal install --lib all
 
 notifications:

hpath-directory/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Revision history for hpath-directory
 
+## 0.13.4 -- 2020-05-08
+
+* Add getDirsFilesStream and use streamly-posix for dircontents (#34)
+
+## 0.13.3 -- 2020-04-14
+
+* Fix tests on mac
+
+## 0.13.2 -- 2020-02-17
+
+* Fix bug in `createDirRecursive` with trailing path separators
+
 ## 0.13.1 -- 2020-01-29
 
 * Split some functionality out into 'hpath-posix'

hpath-directory/hpath-directory.cabal

@@ -1,7 +1,7 @@
 cabal-version:       >=1.10
 
 name:                hpath-directory
-version:             0.13.1
+version:             0.13.4
 synopsis:            Alternative to 'directory' package with ByteString based filepaths
 description:         This provides a safer alternative to the 'directory'
                      package. FilePaths are ByteString based, so this
@@ -43,8 +43,10 @@ library
                        , hpath-posix         >= 0.13
                        , safe-exceptions     >= 0.1
                        , streamly            >= 0.7
-                       , streamly-bytestring >= 0.1.0.1
+                       , streamly-bytestring >= 0.1.2
+                       , streamly-posix      >= 0.1.0.1
                        , time                >= 1.8
+                       , transformers
                        , unix                >= 2.5
                        , unix-bytestring     >= 0.3
                        , utf8-string

hpath-directory/src/System/Posix/RawFilePath/Directory.hs

@@ -27,7 +27,8 @@
 -- Import as:
 -- > import System.Posix.RawFilePath.Directory
 
-{-# LANGUAGE CPP #-}
+{-# LANGUAGE CPP              #-}
+{-# LANGUAGE FlexibleContexts #-} -- streamly
 
 module System.Posix.RawFilePath.Directory
   (
@@ -82,6 +83,7 @@ module System.Posix.RawFilePath.Directory
   -- * Directory reading
   , getDirsFiles
   , getDirsFiles'
+  , getDirsFilesStream
   -- * Filetype operations
   , getFileType
   -- * Others
@@ -93,8 +95,11 @@ where
 
 import           Control.Applicative            ( (<$>) )
 import           Control.Exception.Safe         ( IOException
+                                                , MonadCatch
+                                                , MonadMask
                                                 , bracket
                                                 , bracketOnError
+                                                , onException
                                                 , throwIO
                                                 , finally
                                                 )
@@ -105,6 +110,7 @@ import           Control.Monad                  ( unless
 import           Control.Monad.Catch            ( MonadThrow(..) )
 import           Control.Monad.Fail             ( MonadFail )
 import           Control.Monad.IfElse           ( unlessM )
+import           Control.Monad.IO.Class         ( liftIO )
 import qualified Data.ByteString               as BS
 import           Data.ByteString                ( ByteString )
 import           Data.Traversable               ( for )
@@ -155,6 +161,8 @@ import           Streamly
 import           Streamly.External.ByteString
 import qualified Streamly.External.ByteString.Lazy
                                                as SL
+import qualified Streamly.External.Posix.DirStream
+                                               as SD
 import qualified Streamly.Data.Fold            as FL
 import           Streamly.Memory.Array
 import qualified Streamly.FileSystem.Handle    as FH
@@ -564,7 +572,7 @@ easyCopy from to cm rm = do
 --
 -- Throws:
 --
---    - `InappropriateType` for wrong file type (directory)
+--    - `InappropriateType` or `PermissionDenied` for wrong file type (directory)
 --    - `NoSuchThing` if the file does not exist
 --    - `PermissionDenied` if the directory cannot be read
 --
@@ -742,8 +750,8 @@ createDirRecursive fm p = go p
           | en == eEXIST
           -> unlessM (doesDirectoryExist dest) (ioError e)
           | en == eNOENT
-          -> createDirRecursive fm (takeDirectory dest)
+          -> go (takeDirectory $ dropTrailingPathSeparator dest)
-            >> createDirectory dest fm
+            >> createDir fm dest
           | otherwise
           -> ioError e
 
@@ -1147,11 +1155,17 @@ getDirsFiles p = do
 -- of prepending the base path.
 getDirsFiles' :: RawFilePath        -- ^ dir to read
               -> IO [RawFilePath]
-getDirsFiles' fp = do
+getDirsFiles' fp = getDirsFilesStream fp >>= S.toList
-  fd          <- openFd fp SPI.ReadOnly [SPDF.oNofollow] Nothing
+
-  rawContents <- getDirectoryContents' fd
+
-  fmap catMaybes $ for rawContents $ \(_, f) ->
+-- | Like 'getDirsFiles'', except returning a Stream.
-    if FP.isSpecialDirectoryEntry f then pure Nothing else pure $ Just f
+getDirsFilesStream :: (MonadCatch m, MonadAsync m, MonadMask m)
+                   => RawFilePath
+                   -> IO (SerialT m RawFilePath)
+getDirsFilesStream fp = do
+  fd <- openFd fp SPI.ReadOnly [SPDF.oNofollow] Nothing
+  ds <- SPDT.fdOpendir fd `onException` SPI.closeFd fd
+  pure $ fmap snd $ SD.dirContentsStream ds
 
 
 

hpath-directory/test/Main.hs

@@ -8,6 +8,9 @@ import Test.Hspec.Formatters
 import qualified Spec
 import Utils
 import System.Posix.Temp.ByteString (mkdtemp)
+import System.Posix.Env.ByteString (getEnvDefault)
+import System.Posix.FilePath ((</>))
+import "hpath-directory" System.Posix.RawFilePath.Directory
 
 
 -- TODO: chardev, blockdev, namedpipe, socket
@@ -15,7 +18,8 @@ import System.Posix.Temp.ByteString (mkdtemp)
 
 main :: IO ()
 main = do
-  tmpBase <- mkdtemp "/tmp/"
+  tmpdir <- getEnvDefault "TMPDIR" "/tmp" >>= canonicalizePath
+  tmpBase <- mkdtemp (tmpdir </> "hpath-directory")
   writeIORef baseTmpDir (Just (tmpBase `BS.append` "/"))
   putStrLn $ ("Temporary test directory at: " ++ show tmpBase)
   hspecWith

hpath-directory/test/System/Posix/RawFilePath/Directory/CopyDirRecursiveCollectFailuresSpec.hs

@@ -125,7 +125,7 @@ spec = beforeAll_ (upTmpDir >> setupFiles) $ afterAll_ cleanupFiles $
                         "outputDir"
                         Strict
                         CollectFailures
-      (system $ "diff -r --no-dereference "
+      (system $ "diff -r "
                           ++ toString tmpDir' ++ "inputDir" ++ " "
                           ++ toString tmpDir' ++ "outputDir"
                           ++ " >/dev/null")

hpath-directory/test/System/Posix/RawFilePath/Directory/CopyDirRecursiveOverwriteSpec.hs

@@ -104,7 +104,7 @@ spec = beforeAll_ (upTmpDir >> setupFiles) $ afterAll_ cleanupFiles $
                         "outputDir"
                         Overwrite
                         FailEarly
-      (system $ "diff -r --no-dereference "
+      (system $ "diff -r "
                           ++ toString tmpDir' ++ "inputDir" ++ " "
                           ++ toString tmpDir' ++ "outputDir"
                           ++ " >/dev/null")
@@ -113,7 +113,7 @@ spec = beforeAll_ (upTmpDir >> setupFiles) $ afterAll_ cleanupFiles $
 
     it "copyDirRecursive (Overwrite, FailEarly), destination dir already exists" $ do
       tmpDir' <- getRawTmpDir
-      (system $ "diff -r --no-dereference "
+      (system $ "diff -r "
                           ++ toString tmpDir' ++ "inputDir" ++ " "
                           ++ toString tmpDir' ++ "alreadyExistsD"
                           ++ " >/dev/null")
@@ -122,7 +122,7 @@ spec = beforeAll_ (upTmpDir >> setupFiles) $ afterAll_ cleanupFiles $
                         "alreadyExistsD"
                         Overwrite
                         FailEarly
-      (system $ "diff -r --no-dereference "
+      (system $ "diff -r "
                           ++ toString tmpDir' ++ "inputDir" ++ " "
                           ++ toString tmpDir' ++ "alreadyExistsD"
                           ++ " >/dev/null")

hpath-directory/test/System/Posix/RawFilePath/Directory/CopyDirRecursiveSpec.hs

@@ -89,7 +89,7 @@ spec = beforeAll_ (upTmpDir >> setupFiles) $ afterAll_ cleanupFiles $
                         "outputDir"
                         Strict
                         FailEarly
-      (system $ "diff -r --no-dereference "
+      (system $ "diff -r "
                           ++ toString tmpDir' ++ "inputDir" ++ " "
                           ++ toString tmpDir' ++ "outputDir"
                           ++ " >/dev/null")

hpath-directory/test/System/Posix/RawFilePath/Directory/CreateDirRecursiveSpec.hs

@@ -49,6 +49,11 @@ spec = beforeAll_ (upTmpDir >> setupFiles) $ afterAll_ cleanupFiles $
       createDirRecursive' "newDir"
       deleteDir' "newDir"
 
+    it "createDirRecursive with trailing path separator, all fine" $ do
+      createDirRecursive' "newDir/foo/"
+      deleteDir' "newDir/foo"
+      deleteDir' "newDir"
+
     it "createDirRecursive, parent directories do not exist" $ do
       createDirRecursive' "some/thing/dada"
       deleteDir' "some/thing/dada"

hpath-directory/test/System/Posix/RawFilePath/Directory/DeleteFileSpec.hs

@@ -70,7 +70,7 @@ spec = beforeAll_ (upTmpDir >> setupFiles) $ afterAll_ cleanupFiles $
     it "deleteFile, wrong file type (directory)" $
       deleteFile' "dir"
         `shouldThrow`
-        (\e -> ioeGetErrorType e == InappropriateType)
+        (\e -> ioeGetErrorType e == InappropriateType || ioeGetErrorType e == PermissionDenied)
 
     it "deleteFile, file does not exist" $
       deleteFile' "doesNotExist"

hpath-io/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Revision history for hpath-io
 
+## 0.13.2 -- 2020-05-08
+
+* Add getDirsFilesStream and use streamly-posix for dircontents (#34)
+
 ## 0.13.0 -- 2020-01-26
 
 * switch to using 'hpath-bytestring' for the implementation (this is now just a wrapper module, mostly)

hpath-io/hpath-io.cabal

@@ -1,5 +1,5 @@
 name:                hpath-io
-version:             0.13.1
+version:             0.13.2
 synopsis:            High-level IO operations on files/directories
 description:         High-level IO operations on files/directories, utilizing type-safe Paths
 -- bug-reports:

hpath-io/src/HPath/IO.hs

@@ -27,7 +27,8 @@
 -- For other functions (like `copyFile`), the behavior on these file types is
 -- unreliable/unsafe. Check the documentation of those functions for details.
 
-{-# LANGUAGE PackageImports #-}
+{-# LANGUAGE FlexibleContexts #-} -- streamly
+{-# LANGUAGE PackageImports   #-}
 
 module HPath.IO
   (
@@ -82,6 +83,7 @@ module HPath.IO
   -- * Directory reading
   , getDirsFiles
   , getDirsFiles'
+  , getDirsFilesStream
   -- * Filetype operations
   , getFileType
   -- * Others
@@ -94,7 +96,9 @@ module HPath.IO
 where
 
 
-import           Control.Exception.Safe         ( bracketOnError
+import           Control.Exception.Safe         ( MonadMask
+                                                , MonadCatch
+                                                , bracketOnError
                                                 , finally
                                                 )
 import           Control.Monad.Catch            ( MonadThrow(..) )
@@ -761,6 +765,15 @@ getDirsFiles' (Path fp) = do
   for rawContents $ \r -> parseRel r
 
 
+-- | Like 'getDirsFiles'', except returning a Stream.
+getDirsFilesStream :: (MonadCatch m, MonadAsync m, MonadMask m)
+                   => Path b
+                   -> IO (SerialT m (Path Rel))
+getDirsFilesStream (Path fp) = do
+  s <- RD.getDirsFilesStream fp
+  pure (s >>= parseRel)
+
+
 
 
     ---------------------------

hpath-posix/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Revision history for hpath-posix
 
-## 0.1.0.0 -- 2020-01-29
+## 0.13.2 -- 2020-04-14
+
+* fix macOS compatibility, especially with memory bug in `fdopendir`
+
+## 0.13.1 -- 2020-02-08
+
+* Remove unnecessary dependencies
+
+
+## 0.13.0 -- 2020-01-29
 
 * First version. Released on an unsuspecting world.

hpath-posix/cbits/dirutils.c

@@ -1,7 +1,7 @@
 #include "dirutils.h"
+
 unsigned int
     __posixdir_d_type(struct dirent* d)
     {
       return(d -> d_type);
     }
-

hpath-posix/cbits/dirutils.h

@@ -7,7 +7,9 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 
+
 extern unsigned int
     __posixdir_d_type(struct dirent* d)
     ;
+
 #endif

hpath-posix/hpath-posix.cabal

@@ -1,7 +1,7 @@
 cabal-version:       >=1.10
 
 name:                hpath-posix
-version:             0.13.0
+version:             0.13.2
 synopsis:            Some low-level POSIX glue code, that is not in 'unix'
 homepage:            https://github.com/hasufell/hpath
 bug-reports:         https://github.com/hasufell/hpath/issues
@@ -32,17 +32,9 @@ library
   -- other-extensions:
   c-sources:           cbits/dirutils.c
   build-depends:         base                >= 4.8  && <5
-                       , IfElse
                        , bytestring          >= 0.10
-                       , exceptions          >= 0.10
                        , hpath-filepath      >= 0.10.3
-                       , safe-exceptions     >= 0.1
-                       , streamly            >= 0.7
-                       , streamly-bytestring >= 0.1.0.1
-                       , time                >= 1.8
                        , unix                >= 2.5
-                       , unix-bytestring     >= 0.3
-                       , utf8-string
   if impl(ghc < 8.0)
     build-depends:
         fail >= 4.9

hpath-posix/src/System/Posix/RawFilePath/Directory/Traversals.hs

@@ -10,6 +10,7 @@
 -- Traversal and read operations on directories.
 
 
+{-# LANGUAGE CApiFFI #-}
 {-# LANGUAGE CPP #-}
 {-# LANGUAGE ForeignFunctionInterface #-}
 {-# LANGUAGE OverloadedStrings #-}
@@ -173,11 +174,15 @@ foreign import ccall unsafe "__hscore_d_name"
 foreign import ccall unsafe "__posixdir_d_type"
   c_type :: Ptr CDirent -> IO DirType
 
-foreign import ccall "realpath"
+foreign import capi "stdlib.h realpath"
   c_realpath :: CString -> CString -> IO CString
 
-foreign import ccall unsafe "fdopendir"
+-- Using normal 'ccall' here lead to memory bugs, crashes
-  c_fdopendir :: Posix.Fd -> IO (Ptr ())
+-- and corrupted d_name entries. It appears there are two fdopendirs:
+--   https://opensource.apple.com/source/Libc/Libc-1244.1.7/include/dirent.h.auto.html
+-- The capi call picks the correct one.
+foreign import capi unsafe "dirent.h fdopendir"
+  c_fdopendir :: Posix.Fd -> IO (Ptr CDir)
 
 ----------------------------------------------------------
 -- less dodgy but still lower-level

hpath/src/HPath.hs

@@ -186,11 +186,8 @@ parseRel filepath =
 
 
 
--- | Parses a path, whether it's relative or absolute. Will lose
+-- | Parses a path, whether it's relative or absolute.
--- information on whether it's relative or absolute. If you need to know,
--- reparse it.
 --
--- Filenames must not contain slashes.
 -- Excludes '.' and '..'.
 --
 -- Throws: 'PathParseException'