diff --git a/hnix-store-core/src/System/Nix/Internal/Nar/Effects.hs b/hnix-store-core/src/System/Nix/Internal/Nar/Effects.hs index 7bbf4deb..a2af71e7 100644 --- a/hnix-store-core/src/System/Nix/Internal/Nar/Effects.hs +++ b/hnix-store-core/src/System/Nix/Internal/Nar/Effects.hs @@ -5,6 +5,7 @@ module System.Nix.Internal.Nar.Effects ( NarEffects(..) + , PathType(..) , narEffectsIO ) where @@ -18,9 +19,16 @@ import Data.Int (Int64) import qualified System.Directory as Directory import qualified System.Directory as Directory import qualified System.IO as IO -import System.Posix.Files (createSymbolicLink, fileSize, - getFileStatus, isDirectory, - readSymbolicLink) +import System.Posix.Files (createSymbolicLink, fileSize, readSymbolicLink, + getSymbolicLinkStatus, isRegularFile, isDirectory, isSymbolicLink) + +data PathType = Regular | Directory | Symlink | Unknown deriving Show + +pathTypeFromPosix status + | isRegularFile status = Regular + | isDirectory status = Directory + | isSymbolicLink status = Symlink + | otherwise = Unknown data NarEffects (m :: * -> *) = NarEffects { narReadFile :: FilePath -> m BSL.ByteString @@ -31,8 +39,7 @@ data NarEffects (m :: * -> *) = NarEffects { , narCreateLink :: FilePath -> FilePath -> m () , narGetPerms :: FilePath -> m Directory.Permissions , narSetPerms :: FilePath -> Directory.Permissions -> m () - , narIsDir :: FilePath -> m Bool - , narIsSymLink :: FilePath -> m Bool + , narPathType :: FilePath -> m PathType , narFileSize :: FilePath -> m Int64 , narReadLink :: FilePath -> m FilePath , narDeleteDir :: FilePath -> m () @@ -57,9 +64,8 @@ narEffectsIO = NarEffects { , narCreateLink = \f t -> IO.liftIO $ createSymbolicLink f t , narGetPerms = IO.liftIO . Directory.getPermissions , narSetPerms = \f p -> IO.liftIO $ Directory.setPermissions f p - , narIsDir = \d -> fmap isDirectory $ IO.liftIO (getFileStatus d) - , narIsSymLink = IO.liftIO . Directory.pathIsSymbolicLink - , narFileSize = \n -> fmap (fromIntegral . fileSize) $ IO.liftIO (getFileStatus n) + , narPathType = \f -> fmap pathTypeFromPosix $ IO.liftIO (getSymbolicLinkStatus f) + , narFileSize = \n -> fmap (fromIntegral . fileSize) $ IO.liftIO (getSymbolicLinkStatus n) , narReadLink = IO.liftIO . readSymbolicLink , narDeleteDir = IO.liftIO . Directory.removeDirectoryRecursive , narDeleteFile = IO.liftIO . Directory.removeFile diff --git a/hnix-store-core/src/System/Nix/Internal/Nar/Parser.hs b/hnix-store-core/src/System/Nix/Internal/Nar/Parser.hs index bb596bf8..d06e4df6 100644 --- a/hnix-store-core/src/System/Nix/Internal/Nar/Parser.hs +++ b/hnix-store-core/src/System/Nix/Internal/Nar/Parser.hs @@ -91,10 +91,10 @@ runParser effs (NarParser action) h target = do cleanup :: m () cleanup = do - isDir <- Nar.narIsDir effs target - if isDir - then Nar.narDeleteDir effs target - else Nar.narDeleteFile effs target + pathType <- Nar.narPathType effs target + case pathType of + Nar.Directory -> Nar.narDeleteDir effs target + _ -> Nar.narDeleteFile effs target instance Trans.MonadTrans NarParser where diff --git a/hnix-store-core/src/System/Nix/Internal/Nar/Streamer.hs b/hnix-store-core/src/System/Nix/Internal/Nar/Streamer.hs index 95fd5843..ae554983 100644 --- a/hnix-store-core/src/System/Nix/Internal/Nar/Streamer.hs +++ b/hnix-store-core/src/System/Nix/Internal/Nar/Streamer.hs @@ -4,6 +4,7 @@ {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE LambdaCase #-} module System.Nix.Internal.Nar.Streamer where @@ -28,26 +29,24 @@ import qualified System.Nix.Internal.Nar.Effects as Nar streamNarIO :: forall m.(IO.MonadIO m) => (BS.ByteString -> m ()) + -> (FilePath -> Nar.PathType -> m Bool) -> Nar.NarEffects IO -> FilePath -> m () -streamNarIO yield effs basePath = do +streamNarIO yield filter effs basePath = do yield (str "nix-archive-1") - parens (go basePath) + basePathType <- IO.liftIO $ Nar.narPathType effs basePath + parens (go basePath basePathType) where - go :: FilePath -> m () - go path = do - isDir <- IO.liftIO $ Nar.narIsDir effs path - isSymLink <- IO.liftIO $ Nar.narIsSymLink effs path - let isRegular = not (isDir || isSymLink) - - when isSymLink $ do + go :: FilePath -> Nar.PathType -> m () + go path = \case + Nar.Symlink -> do target <- IO.liftIO $ Nar.narReadLink effs path yield $ strs ["type", "symlink", "target", BSC.pack target] - when isRegular $ do + Nar.Regular -> do isExec <- IO.liftIO $ isExecutable effs path yield $ strs ["type","regular"] when (isExec == Executable) (yield $ strs ["executable", ""]) @@ -56,15 +55,21 @@ streamNarIO yield effs basePath = do yield $ int fSize yieldFile path fSize - when isDir $ do + Nar.Directory -> do fs <- IO.liftIO (Nar.narListDir effs path) yield $ strs ["type", "directory"] forM_ (List.sort fs) $ \f -> do - yield $ str "entry" - parens $ do - let fullName = path f - yield (strs ["name", BSC.pack f, "node"]) - parens (go fullName) + let fullName = path f + pathType <- IO.liftIO $ Nar.narPathType effs fullName + keep <- filter fullName pathType + when keep $ do + yield $ str "entry" + parens $ do + yield (strs ["name", BSC.pack f, "node"]) + parens (go fullName pathType) + + Nar.Unknown -> do + IO.liftIO $ fail $ "Cannot serialise path " ++ path str :: BS.ByteString -> BS.ByteString str t = let len = BS.length t diff --git a/hnix-store-core/src/System/Nix/Nar.hs b/hnix-store-core/src/System/Nix/Nar.hs index d6e004f5..d73561f7 100644 --- a/hnix-store-core/src/System/Nix/Nar.hs +++ b/hnix-store-core/src/System/Nix/Nar.hs @@ -18,6 +18,7 @@ module System.Nix.Nar ( -- * Encoding and Decoding NAR archives buildNarIO , unpackNarIO + , Nar.PathType (..) -- * Experimental , Nar.parseNar @@ -67,7 +68,7 @@ buildNarIO -> IO.Handle -> IO () buildNarIO effs basePath outHandle = do - Nar.streamNarIO (\chunk -> BS.hPut outHandle chunk >> Concurrent.threadDelay 10) effs basePath + Nar.streamNarIO (\chunk -> BS.hPut outHandle chunk >> Concurrent.threadDelay 10) (\p pt -> pure True) effs basePath -- | Read NAR formatted bytes from the @IO.Handle@ and unpack them into diff --git a/hnix-store-core/tests/NarFormat.hs b/hnix-store-core/tests/NarFormat.hs index 1b353dfa..cb20c597 100644 --- a/hnix-store-core/tests/NarFormat.hs +++ b/hnix-store-core/tests/NarFormat.hs @@ -54,7 +54,7 @@ import qualified Text.Printf as Printf import Text.Read (readMaybe) import qualified System.Nix.Internal.Nar.Streamer as Nar -import System.Nix.Nar +import System.Nix.Nar hiding (PathType(..)) diff --git a/hnix-store-remote/hnix-store-remote.cabal b/hnix-store-remote/hnix-store-remote.cabal index 8f7ce895..3a8566d7 100644 --- a/hnix-store-remote/hnix-store-remote.cabal +++ b/hnix-store-remote/hnix-store-remote.cabal @@ -30,22 +30,22 @@ library , System.Nix.Store.Remote.Types , System.Nix.Store.Remote.Util - build-depends: base >=4.10 && <5 - , attoparsec - , bytestring + build-depends: attoparsec + , base >=4.10 && <5 , binary , bytestring , containers , filepath - , text - , unix + , hnix-store-core + , lifted-base + , monad-control + , mtl , network , nix-derivation >= 1.1.1 && <2 - , mtl - , unordered-containers - , filepath + , text , time - , hnix-store-core + , unix + , unordered-containers , vector hs-source-dirs: src default-language: Haskell2010 @@ -55,13 +55,13 @@ test-suite hnix-store-remote-tests if !flag(io-testsuite) buildable: False - build-tool-depends: nix:nix-daemon - ghc-options: -rtsopts -fprof-auto type: exitcode-stdio-1.0 main-is: Driver.hs other-modules: Derivation , NixDaemon + , Spec + , Util hs-source-dirs: tests build-depends: attoparsec @@ -76,6 +76,7 @@ test-suite hnix-store-remote-tests , process , filepath , hspec-expectations-lifted + , quickcheck-text , tasty , tasty-discover , tasty-hspec diff --git a/hnix-store-remote/src/System/Nix/Store/Remote.hs b/hnix-store-remote/src/System/Nix/Store/Remote.hs index 4c9f63c1..06cf02ed 100644 --- a/hnix-store-remote/src/System/Nix/Store/Remote.hs +++ b/hnix-store-remote/src/System/Nix/Store/Remote.hs @@ -9,8 +9,9 @@ {-# LANGUAGE RecordWildCards #-} module System.Nix.Store.Remote ( - addToStore - , addToStoreNar + RemoteStoreT + , System.Nix.Nar.PathType (..) + , addToStore , addTextToStore , addSignatures , addIndirectRoot @@ -38,6 +39,7 @@ module System.Nix.Store.Remote where import Control.Monad (void, unless, when) +import Control.Monad.IO.Class (MonadIO) import Data.ByteString.Lazy (ByteString) import Data.Map.Strict (Map) import Data.Text (Text) @@ -45,7 +47,6 @@ import Data.Text (Text) import Nix.Derivation (Derivation) import System.Nix.Build (BuildMode, BuildResult) import System.Nix.Hash (Digest, NamedAlgo, ValidAlgo, SomeNamedDigest(..)) -import System.Nix.Nar (Nar) import System.Nix.StorePath (StorePath, StorePathName, StorePathSet, StorePathHashAlgo) import System.Nix.StorePathMetadata (StorePathMetadata(..), StorePathTrust(..)) @@ -74,92 +75,41 @@ type CheckSigsFlag = Bool type SubstituteFlag = Bool -- | Pack `FilePath` as `Nar` and add it to the store. -addToStore :: forall a. (ValidAlgo a, NamedAlgo a) +addToStore :: forall a m. (NamedAlgo a, MonadRemoteStore m, MonadIO m) => StorePathName -- ^ Name part of the newly created `StorePath` -> FilePath -- ^ Local `FilePath` to add -> Bool -- ^ Add target directory recursively - -> (FilePath -> Bool) -- ^ Path filter function + -> (FilePath -> System.Nix.Nar.PathType -> m Bool) -- ^ Path filter function -> RepairFlag -- ^ Only used by local store backend - -> MonadStore StorePath -addToStore name pth recursive _pathFilter _repair = do + -> m StorePath +addToStore name pth recursive pathFilter _repair = do - nar :: ByteString <- Control.Monad.IO.Class.liftIO - $ Data.Binary.Put.runPut . System.Nix.Nar.putNar - <$> System.Nix.Nar.localPackNar System.Nix.Nar.narEffectsIO pth + runOpArgsIO AddToStore $ \yield -> do + yield $ Data.ByteString.Lazy.toStrict $ Data.Binary.Put.runPut $ do + putText $ System.Nix.StorePath.unStorePathName name - runOpArgs AddToStore $ do - putText $ System.Nix.StorePath.unStorePathName name + putBool + $ not + $ System.Nix.Hash.algoName @a == "sha256" && recursive - putBool - $ not - $ System.Nix.Hash.algoName @a == "sha256" && recursive + putBool recursive - putBool recursive + putText $ System.Nix.Hash.algoName @a - putText $ System.Nix.Hash.algoName @a - - Data.Binary.Put.putLazyByteString nar + System.Nix.Nar.streamNarIO yield pathFilter System.Nix.Nar.narEffectsIO pth sockGetPath --- | Add `Nar` to the store. --- -addToStoreNar :: StorePathMetadata - -> Nar - -> RepairFlag - -> CheckSigsFlag - -> MonadStore () -addToStoreNar StorePathMetadata{..} nar repair checkSigs = do - -- after the command, protocol asks for data via Read message - -- so we provide it here - let n = Data.Binary.Put.runPut $ System.Nix.Nar.putNar nar - setData n - - void $ runOpArgs AddToStoreNar $ do - putPath path - maybe (putText "") (putPath) deriverPath - let putNarHash :: SomeNamedDigest -> Data.Binary.Put.PutM () - putNarHash (SomeDigest hash) = putByteStringLen - $ Data.ByteString.Lazy.fromStrict - $ Data.Text.Encoding.encodeUtf8 - $ System.Nix.Hash.encodeBase32 hash - - putNarHash narHash - putPaths references - putTime registrationTime - - -- XXX: StorePathMetadata defines this as Maybe - -- `putInt 0` instead of error? - maybe (error "NO NAR BYTES") putInt narBytes - - putBool (trust == BuiltLocally) - - -- XXX: signatures need pubkey from config - putTexts [""] - - maybe - (putText "") - (putText - . Data.Text.Lazy.toStrict - . System.Nix.Store.Remote.Builders.buildContentAddressableAddress - -- this calls for changing the type of addToStoreNar - -- to forall a . (Valid/Named)Algo and a type app - @'System.Nix.Hash.SHA256 - ) - contentAddressableAddress - - putBool repair - putBool (not checkSigs) - -- | Add text to store. -- -- Reference accepts repair but only uses it -- to throw error in case of remote talking to nix-daemon. -addTextToStore :: Text -- ^ Name of the text +addTextToStore :: (MonadIO m, MonadRemoteStore m) + => Text -- ^ Name of the text -> Text -- ^ Actual text to add -> StorePathSet -- ^ Set of `StorePath`s that the added text references -> RepairFlag -- ^ Repair flag, must be `False` in case of remote backend - -> MonadStore StorePath + -> m StorePath addTextToStore name text references' repair = do when repair $ error "repairing is not supported when building through the Nix daemon" runOpArgs AddTextToStore $ do @@ -168,40 +118,43 @@ addTextToStore name text references' repair = do putPaths references' sockGetPath -addSignatures :: StorePath +addSignatures :: (MonadIO m) + => StorePath -> [ByteString] - -> MonadStore () + -> RemoteStoreT m () addSignatures p signatures = do void $ simpleOpArgs AddSignatures $ do putPath p putByteStrings signatures -addIndirectRoot :: StorePath -> MonadStore () +addIndirectRoot :: (MonadIO m) => StorePath -> RemoteStoreT m () addIndirectRoot pn = do void $ simpleOpArgs AddIndirectRoot $ putPath pn -- | Add temporary garbage collector root. -- -- This root is removed as soon as the client exits. -addTempRoot :: StorePath -> MonadStore () +addTempRoot :: (MonadIO m) => StorePath -> RemoteStoreT m () addTempRoot pn = do void $ simpleOpArgs AddTempRoot $ putPath pn -- | Build paths if they are an actual derivations. -- -- If derivation output paths are already valid, do nothing. -buildPaths :: StorePathSet +buildPaths :: (MonadIO m) + => StorePathSet -> BuildMode - -> MonadStore () + -> RemoteStoreT m () buildPaths ps bm = do void $ simpleOpArgs BuildPaths $ do putPaths ps putInt $ fromEnum bm -buildDerivation :: StorePath +buildDerivation :: (MonadIO m) + => StorePath -> Derivation StorePath Text -> BuildMode - -> MonadStore BuildResult + -> RemoteStoreT m BuildResult buildDerivation p drv buildMode = do runOpArgs BuildDerivation $ do putPath p @@ -210,62 +163,64 @@ buildDerivation p drv buildMode = do -- XXX: reason for this is unknown -- but without it protocol just hangs waiting for -- more data. Needs investigation - putInt 0 + putInt (0 :: Int) res <- getSocketIncremental $ getBuildResult return res -ensurePath :: StorePath -> MonadStore () +ensurePath :: (MonadIO m) => StorePath -> RemoteStoreT m () ensurePath pn = do void $ simpleOpArgs EnsurePath $ putPath pn -- | Find garbage collector roots. -findRoots :: MonadStore (Map ByteString StorePath) +findRoots :: (MonadIO m) => RemoteStoreT m (Map ByteString StorePath) findRoots = do runOp FindRoots sd <- getStoreDir res <- getSocketIncremental $ getMany - $ (,) <$> (Data.ByteString.Lazy.fromStrict <$> getByteStringLen) + $ (,) <$> (Data.ByteString.Lazy.fromStrict <$> getByteStringLen) <*> getPath sd r <- catRights res return $ Data.Map.Strict.fromList r where - catRights :: [(a, Either String b)] -> MonadStore [(a, b)] + catRights :: (MonadIO m) => [(a, Either String b)] -> RemoteStoreT m [(a, b)] catRights = mapM ex - ex :: (a, Either [Char] b) -> MonadStore (a, b) + ex :: (MonadIO m) => (a, Either [Char] b) -> RemoteStoreT m (a, b) ex (x, Right y) = return (x, y) ex (_x , Left e) = error $ "Unable to decode root: " ++ e -isValidPathUncached :: StorePath -> MonadStore Bool +isValidPathUncached :: (MonadIO m) => StorePath -> RemoteStoreT m Bool isValidPathUncached p = do simpleOpArgs IsValidPath $ putPath p -- | Query valid paths from set, optionally try to use substitutes. -queryValidPaths :: StorePathSet -- ^ Set of `StorePath`s to query +queryValidPaths :: (MonadIO m) + => StorePathSet -- ^ Set of `StorePath`s to query -> SubstituteFlag -- ^ Try substituting missing paths when `True` - -> MonadStore StorePathSet + -> RemoteStoreT m StorePathSet queryValidPaths ps substitute = do runOpArgs QueryValidPaths $ do putPaths ps putBool substitute sockGetPaths -queryAllValidPaths :: MonadStore StorePathSet +queryAllValidPaths :: (MonadIO m) => RemoteStoreT m StorePathSet queryAllValidPaths = do runOp QueryAllValidPaths sockGetPaths -querySubstitutablePaths :: StorePathSet -> MonadStore StorePathSet +querySubstitutablePaths :: (MonadIO m) => StorePathSet -> RemoteStoreT m StorePathSet querySubstitutablePaths ps = do runOpArgs QuerySubstitutablePaths $ do putPaths ps sockGetPaths -queryPathInfoUncached :: StorePath - -> MonadStore StorePathMetadata +queryPathInfoUncached :: (MonadIO m) + => StorePath + -> RemoteStoreT m StorePathMetadata queryPathInfoUncached path = do runOpArgs QueryPathInfo $ do putPath path @@ -302,31 +257,31 @@ queryPathInfoUncached path = do return $ StorePathMetadata {..} -queryReferrers :: StorePath -> MonadStore StorePathSet +queryReferrers :: (MonadIO m) => StorePath -> RemoteStoreT m StorePathSet queryReferrers p = do runOpArgs QueryReferrers $ do putPath p sockGetPaths -queryValidDerivers :: StorePath -> MonadStore StorePathSet +queryValidDerivers :: (MonadIO m) => StorePath -> RemoteStoreT m StorePathSet queryValidDerivers p = do runOpArgs QueryValidDerivers $ do putPath p sockGetPaths -queryDerivationOutputs :: StorePath -> MonadStore StorePathSet +queryDerivationOutputs :: (MonadIO m) => StorePath -> RemoteStoreT m StorePathSet queryDerivationOutputs p = do runOpArgs QueryDerivationOutputs $ putPath p sockGetPaths -queryDerivationOutputNames :: StorePath -> MonadStore StorePathSet +queryDerivationOutputNames :: (MonadIO m) => StorePath -> RemoteStoreT m StorePathSet queryDerivationOutputNames p = do runOpArgs QueryDerivationOutputNames $ putPath p sockGetPaths -queryPathFromHashPart :: Digest StorePathHashAlgo -> MonadStore StorePath +queryPathFromHashPart :: (MonadIO m) => Digest StorePathHashAlgo -> RemoteStoreT m StorePath queryPathFromHashPart storePathHash = do runOpArgs QueryPathFromHashPart $ putByteStringLen @@ -335,12 +290,13 @@ queryPathFromHashPart storePathHash = do $ System.Nix.Hash.encodeBase32 storePathHash sockGetPath -queryMissing :: StorePathSet - -> MonadStore ( StorePathSet -- Paths that will be built - , StorePathSet -- Paths that have substitutes - , StorePathSet -- Unknown paths - , Integer -- Download size - , Integer) -- Nar size? +queryMissing :: (MonadIO m) + => StorePathSet + -> RemoteStoreT m ( StorePathSet -- Paths that will be built + , StorePathSet -- Paths that have substitutes + , StorePathSet -- Unknown paths + , Integer -- Download size + , Integer) -- Nar size? queryMissing ps = do runOpArgs QueryMissing $ do putPaths ps @@ -352,14 +308,14 @@ queryMissing ps = do narSize' <- sockGetInt return (willBuild, willSubstitute, unknown, downloadSize', narSize') -optimiseStore :: MonadStore () +optimiseStore :: (MonadIO m) => RemoteStoreT m () optimiseStore = void $ simpleOp OptimiseStore -syncWithGC :: MonadStore () +syncWithGC :: (MonadIO m) => RemoteStoreT m () syncWithGC = void $ simpleOp SyncWithGC -- returns True on errors -verifyStore :: CheckFlag -> RepairFlag -> MonadStore Bool +verifyStore :: (MonadIO m) => CheckFlag -> RepairFlag -> RemoteStoreT m Bool verifyStore check repair = simpleOpArgs VerifyStore $ do putBool check putBool repair diff --git a/hnix-store-remote/src/System/Nix/Store/Remote/Logger.hs b/hnix-store-remote/src/System/Nix/Store/Remote/Logger.hs index f4bf690c..45b5fdbb 100644 --- a/hnix-store-remote/src/System/Nix/Store/Remote/Logger.hs +++ b/hnix-store-remote/src/System/Nix/Store/Remote/Logger.hs @@ -1,4 +1,5 @@ {-# LANGUAGE RankNTypes #-} +{-# LANGUAGE ScopedTypeVariables #-} module System.Nix.Store.Remote.Logger ( Logger(..) , Field(..) @@ -30,16 +31,16 @@ controlParser = do 0x52534c54 -> Result <$> getInt <*> getInt <*> getFields x -> fail $ "Invalid control message received:" ++ show x -processOutput :: MonadStore [Logger] +processOutput :: forall m. (MonadRemoteStore m, MonadIO m) => m [Logger] processOutput = go decoder where decoder = runGetIncremental controlParser - go :: Decoder Logger -> MonadStore [Logger] + go :: Decoder Logger -> m [Logger] go (Done _leftover _consumed ctrl) = do case ctrl of e@(Error _ _) -> return [e] Last -> return [Last] Read _n -> do - (mdata, _) <- get + mdata <- getData case mdata of Nothing -> throwError "No data to read provided" Just part -> do @@ -55,7 +56,7 @@ processOutput = go decoder next <- go decoder return $ x:next go (Partial k) = do - soc <- storeSocket <$> ask + soc <- getSocket chunk <- liftIO (Just <$> recv soc 8) go (k chunk) diff --git a/hnix-store-remote/src/System/Nix/Store/Remote/Protocol.hs b/hnix-store-remote/src/System/Nix/Store/Remote/Protocol.hs index 52f0dcb6..fcae86d1 100644 --- a/hnix-store-remote/src/System/Nix/Store/Remote/Protocol.hs +++ b/hnix-store-remote/src/System/Nix/Store/Remote/Protocol.hs @@ -1,12 +1,14 @@ {-# LANGUAGE DataKinds #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} +{-# LANGUAGE FlexibleContexts #-} module System.Nix.Store.Remote.Protocol ( WorkerOp(..) , simpleOp , simpleOpArgs , runOp , runOpArgs + , runOpArgsIO , runStore , runStoreOpts) where @@ -14,15 +16,17 @@ import Control.Exception (bracket) import Control.Monad.Except import Control.Monad.Reader import Control.Monad.State +import Control.Monad.Trans.Control (MonadBaseControl, liftBaseOp) import Data.Binary.Get import Data.Binary.Put +import qualified Data.ByteString import qualified Data.ByteString.Char8 import qualified Data.ByteString.Lazy import Network.Socket (SockAddr(SockAddrUnix)) import qualified Network.Socket -import Network.Socket.ByteString (recv) +import Network.Socket.ByteString (recv, sendAll) import System.Nix.Store.Remote.Binary import System.Nix.Store.Remote.Logger @@ -112,50 +116,58 @@ opNum AddToStoreNar = 39 opNum QueryMissing = 40 -simpleOp :: WorkerOp -> MonadStore Bool +simpleOp :: (MonadIO m) => WorkerOp -> RemoteStoreT m Bool simpleOp op = do simpleOpArgs op $ return () -simpleOpArgs :: WorkerOp -> Put -> MonadStore Bool +simpleOpArgs :: (MonadIO m) => WorkerOp -> Put -> RemoteStoreT m Bool simpleOpArgs op args = do runOpArgs op args err <- gotError case err of True -> do - Error _num msg <- head <$> getError - throwError $ Data.ByteString.Char8.unpack msg + err <- head <$> getError + case err of + Error _num msg -> throwError $ Data.ByteString.Char8.unpack msg + _ -> throwError $ "Well, it should really be an error by now" False -> do sockGetBool -runOp :: WorkerOp -> MonadStore () +runOp :: (MonadIO m) => WorkerOp -> RemoteStoreT m () runOp op = runOpArgs op $ return () -runOpArgs :: WorkerOp -> Put -> MonadStore () -runOpArgs op args = do +runOpArgs :: (MonadIO m, MonadRemoteStore m) => WorkerOp -> Put -> m () +runOpArgs op args = runOpArgsIO op (\encode -> encode $ Data.ByteString.Lazy.toStrict $ runPut args) - -- Temporary hack for printing the messages destined for nix-daemon socket - when False $ - liftIO $ Data.ByteString.Lazy.writeFile "mytestfile2" $ runPut $ do - putInt $ opNum op - args +runOpArgsIO + :: forall m. (MonadIO m, MonadRemoteStore m) + => WorkerOp + -> ((Data.ByteString.ByteString -> m ()) -> m ()) + -> m () +runOpArgsIO op encoder = do sockPut $ do putInt $ opNum op - args + + soc <- getSocket + encoder (liftIO . sendAll soc) out <- processOutput - modify (\(a, b) -> (a, b++out)) + setLog . (++out) =<< getLog err <- gotError when err $ do - Error _num msg <- head <$> getError - throwError $ Data.ByteString.Char8.unpack msg + err <- head <$> getError + case err of + Error _num msg -> throwError $ Data.ByteString.Char8.unpack msg + _ -> throwError $ "Well, it should really be an error by now" + -runStore :: MonadStore a -> IO (Either String a, [Logger]) +runStore :: (MonadIO m, MonadBaseControl IO m) => RemoteStoreT m a -> m (Either String a, [Logger]) runStore = runStoreOpts defaultSockPath "/nix/store" -runStoreOpts :: FilePath -> FilePath -> MonadStore a -> IO (Either String a, [Logger]) +runStoreOpts :: (MonadIO m, MonadBaseControl IO m) => FilePath -> FilePath -> RemoteStoreT m a -> m (Either String a, [Logger]) runStoreOpts sockPath storeRootDir code = do - bracket (open sockPath) (Network.Socket.close . storeSocket) run + liftBaseOp (bracket (open sockPath) (Network.Socket.close . storeSocket)) run where open path = do soc <- @@ -168,9 +180,10 @@ runStoreOpts sockPath storeRootDir code = do return $ StoreConfig { storeSocket = soc , storeDir = storeRootDir } + greet :: MonadIO m => RemoteStoreT m [Logger] greet = do sockPut $ putInt workerMagic1 - soc <- storeSocket <$> ask + soc <- storeSocket <$> RemoteStore ask vermagic <- liftIO $ recv soc 16 let (magic2, _daemonProtoVersion) = flip runGet (Data.ByteString.Lazy.fromStrict vermagic) @@ -184,8 +197,9 @@ runStoreOpts sockPath storeRootDir code = do processOutput - run sock = - fmap (\(res, (_data, logs)) -> (res, logs)) - $ flip runReaderT sock - $ flip runStateT (Nothing, []) - $ runExceptT (greet >> code) + run config = + fmap (\(res, state) -> (res, logs state)) + $ flip runReaderT config + $ flip runStateT (StoreState [] Nothing) + $ runExceptT + $ unStore (greet >> code) diff --git a/hnix-store-remote/src/System/Nix/Store/Remote/Types.hs b/hnix-store-remote/src/System/Nix/Store/Remote/Types.hs index 14a33add..85e9c196 100644 --- a/hnix-store-remote/src/System/Nix/Store/Remote/Types.hs +++ b/hnix-store-remote/src/System/Nix/Store/Remote/Types.hs @@ -2,34 +2,61 @@ {-# LANGUAGE KindSignatures #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} +-- XXX (layus 2020-11) Are all of these needed ? +{-# LANGUAGE DefaultSignatures #-} +{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE TypeSynonymInstances #-} +{-# LANGUAGE MultiParamTypeClasses #-} + module System.Nix.Store.Remote.Types ( - MonadStore + RemoteStoreT(..) + , MonadRemoteStore(..) , StoreConfig(..) + , StoreState(..) , Logger(..) , Field(..) - , getStoreDir - , getLog - , flushLog - , gotError - , getError - , setData - , clearData ) where import Data.ByteString (ByteString) import qualified Data.ByteString.Lazy as BSL import Network.Socket (Socket) +import Control.Applicative (Alternative) import Control.Monad.Except import Control.Monad.Reader import Control.Monad.State +import Control.Monad.Fail ( MonadFail ) data StoreConfig = StoreConfig { storeDir :: FilePath , storeSocket :: Socket } -type MonadStore a = ExceptT String (StateT (Maybe BSL.ByteString, [Logger]) (ReaderT StoreConfig IO)) a +data StoreState = StoreState { + logs :: [Logger] + , mData :: Maybe BSL.ByteString + } + +newtype RemoteStoreT m a = RemoteStore { + unStore :: ExceptT String (StateT StoreState (ReaderT StoreConfig m)) a + } deriving + ( Functor + , Applicative + , Alternative + , Monad + --, MonadReader StoreConfig -- Avoid making the internal state explicit + --, MonadState StoreState -- Avoid making the internal state explicit + , MonadFail + , MonadError String + , MonadIO + ) + +instance MonadTrans RemoteStoreT where + lift = RemoteStore . lift . lift . lift + type ActivityID = Int type ActivityParentID = Int @@ -55,23 +82,69 @@ isError :: Logger -> Bool isError (Error _ _) = True isError _ = False -gotError :: MonadStore Bool -gotError = any isError . snd <$> get +class (Monad m, MonadError String m) => MonadRemoteStore m where + gotError :: m Bool + default gotError :: (MonadTrans t, MonadRemoteStore m', m ~ t m') => m Bool + gotError = lift gotError + + getError :: m [Logger] + default getError :: (MonadTrans t, MonadRemoteStore m', m ~ t m') => m [Logger] + getError = lift getError + + getLog :: m [Logger] + default getLog :: (MonadTrans t, MonadRemoteStore m', m ~ t m') => m [Logger] + getLog = lift getLog + + setLog :: [Logger] -> m () + default setLog :: (MonadTrans t, MonadRemoteStore m', m ~ t m') => [Logger] -> m () + setLog = lift . setLog + + flushLog :: m () + default flushLog :: (MonadTrans t, MonadRemoteStore m', m ~ t m') => m () + flushLog = lift flushLog + + setData :: BSL.ByteString -> m () + default setData :: (MonadTrans t, MonadRemoteStore m', m ~ t m') => BSL.ByteString -> m () + setData = lift . setData + + getData :: m (Maybe BSL.ByteString) + default getData :: (MonadTrans t, MonadRemoteStore m', m ~ t m') => m (Maybe BSL.ByteString) + getData = lift getData + + clearData :: m () + default clearData :: (MonadTrans t, MonadRemoteStore m', m ~ t m') => m () + clearData = lift clearData + + getStoreDir :: m FilePath + default getStoreDir :: (MonadTrans t, MonadRemoteStore m', m ~ t m') => m FilePath + getStoreDir = lift getStoreDir + + getSocket :: m Socket + default getSocket :: (MonadTrans t, MonadRemoteStore m', m ~ t m') => m Socket + getSocket = lift getSocket -getError :: MonadStore [Logger] -getError = filter isError . snd <$> get +instance (MonadRemoteStore m) => MonadRemoteStore (StateT s m) +instance (MonadRemoteStore m) => MonadRemoteStore (ReaderT r m) +-- instance (MonadRemoteStore m) => MonadError String m -getLog :: MonadStore [Logger] -getLog = snd <$> get +instance (Monad m) => MonadRemoteStore (RemoteStoreT m) where + gotError = any isError . logs <$> RemoteStore get + + getError = filter isError . logs <$> RemoteStore get + + getLog = logs <$> RemoteStore get + + flushLog = RemoteStore $ modify (\s -> s { logs = [] }) -flushLog :: MonadStore () -flushLog = modify (\(a, _b) -> (a, [])) + setLog logs = RemoteStore $ modify (\s -> s { logs = logs }) + + setData x = RemoteStore $ modify (\s -> s { mData = Just x }) + + clearData = RemoteStore $ modify (\s -> s { mData = Nothing }) -setData :: BSL.ByteString -> MonadStore () -setData x = modify (\(_, b) -> (Just x, b)) + getData = RemoteStore $ gets mData + + getStoreDir = storeDir <$> RemoteStore ask -clearData :: MonadStore () -clearData = modify (\(_, b) -> (Nothing, b)) + getSocket = storeSocket <$> RemoteStore ask -getStoreDir :: MonadStore FilePath -getStoreDir = storeDir <$> ask diff --git a/hnix-store-remote/src/System/Nix/Store/Remote/Util.hs b/hnix-store-remote/src/System/Nix/Store/Remote/Util.hs index afe0cf77..d8db78f2 100644 --- a/hnix-store-remote/src/System/Nix/Store/Remote/Util.hs +++ b/hnix-store-remote/src/System/Nix/Store/Remote/Util.hs @@ -11,6 +11,9 @@ import Data.Binary.Get import Data.Binary.Put import Data.Text (Text) import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.Encoding as TL import Data.Time import Data.Time.Clock.POSIX import Data.ByteString (ByteString) @@ -41,35 +44,35 @@ genericIncremental getsome parser = go decoder go (Fail _leftover _consumed msg) = do error msg -getSocketIncremental :: Get a -> MonadStore a +getSocketIncremental :: (MonadIO m, MonadRemoteStore m) => Get a -> m a getSocketIncremental = genericIncremental sockGet8 where - sockGet8 :: MonadStore (Maybe BSC.ByteString) + sockGet8 :: (MonadIO m, MonadRemoteStore m) => m (Maybe BSC.ByteString) sockGet8 = do - soc <- storeSocket <$> ask + soc <- getSocket liftIO $ Just <$> recv soc 8 -sockPut :: Put -> MonadStore () +sockPut :: (MonadIO m, MonadRemoteStore m) => Put -> m () sockPut p = do - soc <- storeSocket <$> ask + soc <- getSocket liftIO $ sendAll soc $ BSL.toStrict $ runPut p -sockGet :: Get a -> MonadStore a +sockGet :: (MonadIO m, MonadRemoteStore m) => Get a -> m a sockGet = getSocketIncremental -sockGetInt :: Integral a => MonadStore a +sockGetInt :: (MonadIO m, MonadRemoteStore m) => Integral a => m a sockGetInt = getSocketIncremental getInt -sockGetBool :: MonadStore Bool +sockGetBool :: (MonadIO m, MonadRemoteStore m) => m Bool sockGetBool = (== (1 :: Int)) <$> sockGetInt -sockGetStr :: MonadStore ByteString +sockGetStr :: (MonadIO m, MonadRemoteStore m) => m ByteString sockGetStr = getSocketIncremental getByteStringLen -sockGetStrings :: MonadStore [ByteString] +sockGetStrings :: (MonadIO m, MonadRemoteStore m) => m [ByteString] sockGetStrings = getSocketIncremental getByteStrings -sockGetPath :: MonadStore StorePath +sockGetPath :: (MonadIO m, MonadRemoteStore m) => m StorePath sockGetPath = do sd <- getStoreDir pth <- getSocketIncremental (getPath sd) @@ -77,7 +80,7 @@ sockGetPath = do Left e -> throwError e Right x -> return x -sockGetPathMay :: MonadStore (Maybe StorePath) +sockGetPathMay :: (MonadIO m, MonadRemoteStore m) => m (Maybe StorePath) sockGetPathMay = do sd <- getStoreDir pth <- getSocketIncremental (getPath sd) @@ -85,19 +88,22 @@ sockGetPathMay = do Left _e -> Nothing Right x -> Just x -sockGetPaths :: MonadStore StorePathSet +sockGetPaths :: (MonadIO m, MonadRemoteStore m) => m StorePathSet sockGetPaths = do sd <- getStoreDir getSocketIncremental (getPaths sd) bsToText :: ByteString -> Text -bsToText = T.pack . BSC.unpack +bsToText = T.decodeUtf8 + +textToBS :: Text -> ByteString +textToBS = T.encodeUtf8 bslToText :: BSL.ByteString -> Text -bslToText = T.pack . BSC.unpack . BSL.toStrict +bslToText = TL.toStrict . TL.decodeUtf8 textToBSL :: Text -> BSL.ByteString -textToBSL = BSL.fromStrict . BSC.pack . T.unpack +textToBSL = TL.encodeUtf8 . TL.fromStrict putText :: Text -> Put putText = putByteStringLen . textToBSL diff --git a/hnix-store-remote/tests/Derivation.hs b/hnix-store-remote/tests/Derivation.hs index 5cfd2ce2..5a004f8b 100644 --- a/hnix-store-remote/tests/Derivation.hs +++ b/hnix-store-remote/tests/Derivation.hs @@ -41,7 +41,7 @@ withBash action = do Nothing -> error "No bash executable found" Just fp -> do let Right n = System.Nix.StorePath.makeStorePathName "bash" - path <- addToStore @SHA256 n fp False (pure True) False + path <- addToStore @SHA256 n fp False (\p pt -> pure True) False action path withBuildScript action = do diff --git a/hnix-store-remote/tests/Driver.hs b/hnix-store-remote/tests/Driver.hs index 604f0c43..f3d4fd5f 100644 --- a/hnix-store-remote/tests/Driver.hs +++ b/hnix-store-remote/tests/Driver.hs @@ -1,8 +1,9 @@ import Test.Tasty.Hspec import NixDaemon +import qualified Spec -- we run remote tests in -- Linux namespaces to avoid interacting with systems store main = do enterNamespaces - hspec spec_protocol + Spec.main diff --git a/hnix-store-remote/tests/NixDaemon.hs b/hnix-store-remote/tests/NixDaemon.hs index caca4728..32a1a9b5 100644 --- a/hnix-store-remote/tests/NixDaemon.hs +++ b/hnix-store-remote/tests/NixDaemon.hs @@ -106,7 +106,7 @@ accepted connection from pid 22959, user root (trusted) error: changing ownership of path '/run/user/1000/test-nix-store-06b0d249e5616122/store': Invalid argument -} -startDaemon :: FilePath -> IO (P.ProcessHandle, MonadStore a -> IO (Either String a, [Logger])) +startDaemon :: FilePath -> IO (P.ProcessHandle, RemoteStoreT IO a -> IO (Either String a, [Logger])) startDaemon fp = do writeConf (fp "etc" "nix.conf") p <- createProcessEnv fp "nix-daemon" [] @@ -155,7 +155,7 @@ withPath action = do -- | dummy path, adds /dummpy with "Hello World" contents dummy = do let Right n = makeStorePathName "dummy" - res <- addToStore @SHA256 n "dummy" False (pure True) False + res <- addToStore @SHA256 n "dummy" False (\p pt -> pure True) False return res invalidPath :: StorePath @@ -163,39 +163,6 @@ invalidPath = let Right n = makeStorePathName "invalid" in StorePath (hash "invalid") n "no_such_root" -withNar act = do - nar <- liftIO $ localPackNar narEffectsIO "dummy" - now <- liftIO $ getCurrentTime - - let narContents = runPut $ putNar nar - narHash = hashLazy @SHA256 narContents - -- narSize vs narBytes - narBytes = BSL.length narContents - - deriver <- addTextToStore "some-deriver" "" (HS.fromList []) False - - sd <- getStoreDir - let Right n = makeStorePathName "nar-path" - path = makeFixedOutputPath sd False narHash n - - addTempRoot path - - let vp = VP.StorePathMetadata - { VP.path = path - , VP.deriverPath = Just deriver - , VP.narHash = SomeDigest narHash - , VP.references = HS.empty - , VP.registrationTime = now - , VP.narBytes = Just $ fromIntegral narBytes - , VP.trust = VP.BuiltLocally - , VP.sigs = S.empty -- [] - , VP.contentAddressableAddress = Nothing - } - - addToStoreNar vp nar False False - - act path - withBuilder action = do path <- addTextToStore "builder" builderSh (HS.fromList []) False action path @@ -235,7 +202,7 @@ spec_protocol = Hspec.around withNixDaemon $ do itRights "non-empty query" $ withPath $ \path -> queryAllValidPaths `shouldReturn` (HS.fromList [path]) context "queryPathInfoUncached" $ do - itRights "queries path info" $ withPath $ queryPathInfoUncached @SHA256 + itRights "queries path info" $ withPath $ queryPathInfoUncached context "ensurePath" $ do itRights "simple ensure" $ withPath $ ensurePath @@ -275,17 +242,11 @@ spec_protocol = Hspec.around withNixDaemon $ do let pathSet = HS.fromList [path] queryMissing pathSet `shouldReturn` (HS.empty, HS.empty, HS.empty, 0, 0) - context "addToStoreNar" $ do - itRights "simple" $ withNar $ const return () - itRights "valid" $ withNar $ \narPath -> do - liftIO $ print narPath - (isValidPathUncached narPath) `shouldReturn` True - context "addToStore" $ do itRights "adds file to store" $ do fp <- liftIO $ writeSystemTempFile "addition" "lal" let Right n = makeStorePathName "tmp-addition" - res <- addToStore @SHA256 n fp False (pure True) False + res <- addToStore @SHA256 n fp False (\p pt -> pure True) False liftIO $ print res context "with dummy" $ do diff --git a/hnix-store-remote/tests/Spec.hs b/hnix-store-remote/tests/Spec.hs new file mode 100644 index 00000000..203ed407 --- /dev/null +++ b/hnix-store-remote/tests/Spec.hs @@ -0,0 +1 @@ +{-# OPTIONS_GHC -F -pgmF tasty-discover -optF --generated-module=Spec #-} diff --git a/hnix-store-remote/tests/Util.hs b/hnix-store-remote/tests/Util.hs new file mode 100644 index 00000000..5cfaa43b --- /dev/null +++ b/hnix-store-remote/tests/Util.hs @@ -0,0 +1,13 @@ + +module Util where + +import Test.Tasty.QuickCheck +import Data.Text.Arbitrary + +import System.Nix.Store.Remote.Util + +prop_TextToBSLRoundtrip x = + bslToText (textToBSL x) === x + +prop_TextToBSRoundtrip x = + bsToText (textToBS x) === x