diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2870d3076..5aae5cf86b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -114,7 +114,7 @@ jobs: - if: needs.pre_job.outputs.should_skip_ghcide != 'true' && matrix.test name: Test ghcide # run the tests without parallelism to avoid running out of memory - run: cabal test ghcide-tests || cabal test ghcide-tests + run: cabal test ghcide-tests - if: matrix.test name: Test hls-plugin-api diff --git a/cabal.project b/cabal.project index 8d8bd080af..d83e432492 100644 --- a/cabal.project +++ b/cabal.project @@ -56,3 +56,9 @@ if impl(ghc >= 9.11) allow-newer: cabal-install-parsers:base, cabal-install-parsers:time, + +source-repository-package + type: git + location: https://github.com/soulomoon/lsp.git + tag: 640c7c755bf16128e3cb19c257688aa3305ff9f5 + subdir: lsp lsp-types lsp-test diff --git a/ghcide/ghcide.cabal b/ghcide/ghcide.cabal index 7dd12f9fef..359b742771 100644 --- a/ghcide/ghcide.cabal +++ b/ghcide/ghcide.cabal @@ -142,7 +142,6 @@ library Development.IDE.Core.Shake Development.IDE.Core.Tracing Development.IDE.Core.UseStale - Development.IDE.Core.WorkerThread Development.IDE.GHC.Compat Development.IDE.GHC.Compat.Core Development.IDE.GHC.Compat.CmdLine @@ -179,7 +178,9 @@ library Development.IDE.Plugin.Test Development.IDE.Plugin.TypeLenses Development.IDE.Session + Development.IDE.Session.Dependency Development.IDE.Session.Diagnostics + Development.IDE.Session.Ghc Development.IDE.Session.Implicit Development.IDE.Spans.AtPoint Development.IDE.Spans.Common @@ -202,6 +203,7 @@ library Development.IDE.GHC.CPP Development.IDE.GHC.Warnings Development.IDE.Types.Action + Development.IDE.Session.OrderedSet if flag(pedantic) ghc-options: diff --git a/ghcide/session-loader/Development/IDE/Session.hs b/ghcide/session-loader/Development/IDE/Session.hs index dde1cfdea5..5de220dd39 100644 --- a/ghcide/session-loader/Development/IDE/Session.hs +++ b/ghcide/session-loader/Development/IDE/Session.hs @@ -1,4 +1,3 @@ -{-# LANGUAGE CPP #-} {-# LANGUAGE TypeFamilies #-} {-| @@ -25,49 +24,38 @@ import Control.Exception.Safe as Safe import Control.Monad import Control.Monad.Extra as Extra import Control.Monad.IO.Class +import Control.Monad.Trans.Maybe (MaybeT (MaybeT, runMaybeT)) import qualified Crypto.Hash.SHA1 as H -import Data.Aeson hiding (Error) -import Data.Bifunctor +import Data.Aeson hiding (Error, Key) import qualified Data.ByteString.Base16 as B16 import qualified Data.ByteString.Char8 as B import Data.Default -import Data.Either.Extra -import Data.Function import Data.Hashable hiding (hash) import qualified Data.HashMap.Strict as HM -import Data.IORef import Data.List import Data.List.Extra as L -import Data.List.NonEmpty (NonEmpty (..)) -import qualified Data.List.NonEmpty as NE import qualified Data.Map.Strict as Map import Data.Maybe import Data.Proxy import qualified Data.Text as T -import Data.Time.Clock import Data.Version import Development.IDE.Core.RuleTypes import Development.IDE.Core.Shake hiding (Log, knownTargets, withHieDb) import qualified Development.IDE.GHC.Compat as Compat -import Development.IDE.GHC.Compat.CmdLine import Development.IDE.GHC.Compat.Core hiding (Target, TargetFile, TargetModule, Var, Warning, getOptions) -import qualified Development.IDE.GHC.Compat.Core as GHC import Development.IDE.GHC.Compat.Env hiding (Logger) -import Development.IDE.GHC.Compat.Units (UnitId) import Development.IDE.GHC.Util -import Development.IDE.Graph (Action) +import Development.IDE.Graph (Action, Key) import qualified Development.IDE.Session.Implicit as GhcIde import Development.IDE.Types.Diagnostics import Development.IDE.Types.Exports -import Development.IDE.Types.HscEnvEq (HscEnvEq, newHscEnvEq) +import Development.IDE.Types.HscEnvEq (HscEnvEq) import Development.IDE.Types.Location import Development.IDE.Types.Options -import GHC.ResponseFile import qualified HIE.Bios as HieBios -import qualified HIE.Bios.Cradle.Utils as HieBios import HIE.Bios.Environment hiding (getCacheDir) import HIE.Bios.Types hiding (Log) import qualified HIE.Bios.Types as HieBios @@ -78,7 +66,8 @@ import Ide.Logger (Pretty (pretty), nest, toCologActionWithPrio, vcat, viaShow, (<+>)) -import Ide.Types (SessionLoadingPreferenceConfig (..), +import Ide.Types (Config, + SessionLoadingPreferenceConfig (..), sessionLoading) import Language.LSP.Protocol.Message import Language.LSP.Server @@ -92,26 +81,19 @@ import Data.Void import Control.Concurrent.STM.Stats (atomically, modifyTVar', readTVar, writeTVar) -import Control.Concurrent.STM.TQueue -import Control.DeepSeq -import Control.Exception (evaluate) -import Control.Monad.IO.Unlift (MonadUnliftIO) import Control.Monad.Trans.Cont (ContT (ContT, runContT)) import Data.Foldable (for_) import Data.HashMap.Strict (HashMap) import Data.HashSet (HashSet) import qualified Data.HashSet as Set -import qualified Data.Set as OS import Database.SQLite.Simple import Development.IDE.Core.Tracing (withTrace) -import Development.IDE.Core.WorkerThread (awaitRunInThread, - withWorkerQueue) -import qualified Development.IDE.GHC.Compat.Util as Compat +import Development.IDE.Session.Dependency import Development.IDE.Session.Diagnostics (renderCradleError) +import Development.IDE.Session.Ghc hiding (Log) import Development.IDE.Types.Shake (WithHieDb, WithHieDbShield (..), toNoFileKey) -import GHC.Data.Graph.Directed import HieDb.Create import HieDb.Types import Ide.PluginUtils (toAbsolute) @@ -119,15 +101,14 @@ import qualified System.Random as Random import System.Random (RandomGen) import Text.ParserCombinators.ReadP (readP_to_S) -import GHC.Driver.Env (hsc_all_home_unit_ids) -import GHC.Driver.Errors.Types -import GHC.Types.Error (errMsgDiagnostic, - singleMessage) -import GHC.Unit.State - -#if MIN_VERSION_ghc(9,13,0) -import GHC.Driver.Make (checkHomeUnitsClosed) -#endif +import Control.Concurrent.STM (STM, TVar) +import qualified Control.Monad.STM as STM +import Control.Monad.Trans.Reader +import qualified Development.IDE.Session.Ghc as Ghc +import qualified Development.IDE.Session.OrderedSet as S +import Development.IDE.WorkerThread +import qualified Focus +import qualified StmContainers.Map as STM data Log = LogSettingInitialDynFlags @@ -137,22 +118,34 @@ data Log | LogHieDbRetriesExhausted !Int !Int !Int !SomeException | LogHieDbWriterThreadSQLiteError !SQLError | LogHieDbWriterThreadException !SomeException - | LogInterfaceFilesCacheDir !FilePath | LogKnownFilesUpdated !(HashMap Target (HashSet NormalizedFilePath)) - | LogMakingNewHscEnv ![UnitId] - | LogDLLLoadError !String | LogCradlePath !FilePath | LogCradleNotFound !FilePath | LogSessionLoadingResult !(Either [CradleError] (ComponentOptions, FilePath, String)) | LogCradle !(Cradle Void) | LogNoneCradleFound FilePath - | LogNewComponentCache !(([FileDiagnostic], Maybe HscEnvEq), DependencyInfo) | LogHieBios HieBios.Log | LogSessionLoadingChanged + | LogSessionNewLoadedFiles ![FilePath] + | LogSessionReloadOnError FilePath ![FilePath] + | LogGetOptionsLoop !FilePath + | LogLookupSessionCache !FilePath + | LogTime !String + | LogSessionGhc Ghc.Log + | LogSessionWorkerThread LogWorkerThread deriving instance Show Log + instance Pretty Log where pretty = \case + LogSessionWorkerThread lt -> pretty lt + LogTime s -> "Time:" <+> pretty s + LogLookupSessionCache path -> "Looking up session cache for" <+> pretty path + LogGetOptionsLoop fp -> "Loop: getOptions for" <+> pretty fp + LogSessionReloadOnError path files -> + "Reloading file due to error in" <+> pretty path <+> "with files:" <+> pretty files + LogSessionNewLoadedFiles files -> + "New loaded files:" <+> pretty files LogNoneCradleFound path -> "None cradle found for" <+> pretty path <+> ", ignoring the file" LogSettingInitialDynFlags -> @@ -193,18 +186,12 @@ instance Pretty Log where vcat [ "HieDb writer thread exception:" , pretty (displayException e) ] - LogInterfaceFilesCacheDir path -> - "Interface files cache directory:" <+> pretty path LogKnownFilesUpdated targetToPathsMap -> nest 2 $ vcat [ "Known files updated:" , viaShow $ (HM.map . Set.map) fromNormalizedFilePath targetToPathsMap ] - LogMakingNewHscEnv inPlaceUnitIds -> - "Making new HscEnv. In-place unit ids:" <+> pretty (map show inPlaceUnitIds) - LogDLLLoadError errorString -> - "Error dynamically loading libm.so.6:" <+> pretty errorString LogCradlePath path -> "Cradle path:" <+> pretty path LogCradleNotFound path -> @@ -216,9 +203,8 @@ instance Pretty Log where "Session loading result:" <+> viaShow e LogCradle cradle -> "Cradle:" <+> viaShow cradle - LogNewComponentCache componentCache -> - "New component cache HscEnvEq:" <+> viaShow componentCache LogHieBios msg -> pretty msg + LogSessionGhc msg -> pretty msg LogSessionLoadingChanged -> "Session Loading config changed, reloading the full session." @@ -226,9 +212,6 @@ instance Pretty Log where hiedbDataVersion :: String hiedbDataVersion = "2" -data CacheDirs = CacheDirs - { hiCacheDir, hieCacheDir, oCacheDir :: Maybe FilePath} - data SessionLoadingOptions = SessionLoadingOptions { findCradle :: FilePath -> IO (Maybe FilePath) -- | Load the cradle with an optional 'hie.yaml' location. @@ -381,7 +364,7 @@ runWithDb recorder fp = ContT $ \k -> do _ <- withWriteDbRetryable deleteMissingRealFiles _ <- withWriteDbRetryable garbageCollectTypeNames - runContT (withWorkerQueue (writer withWriteDbRetryable)) $ \chan -> + runContT (withWorkerQueue (logWith (cmapWithPrio LogSessionWorkerThread recorder) Debug) "hiedb thread" (writer withWriteDbRetryable)) $ \chan -> withHieDb fp (\readDb -> k (WithHieDbShield $ makeWithHieDbRetryable recorder rng readDb, chan)) where writer withHieDbRetryable l = do @@ -401,6 +384,199 @@ getHieDbLoc dir = do createDirectoryIfMissing True cDir pure (cDir db) +{- Note [SessionState and batch load] +SessionState manages the state for batch loading files in the session loader. + +- When a new file needs to be loaded, it is added to the pendingFiles set. +- The loader processes files from pendingFiles, attempting to load them in batches. +- (SBL1) If a file is already in failedFiles, it is loaded individually (single-file mode). +- (SBL2) Otherwise, the loader tries to load as many files as possible together (batch mode). + +On success: + - (SBL3) All successfully loaded files are removed from pendingFiles and failedFiles, + and added to loadedFiles. + +On failure: + - (SBL4) If loading a single file fails, it is added to failedFiles and removed from loadedFiles and pendingFiles. + - (SBL5) If batch loading fails, all files attempted are added to failedFiles. + +This approach ensures efficient batch loading while isolating problematic files for individual handling. +-} + +-- SBL3 +handleBatchLoadSuccess :: Foldable t => Recorder (WithPriority Log) -> SessionState -> Maybe FilePath -> HashMap NormalizedFilePath (IdeResult HscEnvEq, DependencyInfo) -> t TargetDetails -> IO () +handleBatchLoadSuccess recorder sessionState hieYaml this_flags_map all_targets = do + pendings <- getPendingFiles sessionState + -- this_flags_map might contains files not in pendingFiles, take the intersection + let newLoaded = pendings `Set.intersection` Set.fromList (fromNormalizedFilePath <$> HM.keys this_flags_map) + atomically $ do + STM.insert this_flags_map hieYaml (fileToFlags sessionState) + insertAllFileMappings sessionState $ map ((hieYaml,) . fst) $ concatMap toFlagsMap all_targets + logWith recorder Info $ LogSessionNewLoadedFiles $ Set.toList newLoaded + atomically $ forM_ (Set.toList newLoaded) $ flip S.delete (pendingFiles sessionState) + mapM_ (removeErrorLoadingFile sessionState) (Set.toList newLoaded) + addCradleFiles sessionState newLoaded + +-- SBL5 +handleBatchLoadFailure :: SessionState -> [FilePath] -> IO () +handleBatchLoadFailure sessionState files = do + mapM_ (addErrorLoadingFile sessionState) files + +-- SBL4 +handleSingleLoadFailure :: SessionState -> FilePath -> IO () +handleSingleLoadFailure sessionState file = do + addErrorLoadingFile sessionState file + removeErrorLoadingFile sessionState file + atomically $ S.delete file (pendingFiles sessionState) + removeCradleFile sessionState file + +data SessionState = SessionState + { loadedFiles :: !(Var (HashSet FilePath)), + failedFiles :: !(Var (HashSet FilePath)), + pendingFiles :: !(S.OrderedSet FilePath), + hscEnvs :: !(Var HieMap), + fileToFlags :: !FlagsMap, + filesMap :: !FilesMap, + version :: !(Var Int), + sessionLoadingPreferenceConfig :: !(Var (Maybe SessionLoadingPreferenceConfig)) + } + +-- | Helper functions for SessionState management +-- These functions encapsulate common operations on the SessionState + +-- | Add a file to the set of files with errors during loading +addErrorLoadingFile :: MonadIO m => SessionState -> FilePath -> m () +addErrorLoadingFile state file = + liftIO $ modifyVar_' (failedFiles state) (\xs -> return $ Set.insert file xs) + +-- | Remove a file from the set of files with errors during loading +removeErrorLoadingFile :: MonadIO m => SessionState -> FilePath -> m () +removeErrorLoadingFile state file = + liftIO $ modifyVar_' (failedFiles state) (\xs -> return $ Set.delete file xs) + +addCradleFiles :: MonadIO m => SessionState -> HashSet FilePath -> m () +addCradleFiles state files = + liftIO $ modifyVar_' (loadedFiles state) (\xs -> return $ files <> xs) + +-- | Remove a file from the cradle files set +removeCradleFile :: MonadIO m => SessionState -> FilePath -> m () +removeCradleFile state file = + liftIO $ modifyVar_' (loadedFiles state) (\xs -> return $ Set.delete file xs) + +-- | Clear error loading files and reset to empty set +clearErrorLoadingFiles :: MonadIO m => SessionState -> m () +clearErrorLoadingFiles state = + liftIO $ modifyVar_' (failedFiles state) (const $ return Set.empty) + +-- | Clear cradle files and reset to empty set +clearCradleFiles :: MonadIO m => SessionState -> m () +clearCradleFiles state = + liftIO $ modifyVar_' (loadedFiles state) (const $ return Set.empty) + +-- | Reset the file maps in the session state +resetFileMaps :: SessionState -> STM () +resetFileMaps state = do + STM.reset (filesMap state) + STM.reset (fileToFlags state) + +-- | Insert or update file flags for a specific hieYaml and normalized file path +insertFileFlags :: SessionState -> Maybe FilePath -> NormalizedFilePath -> (IdeResult HscEnvEq, DependencyInfo) -> STM () +insertFileFlags state hieYaml ncfp flags = + STM.focus (Focus.insertOrMerge HM.union (HM.singleton ncfp flags)) hieYaml (fileToFlags state) + +-- | Insert a file mapping from normalized path to hieYaml location +insertFileMapping :: SessionState -> Maybe FilePath -> NormalizedFilePath -> STM () +insertFileMapping state hieYaml ncfp = + STM.insert hieYaml ncfp (filesMap state) + +-- | Remove a file from the pending file set +removeFromPending :: SessionState -> FilePath -> STM () +removeFromPending state file = + S.delete file (pendingFiles state) + +-- | Add a file to the pending file set +addToPending :: SessionState -> FilePath -> STM () +addToPending state file = + S.insert file (pendingFiles state) + +-- | Insert multiple file mappings at once +insertAllFileMappings :: SessionState -> [(Maybe FilePath, NormalizedFilePath)] -> STM () +insertAllFileMappings state mappings = + mapM_ (\(yaml, path) -> insertFileMapping state yaml path) mappings + +-- | Increment the version counter +incrementVersion :: SessionState -> IO Int +incrementVersion state = modifyVar' (version state) succ + +-- | Get files from the pending file set +getPendingFiles :: SessionState -> IO (HashSet FilePath) +getPendingFiles state = atomically $ S.toHashSet (pendingFiles state) + +-- | Handle errors during session loading by recording file as having error and removing from pending +handleSingleFileProcessingError' :: SessionState -> Maybe FilePath -> FilePath -> PackageSetupException -> SessionM () +handleSingleFileProcessingError' state hieYaml file e = do + handleSingleFileProcessingError state hieYaml file [renderPackageSetupException file e] mempty + +-- | Common pattern: Insert file flags, insert file mapping, and remove from pending +handleSingleFileProcessingError :: SessionState -> Maybe FilePath -> FilePath -> [FileDiagnostic] -> [FilePath] -> SessionM () +handleSingleFileProcessingError state hieYaml file diags extraDepFiles = liftIO $ do + dep <- getDependencyInfo $ maybeToList hieYaml <> extraDepFiles + let ncfp = toNormalizedFilePath' file + let flags = ((diags, Nothing), dep) + handleSingleLoadFailure state file + atomically $ do + insertFileFlags state hieYaml ncfp flags + insertFileMapping state hieYaml ncfp + +-- | Get the set of extra files to load based on the current file path +-- If the current file is in error loading files, we fallback to single loading mode (empty set) +-- Otherwise, we remove error files from pending files and also exclude the current file +getExtraFilesToLoad :: SessionState -> FilePath -> IO [FilePath] +getExtraFilesToLoad state cfp = do + pendingFiles <- getPendingFiles state + errorFiles <- readVar (failedFiles state) + old_files <- readVar (loadedFiles state) + -- if the file is in error loading files, we fall back to single loading mode + return $ + Set.toList $ + if cfp `Set.member` errorFiles + then Set.empty + -- remove error files from pending files since error loading need to load one by one + else (Set.delete cfp $ pendingFiles `Set.difference` errorFiles) <> old_files + +-- | We allow users to specify a loading strategy. +-- Check whether this config was changed since the last time we have loaded +-- a session. +-- +-- If the loading configuration changed, we likely should restart the session +-- in its entirety. +didSessionLoadingPreferenceConfigChange :: SessionState -> SessionM Bool +didSessionLoadingPreferenceConfigChange s = do + clientConfig <- asks sessionClientConfig + let biosSessionLoadingVar = sessionLoadingPreferenceConfig s + mLoadingConfig <- liftIO $ readVar biosSessionLoadingVar + case mLoadingConfig of + Nothing -> do + liftIO $ writeVar biosSessionLoadingVar (Just (sessionLoading clientConfig)) + pure False + Just loadingConfig -> do + liftIO $ writeVar biosSessionLoadingVar (Just (sessionLoading clientConfig)) + pure (loadingConfig /= sessionLoading clientConfig) + +newSessionState :: IO SessionState +newSessionState = do + -- Initialize SessionState + sessionState <- SessionState + <$> newVar (Set.fromList []) -- loadedFiles + <*> newVar (Set.fromList []) -- failedFiles + <*> S.newIO -- pendingFiles + <*> newVar Map.empty -- hscEnvs + <*> STM.newIO -- fileToFlags + <*> STM.newIO -- filesMap + <*> newVar 0 -- version + <*> newVar Nothing -- sessionLoadingPreferenceConfig + return sessionState + -- | Given a root directory, return a Shake 'Action' which setups an -- 'IdeGhcSession' given a file. -- Some of the many things this does: @@ -415,23 +591,13 @@ getHieDbLoc dir = do -- components mapping to the same hie.yaml file are mapped to the same -- HscEnv which is updated as new components are discovered. -loadSessionWithOptions :: Recorder (WithPriority Log) -> SessionLoadingOptions -> FilePath -> TQueue (IO ()) -> IO (Action IdeGhcSession) +loadSessionWithOptions :: Recorder (WithPriority Log) -> SessionLoadingOptions -> FilePath -> TaskQueue (IO ()) -> IO (Action IdeGhcSession) loadSessionWithOptions recorder SessionLoadingOptions{..} rootDir que = do let toAbsolutePath = toAbsolute rootDir -- see Note [Root Directory] - cradle_files <- newIORef [] - -- Mapping from hie.yaml file to HscEnv, one per hie.yaml file - hscEnvs <- newVar Map.empty :: IO (Var HieMap) - -- Mapping from a Filepath to HscEnv - fileToFlags <- newVar Map.empty :: IO (Var FlagsMap) - -- Mapping from a Filepath to its 'hie.yaml' location. - -- Should hold the same Filepaths as 'fileToFlags', otherwise - -- they are inconsistent. So, everywhere you modify 'fileToFlags', - -- you have to modify 'filesMap' as well. - filesMap <- newVar HM.empty :: IO (Var FilesMap) - -- Version of the mappings above - version <- newVar 0 - biosSessionLoadingVar <- newVar Nothing :: IO (Var (Maybe SessionLoadingPreferenceConfig)) - let returnWithVersion fun = IdeGhcSession fun <$> liftIO (readVar version) + + sessionState <- newSessionState + let returnWithVersion fun = IdeGhcSession fun <$> liftIO (readVar (version sessionState)) + -- This caches the mapping from Mod.hs -> hie.yaml cradleLoc <- liftIO $ memoIO $ \v -> do res <- findCradle v @@ -443,277 +609,350 @@ loadSessionWithOptions recorder SessionLoadingOptions{..} rootDir que = do return $ do clientConfig <- getClientConfigAction - extras@ShakeExtras{restartShakeSession, ideNc, knownTargetsVar, lspEnv + extras@ShakeExtras{ideNc, knownTargetsVar } <- getShakeExtras let invalidateShakeCache = do - void $ modifyVar' version succ + void $ incrementVersion sessionState return $ toNoFileKey GhcSessionIO - IdeOptions{ optTesting = IdeTesting optTesting - , optCheckProject = getCheckProject - , optExtensions - , optHaddockParse - } <- getIdeOptions - - -- populate the knownTargetsVar with all the - -- files in the project so that `knownFiles` can learn about them and - -- we can generate a complete module graph - let extendKnownTargets newTargets = do - knownTargets <- concatForM newTargets $ \TargetDetails{..} -> - case targetTarget of - TargetFile f -> do - -- If a target file has multiple possible locations, then we - -- assume they are all separate file targets. - -- This happens with '.hs-boot' files if they are in the root directory of the project. - -- GHC reports options such as '-i. A' as 'TargetFile A.hs' instead of 'TargetModule A'. - -- In 'fromTargetId', we dutifully look for '.hs-boot' files and add them to the - -- targetLocations of the TargetDetails. Then we add everything to the 'knownTargetsVar'. - -- However, when we look for a 'Foo.hs-boot' file in 'FindImports.hs', we look for either - -- - -- * TargetFile Foo.hs-boot - -- * TargetModule Foo - -- - -- If we don't generate a TargetFile for each potential location, we will only have - -- 'TargetFile Foo.hs' in the 'knownTargetsVar', thus not find 'TargetFile Foo.hs-boot' - -- and also not find 'TargetModule Foo'. - fs <- filterM (IO.doesFileExist . fromNormalizedFilePath) targetLocations - pure $ map (\fp -> (TargetFile fp, Set.singleton fp)) (nubOrd (f:fs)) - TargetModule _ -> do - found <- filterM (IO.doesFileExist . fromNormalizedFilePath) targetLocations - return [(targetTarget, Set.fromList found)] - hasUpdate <- atomically $ do - known <- readTVar knownTargetsVar - let known' = flip mapHashed known $ \k -> unionKnownTargets k (mkKnownTargets knownTargets) - hasUpdate = if known /= known' then Just (unhashed known') else Nothing - writeTVar knownTargetsVar known' - pure hasUpdate - for_ hasUpdate $ \x -> - logWith recorder Debug $ LogKnownFilesUpdated (targetMap x) - return $ toNoFileKey GetKnownTargets - - -- Create a new HscEnv from a hieYaml root and a set of options - let packageSetup :: (Maybe FilePath, NormalizedFilePath, ComponentOptions, FilePath) - -> IO ([ComponentInfo], [ComponentInfo]) - packageSetup (hieYaml, cfp, opts, libDir) = do - -- Parse DynFlags for the newly discovered component - hscEnv <- emptyHscEnv ideNc libDir - newTargetDfs <- evalGhcEnv hscEnv $ setOptions optHaddockParse cfp opts (hsc_dflags hscEnv) rootDir - let deps = componentDependencies opts ++ maybeToList hieYaml - dep_info <- getDependencyInfo (fmap toAbsolutePath deps) - -- Now lookup to see whether we are combining with an existing HscEnv - -- or making a new one. The lookup returns the HscEnv and a list of - -- information about other components loaded into the HscEnv - -- (unitId, DynFlag, Targets) - modifyVar hscEnvs $ \m -> do - -- Just deps if there's already an HscEnv - -- Nothing is it's the first time we are making an HscEnv - let oldDeps = Map.lookup hieYaml m - let -- Add the raw information about this component to the list - -- We will modify the unitId and DynFlags used for - -- compilation but these are the true source of - -- information. - new_deps = fmap (\(df, targets) -> RawComponentInfo (homeUnitId_ df) df targets cfp opts dep_info) newTargetDfs - all_deps = new_deps `NE.appendList` fromMaybe [] oldDeps - -- Get all the unit-ids for things in this component - _inplace = map rawComponentUnitId $ NE.toList all_deps - - all_deps' <- forM all_deps $ \RawComponentInfo{..} -> do - let prefix = show rawComponentUnitId - -- See Note [Avoiding bad interface files] - let cacheDirOpts = componentOptions opts - cacheDirs <- liftIO $ getCacheDirs prefix cacheDirOpts - processed_df <- setCacheDirs recorder cacheDirs rawComponentDynFlags - -- The final component information, mostly the same but the DynFlags don't - -- contain any packages which are also loaded - -- into the same component. - pure $ ComponentInfo - { componentUnitId = rawComponentUnitId - , componentDynFlags = processed_df - , componentTargets = rawComponentTargets - , componentFP = rawComponentFP - , componentCOptions = rawComponentCOptions - , componentDependencyInfo = rawComponentDependencyInfo - } - -- Modify the map so the hieYaml now maps to the newly updated - -- ComponentInfos - -- Returns - -- . The information for the new component which caused this cache miss - -- . The modified information (without -inplace flags) for - -- existing packages - let (new,old) = NE.splitAt (NE.length new_deps) all_deps' - pure (Map.insert hieYaml (NE.toList all_deps) m, (new,old)) - - - let session :: (Maybe FilePath, NormalizedFilePath, ComponentOptions, FilePath) - -> IO (IdeResult HscEnvEq,[FilePath]) - session args@(hieYaml, _cfp, _opts, _libDir) = do - (new_deps, old_deps) <- packageSetup args - - -- For each component, now make a new HscEnvEq which contains the - -- HscEnv for the hie.yaml file but the DynFlags for that component - -- For GHC's supporting multi component sessions, we create a shared - -- HscEnv but set the active component accordingly - hscEnv <- emptyHscEnv ideNc _libDir - let new_cache = newComponentCache recorder optExtensions _cfp hscEnv - all_target_details <- new_cache old_deps new_deps - - this_dep_info <- getDependencyInfo $ maybeToList hieYaml - let (all_targets, this_flags_map, this_options) - = case HM.lookup _cfp flags_map' of - Just this -> (all_targets', flags_map', this) - Nothing -> (this_target_details : all_targets', HM.insert _cfp this_flags flags_map', this_flags) - where all_targets' = concat all_target_details - flags_map' = HM.fromList (concatMap toFlagsMap all_targets') - this_target_details = TargetDetails (TargetFile _cfp) this_error_env this_dep_info [_cfp] - this_flags = (this_error_env, this_dep_info) - this_error_env = ([this_error], Nothing) - this_error = ideErrorWithSource (Just "cradle") (Just DiagnosticSeverity_Error) _cfp - (T.unlines - [ "No cradle target found. Is this file listed in the targets of your cradle?" - , "If you are using a .cabal file, please ensure that this module is listed in either the exposed-modules or other-modules section" - ]) - Nothing - - void $ modifyVar' fileToFlags $ Map.insert hieYaml this_flags_map - void $ modifyVar' filesMap $ flip HM.union (HM.fromList (map ((,hieYaml) . fst) $ concatMap toFlagsMap all_targets)) - -- The VFS doesn't change on cradle edits, re-use the old one. - -- Invalidate all the existing GhcSession build nodes by restarting the Shake session - keys2 <- invalidateShakeCache - restartShakeSession VFSUnmodified "new component" [] $ do - keys1 <- extendKnownTargets all_targets - return [keys1, keys2] - - -- Typecheck all files in the project on startup - checkProject <- getCheckProject - unless (null new_deps || not checkProject) $ do - cfps' <- liftIO $ filterM (IO.doesFileExist . fromNormalizedFilePath) (concatMap targetLocations all_targets) - void $ shakeEnqueue extras $ mkDelayedAction "InitialLoad" Debug $ void $ do - mmt <- uses GetModificationTime cfps' - let cs_exist = catMaybes (zipWith (<$) cfps' mmt) - modIfaces <- uses GetModIface cs_exist - -- update exports map - shakeExtras <- getShakeExtras - let !exportsMap' = createExportsMap $ mapMaybe (fmap hirModIface) modIfaces - liftIO $ atomically $ modifyTVar' (exportsMap shakeExtras) (exportsMap' <>) - - return $ second Map.keys this_options - - let consultCradle :: Maybe FilePath -> FilePath -> IO (IdeResult HscEnvEq, [FilePath]) - consultCradle hieYaml cfp = do - let lfpLog = makeRelative rootDir cfp - logWith recorder Info $ LogCradlePath lfpLog - when (isNothing hieYaml) $ - logWith recorder Warning $ LogCradleNotFound lfpLog - cradle <- loadCradle recorder hieYaml rootDir - when optTesting $ mRunLspT lspEnv $ - sendNotification (SMethod_CustomMethod (Proxy @"ghcide/cradle/loaded")) (toJSON cfp) - - -- Display a user friendly progress message here: They probably don't know what a cradle is - let progMsg = "Setting up " <> T.pack (takeBaseName (cradleRootDir cradle)) - <> " (for " <> T.pack lfpLog <> ")" - eopts <- mRunLspTCallback lspEnv (\act -> withIndefiniteProgress progMsg Nothing NotCancellable (const act)) $ - withTrace "Load cradle" $ \addTag -> do - addTag "file" lfpLog - old_files <- readIORef cradle_files - res <- cradleToOptsAndLibDir recorder (sessionLoading clientConfig) cradle cfp old_files - addTag "result" (show res) - return res - - logWith recorder Debug $ LogSessionLoadingResult eopts - case eopts of - -- The cradle gave us some options so get to work turning them - -- into and HscEnv. - Right (opts, libDir, version) -> do - let compileTime = fullCompilerVersion - case reverse $ readP_to_S parseVersion version of - [] -> error $ "GHC version could not be parsed: " <> version - ((runTime, _):_) - | compileTime == runTime -> do - atomicModifyIORef' cradle_files (\xs -> (cfp:xs,())) - session (hieYaml, toNormalizedFilePath' cfp, opts, libDir) - | otherwise -> return (([renderPackageSetupException cfp GhcVersionMismatch{..}], Nothing),[]) - -- Failure case, either a cradle error or the none cradle - Left err -> do - dep_info <- getDependencyInfo (maybeToList hieYaml) - let ncfp = toNormalizedFilePath' cfp - let res = (map (\err' -> renderCradleError err' cradle ncfp) err, Nothing) - void $ modifyVar' fileToFlags $ - Map.insertWith HM.union hieYaml (HM.singleton ncfp (res, dep_info)) - void $ modifyVar' filesMap $ HM.insert ncfp hieYaml - return (res, maybe [] pure hieYaml ++ concatMap cradleErrorDependencies err) - - let - -- | We allow users to specify a loading strategy. - -- Check whether this config was changed since the last time we have loaded - -- a session. + ideOptions <- getIdeOptions + + -- see Note [Serializing runs in separate thread] + -- Start the getOptionsLoop if the queue is empty + liftIO $ atomically $ Extra.whenM (isEmptyTaskQueue que) $ do + let newSessionLoadingOptions = SessionLoadingOptions + { findCradle = cradleLoc + , .. + } + sessionShake = SessionShake + { restartSession = restartShakeSession extras + , invalidateCache = invalidateShakeCache + , enqueueActions = shakeEnqueue extras + } + sessionEnv = SessionEnv + { sessionLspContext = lspEnv extras + , sessionRootDir = rootDir + , sessionIdeOptions = ideOptions + , sessionClientConfig = clientConfig + , sessionSharedNameCache = ideNc + , sessionLoadingOptions = newSessionLoadingOptions + } + + writeTaskQueue que (runReaderT (getOptionsLoop recorder sessionShake sessionState knownTargetsVar) sessionEnv) + + -- Each one of deps will be registered as a FileSystemWatcher in the GhcSession action + -- so that we can get a workspace/didChangeWatchedFiles notification when a dep changes. + -- The GlobPattern of a FileSystemWatcher can be absolute or relative. + -- We use the absolute one because it is supported by more LSP clients. + -- Here we make sure deps are absolute and later we use those absolute deps as GlobPattern. + let absolutePathsCradleDeps (eq, deps) = (eq, fmap toAbsolutePath $ Map.keys deps) + returnWithVersion $ \file -> do + let absFile = toAbsolutePath file + absolutePathsCradleDeps <$> lookupOrWaitCache recorder sessionState absFile + +-- | Given a file, this function will return the HscEnv and the dependencies +-- it would look up the cache first, if the cache is not available, it would +-- submit a request to the getOptionsLoop to get the options for the file +-- and wait until the options are available +lookupOrWaitCache :: Recorder (WithPriority Log) -> SessionState -> FilePath -> IO (IdeResult HscEnvEq, DependencyInfo) +lookupOrWaitCache recorder sessionState absFile = do + let ncfp = toNormalizedFilePath' absFile + res <- atomically $ do + -- wait until target file is not in pendingFiles + Extra.whenM (S.lookup absFile (pendingFiles sessionState)) STM.retry + -- check if in the cache + checkInCache sessionState ncfp + logWith recorder Debug $ LogLookupSessionCache absFile + updateDateRes <- case res of + Just r -> do + depOk <- checkDependencyInfo (snd r) + if depOk + then return $ Just r + else return Nothing + _ -> return Nothing + case updateDateRes of + Just r -> return r + Nothing -> do + -- if not ok, we need to reload the session + atomically $ addToPending sessionState absFile + lookupOrWaitCache recorder sessionState absFile + +checkInCache :: SessionState -> NormalizedFilePath -> STM (Maybe (IdeResult HscEnvEq, DependencyInfo)) +checkInCache sessionState ncfp = runMaybeT $ do + cachedHieYamlLocation <- MaybeT $ STM.lookup ncfp (filesMap sessionState) + m <- MaybeT $ STM.lookup cachedHieYamlLocation (fileToFlags sessionState) + MaybeT $ pure $ HM.lookup ncfp m + +data SessionShake = SessionShake + { restartSession :: VFSModified -> String -> [DelayedAction ()] -> IO [Key] -> IO () + , invalidateCache :: IO Key + , enqueueActions :: DelayedAction () -> IO (IO ()) + } + +data SessionEnv = SessionEnv + { sessionLspContext :: Maybe (LanguageContextEnv Config) + , sessionRootDir :: FilePath + , sessionIdeOptions :: IdeOptions + , sessionClientConfig :: Config + , sessionSharedNameCache :: NameCache + , sessionLoadingOptions :: SessionLoadingOptions + } + +type SessionM = ReaderT SessionEnv IO + +-- | The main function which gets options for a file. +-- +-- The general approach is as follows: +-- 1. Find the 'hie.yaml' for the next file target, if there is any. +-- 2. Check in the cache, whether the given 'hie.yaml' was already loaded before +-- 3.1. If it wasn't, initialise a new session and continue with step 4. +-- 3.2. If it is loaded, check whether we need to reload the session, e.g. because the `.cabal` file was modified +-- 3.2.1. If we need to reload, remove the +getOptionsLoop :: Recorder (WithPriority Log) -> SessionShake -> SessionState -> TVar (Hashed KnownTargets) -> SessionM () +getOptionsLoop recorder sessionShake sessionState knownTargetsVar = forever $ do + -- Get the next file to load + file <- liftIO $ atomically $ S.readQueue (pendingFiles sessionState) + logWith recorder Debug (LogGetOptionsLoop file) + let ncfp = toNormalizedFilePath' file + cachedHieYamlLocation <- join <$> liftIO (atomically (STM.lookup ncfp (filesMap sessionState))) + sessionLoadingOptions <- asks sessionLoadingOptions + hieYaml <- liftIO $ findCradle sessionLoadingOptions file + let hieLoc = cachedHieYamlLocation <|> hieYaml + sessionOpts recorder sessionShake sessionState knownTargetsVar (hieLoc, file) + `Safe.catch` handleSingleFileProcessingError' sessionState hieLoc file + +-- | This caches the mapping from hie.yaml + Mod.hs -> [String] +-- Returns the Ghc session and the cradle dependencies +sessionOpts :: Recorder (WithPriority Log) -> SessionShake -> SessionState -> TVar (Hashed KnownTargets) -> (Maybe FilePath, FilePath) -> SessionM () +sessionOpts recorder sessionShake sessionState knownTargetsVar (hieYaml, file) = do + Extra.whenM (didSessionLoadingPreferenceConfigChange sessionState) $ do + logWith recorder Info LogSessionLoadingChanged + liftIO $ atomically $ resetFileMaps sessionState + -- Don't even keep the name cache, we start from scratch here! + liftIO $ modifyVar_ (hscEnvs sessionState) (const (return Map.empty)) + -- cleanup error loading files and cradle files + clearErrorLoadingFiles sessionState + clearCradleFiles sessionState + cacheKey <- liftIO $ invalidateCache sessionShake + liftIO $ restartSession sessionShake VFSUnmodified "didSessionLoadingPreferenceConfigChange" [] (return [cacheKey]) + + v <- liftIO $ atomically $ STM.lookup hieYaml (fileToFlags sessionState) + case v >>= HM.lookup (toNormalizedFilePath' file) of + Just (_opts, old_di) -> do + deps_ok <- liftIO $ checkDependencyInfo old_di + if not deps_ok + then do + -- if deps are old, we can try to load the error files again + removeErrorLoadingFile sessionState file + removeCradleFile sessionState file + -- If the dependencies are out of date then clear both caches and start + -- again. + liftIO $ atomically $ resetFileMaps sessionState + -- Keep the same name cache + liftIO $ modifyVar_ (hscEnvs sessionState) (return . Map.adjust (const []) hieYaml) + consultCradle recorder sessionShake sessionState knownTargetsVar hieYaml file + else do + -- if deps are ok, we can just remove the file from pending files + liftIO $ atomically $ removeFromPending sessionState file + Nothing -> + consultCradle recorder sessionShake sessionState knownTargetsVar hieYaml file + +consultCradle :: Recorder (WithPriority Log) -> SessionShake -> SessionState -> TVar (Hashed KnownTargets) -> Maybe FilePath -> FilePath -> SessionM () +consultCradle recorder sessionShake sessionState knownTargetsVar hieYaml cfp = do + loadingOptions <- asks sessionLoadingOptions + (cradle, eopts) <- loadCradleWithNotifications recorder + sessionState + (loadCradle loadingOptions recorder) + hieYaml cfp + logWith recorder Debug $ LogSessionLoadingResult eopts + let ncfp = toNormalizedFilePath' cfp + case eopts of + -- The cradle gave us some options so get to work turning them + -- into and HscEnv. + Right (opts, libDir, version) -> do + let compileTime = fullCompilerVersion + case reverse $ readP_to_S parseVersion version of + [] -> error $ "GHC version could not be parsed: " <> version + ((runTime, _):_) + | compileTime == runTime -> session recorder sessionShake sessionState knownTargetsVar (hieYaml, ncfp, opts, libDir) + | otherwise -> handleSingleFileProcessingError' sessionState hieYaml cfp (GhcVersionMismatch{..}) + -- Failure case, either a cradle error or the none cradle + Left err -> do + -- what if the error to load file is one of old_files ? + let attemptToLoadFiles = Set.delete cfp $ Set.fromList $ concatMap cradleErrorLoadingFiles err + old_files <- liftIO $ readVar (loadedFiles sessionState) + let errorToLoadNewFiles = cfp : Set.toList (attemptToLoadFiles `Set.difference` old_files) + if length errorToLoadNewFiles > 1 + then do + -- we are loading more files and failed, we need to retry + -- mark as less loaded files as failedLoadingFiles as possible + -- limitation is that when we are loading files, and the dependencies of old_files + -- are changed, and old_files are not valid anymore. + -- but they will still be in the old_files, and will not move to failedFiles. + -- And make other files failed to load in batch mode. + liftIO $ handleBatchLoadFailure sessionState errorToLoadNewFiles + -- retry without other files + logWith recorder Info $ LogSessionReloadOnError cfp (Set.toList attemptToLoadFiles) + consultCradle recorder sessionShake sessionState knownTargetsVar hieYaml cfp + else do + -- we are only loading this file and it failed + let res = map (\err' -> renderCradleError err' cradle ncfp) err + handleSingleFileProcessingError sessionState hieYaml cfp res $ concatMap cradleErrorDependencies err + +session :: + Recorder (WithPriority Log) -> + SessionShake -> + SessionState -> + TVar (Hashed KnownTargets) -> + (Maybe FilePath, NormalizedFilePath, ComponentOptions, FilePath) -> + SessionM () +session recorder sessionShake sessionState knownTargetsVar(hieYaml, cfp, opts, libDir) = do + let initEmptyHscEnv = emptyHscEnvM libDir + (new_deps, old_deps) <- packageSetup recorder sessionState initEmptyHscEnv (hieYaml, cfp, opts) + + -- For each component, now make a new HscEnvEq which contains the + -- HscEnv for the hie.yaml file but the DynFlags for that component + -- For GHC's supporting multi component sessions, we create a shared + -- HscEnv but set the active component accordingly + hscEnv <- initEmptyHscEnv + ideOptions <- asks sessionIdeOptions + let new_cache = newComponentCache (cmapWithPrio LogSessionGhc recorder) (optExtensions ideOptions) cfp hscEnv + all_target_details <- liftIO $ new_cache old_deps new_deps + (all_targets, this_flags_map) <- liftIO $ addErrorTargetIfUnknown all_target_details hieYaml cfp + -- The VFS doesn't change on cradle edits, re-use the old one. + -- Invalidate all the existing GhcSession build nodes by restarting the Shake session + liftIO $ do + checkProject <- optCheckProject ideOptions + restartSession sessionShake VFSUnmodified "new component" [] $ do + -- It is necessary to call handleBatchLoadSuccess in restartSession + -- to ensure the GhcSession rule does not return before a new session is started. + -- Otherwise, invalid compilation results may propagate to downstream rules, + -- potentially resulting in lost diagnostics and other issues. + handleBatchLoadSuccess recorder sessionState hieYaml this_flags_map all_targets + keys2 <- invalidateCache sessionShake + keys1 <- extendKnownTargets recorder knownTargetsVar all_targets + -- Typecheck all files in the project on startup + unless (null new_deps || not checkProject) $ do + cfps' <- liftIO $ filterM (IO.doesFileExist . fromNormalizedFilePath) (concatMap targetLocations all_targets) + void $ enqueueActions sessionShake $ mkDelayedAction "InitialLoad" Debug $ void $ do + mmt <- uses GetModificationTime cfps' + let cs_exist = catMaybes (zipWith (<$) cfps' mmt) + modIfaces <- uses GetModIface cs_exist + -- update exports map + shakeExtras <- getShakeExtras + let !exportsMap' = createExportsMap $ mapMaybe (fmap hirModIface) modIfaces + liftIO $ atomically $ modifyTVar' (exportsMap shakeExtras) (exportsMap' <>) + return [keys1, keys2] + +-- | Create a new HscEnv from a hieYaml root and a set of options +packageSetup :: Recorder (WithPriority Log) -> SessionState -> SessionM HscEnv -> (Maybe FilePath, NormalizedFilePath, ComponentOptions) -> SessionM ([ComponentInfo], [ComponentInfo]) +packageSetup recorder sessionState newEmptyHscEnv (hieYaml, cfp, opts) = do + getCacheDirs <- asks (getCacheDirs . sessionLoadingOptions) + haddockparse <- asks (optHaddockParse . sessionIdeOptions) + rootDir <- asks sessionRootDir + -- Parse DynFlags for the newly discovered component + hscEnv <- newEmptyHscEnv + newTargetDfs <- liftIO $ mask_ $ evalGhcEnv hscEnv $ setOptions haddockparse cfp opts (hsc_dflags hscEnv) rootDir + let deps = componentDependencies opts ++ maybeToList hieYaml + dep_info <- liftIO $ getDependencyInfo (fmap (toAbsolute rootDir) deps) + -- Now lookup to see whether we are combining with an existing HscEnv + -- or making a new one. The lookup returns the HscEnv and a list of + -- information about other components loaded into the HscEnv + -- (unitId, DynFlag, Targets) + liftIO $ modifyVar (hscEnvs sessionState) $ + addComponentInfo (cmapWithPrio LogSessionGhc recorder) getCacheDirs dep_info newTargetDfs (hieYaml, cfp, opts) + +addErrorTargetIfUnknown :: Foldable t => t [TargetDetails] -> Maybe FilePath -> NormalizedFilePath -> IO ([TargetDetails], HashMap NormalizedFilePath (IdeResult HscEnvEq, DependencyInfo)) +addErrorTargetIfUnknown all_target_details hieYaml cfp = do + let flags_map' = HM.fromList (concatMap toFlagsMap all_targets') + all_targets' = concat all_target_details + this_dep_info <- getDependencyInfo $ maybeToList hieYaml + let (all_targets, this_flags_map) = case HM.lookup cfp flags_map' of + Just _ -> (all_targets', flags_map') + Nothing -> (this_target_details : all_targets', HM.insert cfp this_flags flags_map') + where + this_target_details = TargetDetails (TargetFile cfp) this_error_env this_dep_info [cfp] + this_flags = (this_error_env, this_dep_info) + this_error_env = ([this_error], Nothing) + this_error = ideErrorWithSource (Just "cradle") (Just DiagnosticSeverity_Error) cfp + (T.unlines + [ "No cradle target found. Is this file listed in the targets of your cradle?" + , "If you are using a .cabal file, please ensure that this module is listed in either the exposed-modules or other-modules section" + ]) + Nothing + pure (all_targets, this_flags_map) + +-- | Populate the knownTargetsVar with all the +-- files in the project so that `knownFiles` can learn about them and +-- we can generate a complete module graph +extendKnownTargets :: Recorder (WithPriority Log) -> TVar (Hashed KnownTargets) -> [TargetDetails] -> IO Key +extendKnownTargets recorder knownTargetsVar newTargets = do + knownTargets <- concatForM newTargets $ \TargetDetails{..} -> + case targetTarget of + TargetFile f -> do + -- If a target file has multiple possible locations, then we + -- assume they are all separate file targets. + -- This happens with '.hs-boot' files if they are in the root directory of the project. + -- GHC reports options such as '-i. A' as 'TargetFile A.hs' instead of 'TargetModule A'. + -- In 'fromTargetId', we dutifully look for '.hs-boot' files and add them to the + -- targetLocations of the TargetDetails. Then we add everything to the 'knownTargetsVar'. + -- However, when we look for a 'Foo.hs-boot' file in 'FindImports.hs', we look for either -- - -- If the loading configuration changed, we likely should restart the session - -- in its entirety. - didSessionLoadingPreferenceConfigChange :: IO Bool - didSessionLoadingPreferenceConfigChange = do - mLoadingConfig <- readVar biosSessionLoadingVar - case mLoadingConfig of - Nothing -> do - writeVar biosSessionLoadingVar (Just (sessionLoading clientConfig)) - pure False - Just loadingConfig -> do - writeVar biosSessionLoadingVar (Just (sessionLoading clientConfig)) - pure (loadingConfig /= sessionLoading clientConfig) - - -- This caches the mapping from hie.yaml + Mod.hs -> [String] - -- Returns the Ghc session and the cradle dependencies - let sessionOpts :: (Maybe FilePath, FilePath) - -> IO (IdeResult HscEnvEq, [FilePath]) - sessionOpts (hieYaml, file) = do - Extra.whenM didSessionLoadingPreferenceConfigChange $ do - logWith recorder Info LogSessionLoadingChanged - -- If the dependencies are out of date then clear both caches and start - -- again. - modifyVar_ fileToFlags (const (return Map.empty)) - modifyVar_ filesMap (const (return HM.empty)) - -- Don't even keep the name cache, we start from scratch here! - modifyVar_ hscEnvs (const (return Map.empty)) - - v <- Map.findWithDefault HM.empty hieYaml <$> readVar fileToFlags - let cfp = toAbsolutePath file - case HM.lookup (toNormalizedFilePath' cfp) v of - Just (opts, old_di) -> do - deps_ok <- checkDependencyInfo old_di - if not deps_ok - then do - -- If the dependencies are out of date then clear both caches and start - -- again. - modifyVar_ fileToFlags (const (return Map.empty)) - modifyVar_ filesMap (const (return HM.empty)) - -- Keep the same name cache - modifyVar_ hscEnvs (return . Map.adjust (const []) hieYaml ) - consultCradle hieYaml cfp - else return (opts, Map.keys old_di) - Nothing -> consultCradle hieYaml cfp - - -- The main function which gets options for a file. We only want one of these running - -- at a time. Therefore the IORef contains the currently running cradle, if we try - -- to get some more options then we wait for the currently running action to finish - -- before attempting to do so. - let getOptions :: FilePath -> IO (IdeResult HscEnvEq, [FilePath]) - getOptions file = do - let ncfp = toNormalizedFilePath' (toAbsolutePath file) - cachedHieYamlLocation <- HM.lookup ncfp <$> readVar filesMap - hieYaml <- cradleLoc file - let - -- Each one of deps will be registered as a FileSystemWatcher in the GhcSession action - -- so that we can get a workspace/didChangeWatchedFiles notification when a dep changes. - -- The GlobPattern of a FileSystemWatcher can be absolute or relative. - -- We use the absolute one because it is supported by more LSP clients. - -- Here we make sure deps are absolute and later we use those absolute deps as GlobPattern. - absolutePathsCradleDeps (eq, deps) - = (eq, fmap toAbsolutePath deps) - (absolutePathsCradleDeps <$> sessionOpts (join cachedHieYamlLocation <|> hieYaml, file)) `Safe.catch` \e -> - return (([renderPackageSetupException file e], Nothing), maybe [] pure hieYaml) + -- * TargetFile Foo.hs-boot + -- * TargetModule Foo + -- + -- If we don't generate a TargetFile for each potential location, we will only have + -- 'TargetFile Foo.hs' in the 'knownTargetsVar', thus not find 'TargetFile Foo.hs-boot' + -- and also not find 'TargetModule Foo'. + fs <- filterM (IO.doesFileExist . fromNormalizedFilePath) targetLocations + pure $ map (\fp -> (TargetFile fp, Set.singleton fp)) (nubOrd (f:fs)) + TargetModule _ -> do + found <- filterM (IO.doesFileExist . fromNormalizedFilePath) targetLocations + return [(targetTarget, Set.fromList found)] + hasUpdate <- atomically $ do + known <- readTVar knownTargetsVar + let known' = flip mapHashed known $ \k -> unionKnownTargets k (mkKnownTargets knownTargets) + hasUpdate = if known /= known' then Just (unhashed known') else Nothing + writeTVar knownTargetsVar known' + pure hasUpdate + for_ hasUpdate $ \x -> + logWith recorder Debug $ LogKnownFilesUpdated (targetMap x) + return $ toNoFileKey GetKnownTargets + + +loadCradleWithNotifications :: + Recorder (WithPriority Log) -> + SessionState -> + (Maybe FilePath -> FilePath -> IO (Cradle Void)) -> + Maybe FilePath -> + FilePath -> + SessionM (Cradle Void, Either [CradleError] (ComponentOptions, FilePath, String)) +loadCradleWithNotifications recorder sessionState loadCradle hieYaml cfp = do + IdeTesting isTesting <- asks (optTesting . sessionIdeOptions) + sessionPref <- asks (sessionLoading . sessionClientConfig) + lspEnv <- asks sessionLspContext + rootDir <- asks sessionRootDir + let lfpLog = makeRelative rootDir cfp + logWith recorder Info $ LogCradlePath lfpLog + when (isNothing hieYaml) $ + logWith recorder Warning $ LogCradleNotFound lfpLog + cradle <- liftIO $ loadCradle hieYaml rootDir + when isTesting $ mRunLspT lspEnv $ + sendNotification (SMethod_CustomMethod (Proxy @"ghcide/cradle/loaded")) (toJSON cfp) + + -- Display a user friendly progress message here: They probably don't know what a cradle is + let progMsg = "Setting up " <> T.pack (takeBaseName (cradleRootDir cradle)) + <> " (for " <> T.pack lfpLog <> ")" + + extraToLoads <- liftIO $ getExtraFilesToLoad sessionState cfp + eopts <- mRunLspTCallback lspEnv (\act -> withIndefiniteProgress progMsg Nothing NotCancellable (const act)) $ + withTrace "Load cradle" $ \addTag -> do + addTag "file" lfpLog + res <- liftIO $ cradleToOptsAndLibDir recorder sessionPref cradle cfp extraToLoads + addTag "result" (show res) + return res + pure (cradle, eopts) - returnWithVersion $ \file -> do - -- see Note [Serializing runs in separate thread] - awaitRunInThread que $ getOptions file -- | Run the specific cradle on a specific FilePath via hie-bios. -- This then builds dependencies or whatever based on the cradle, gets the @@ -749,340 +988,26 @@ cradleToOptsAndLibDir recorder loadConfig cradle file old_fps = do PreferSingleComponentLoading -> LoadFile PreferMultiComponentLoading -> LoadWithContext old_fps -emptyHscEnv :: NameCache -> FilePath -> IO HscEnv -emptyHscEnv nc libDir = do - -- We call setSessionDynFlags so that the loader is initialised - -- We need to do this before we call initUnits. - env <- runGhc (Just libDir) $ - getSessionDynFlags >>= setSessionDynFlags >> getSession - pure $ setNameCache nc (hscSetFlags ((hsc_dflags env){useUnicode = True }) env) - -data TargetDetails = TargetDetails - { - targetTarget :: !Target, - targetEnv :: !(IdeResult HscEnvEq), - targetDepends :: !DependencyInfo, - targetLocations :: ![NormalizedFilePath] - } +-- ---------------------------------------------------------------------------- +-- Utilities +-- ---------------------------------------------------------------------------- -fromTargetId :: [FilePath] -- ^ import paths - -> [String] -- ^ extensions to consider - -> TargetId - -> IdeResult HscEnvEq - -> DependencyInfo - -> IO [TargetDetails] --- For a target module we consider all the import paths -fromTargetId is exts (GHC.TargetModule modName) env dep = do - let fps = [i moduleNameSlashes modName -<.> ext <> boot - | ext <- exts - , i <- is - , boot <- ["", "-boot"] - ] - let locs = fmap toNormalizedFilePath' fps - return [TargetDetails (TargetModule modName) env dep locs] --- For a 'TargetFile' we consider all the possible module names -fromTargetId _ _ (GHC.TargetFile f _) env deps = do - let nf = toNormalizedFilePath' f - let other - | "-boot" `isSuffixOf` f = toNormalizedFilePath' (L.dropEnd 5 $ fromNormalizedFilePath nf) - | otherwise = toNormalizedFilePath' (fromNormalizedFilePath nf ++ "-boot") - return [TargetDetails (TargetFile nf) env deps [nf, other]] +emptyHscEnvM :: FilePath -> SessionM HscEnv +emptyHscEnvM libDir = do + nc <- asks sessionSharedNameCache + liftIO $ Ghc.emptyHscEnv nc libDir toFlagsMap :: TargetDetails -> [(NormalizedFilePath, (IdeResult HscEnvEq, DependencyInfo))] toFlagsMap TargetDetails{..} = [ (l, (targetEnv, targetDepends)) | l <- targetLocations] - -setNameCache :: NameCache -> HscEnv -> HscEnv -setNameCache nc hsc = hsc { hsc_NC = nc } - -#if MIN_VERSION_ghc(9,13,0) --- Moved back to implementation in GHC. -checkHomeUnitsClosed' :: UnitEnv -> OS.Set UnitId -> [DriverMessages] -checkHomeUnitsClosed' ue _ = checkHomeUnitsClosed ue -#else --- This function checks the important property that if both p and q are home units --- then any dependency of p, which transitively depends on q is also a home unit. --- GHC had an implementation of this function, but it was horribly inefficient --- We should move back to the GHC implementation on compilers where --- https://gitlab.haskell.org/ghc/ghc/-/merge_requests/12162 is included -checkHomeUnitsClosed' :: UnitEnv -> OS.Set UnitId -> Maybe (Compat.MsgEnvelope DriverMessage) -checkHomeUnitsClosed' ue home_id_set - | OS.null bad_unit_ids = Nothing - | otherwise = Just (GHC.mkPlainErrorMsgEnvelope rootLoc $ DriverHomePackagesNotClosed (OS.toList bad_unit_ids)) - where - bad_unit_ids = upwards_closure OS.\\ home_id_set - rootLoc = mkGeneralSrcSpan (Compat.fsLit "") - - graph :: Graph (Node UnitId UnitId) - graph = graphFromEdgedVerticesUniq graphNodes - - -- downwards closure of graph - downwards_closure - = graphFromEdgedVerticesUniq [ DigraphNode uid uid (OS.toList deps) - | (uid, deps) <- Map.toList (allReachable graph node_key)] - - inverse_closure = transposeG downwards_closure - - upwards_closure = OS.fromList $ map node_key $ reachablesG inverse_closure [DigraphNode uid uid [] | uid <- OS.toList home_id_set] - - all_unit_direct_deps :: UniqMap UnitId (OS.Set UnitId) - all_unit_direct_deps - = unitEnv_foldWithKey go emptyUniqMap $ ue_home_unit_graph ue - where - go rest this this_uis = - plusUniqMap_C OS.union - (addToUniqMap_C OS.union external_depends this (OS.fromList this_deps)) - rest - where - external_depends = mapUniqMap (OS.fromList . unitDepends) -#if !MIN_VERSION_ghc(9,7,0) - $ listToUniqMap $ Map.toList -#endif - - $ unitInfoMap this_units - this_units = homeUnitEnv_units this_uis - this_deps = [ Compat.toUnitId unit | (unit,Just _) <- explicitUnits this_units] - - graphNodes :: [Node UnitId UnitId] - graphNodes = go OS.empty home_id_set - where - go done todo - = case OS.minView todo of - Nothing -> [] - Just (uid, todo') - | OS.member uid done -> go done todo' - | otherwise -> case lookupUniqMap all_unit_direct_deps uid of - Nothing -> pprPanic "uid not found" (Compat.ppr (uid, all_unit_direct_deps)) - Just depends -> - let todo'' = (depends OS.\\ done) `OS.union` todo' - in DigraphNode uid uid (OS.toList depends) : go (OS.insert uid done) todo'' -#endif - --- | Create a mapping from FilePaths to HscEnvEqs --- This combines all the components we know about into --- an appropriate session, which is a multi component --- session on GHC 9.4+ -newComponentCache - :: Recorder (WithPriority Log) - -> [String] -- ^ File extensions to consider - -> NormalizedFilePath -- ^ Path to file that caused the creation of this component - -> HscEnv -- ^ An empty HscEnv - -> [ComponentInfo] -- ^ New components to be loaded - -> [ComponentInfo] -- ^ old, already existing components - -> IO [ [TargetDetails] ] -newComponentCache recorder exts _cfp hsc_env old_cis new_cis = do - let cis = Map.unionWith unionCIs (mkMap new_cis) (mkMap old_cis) - -- When we have multiple components with the same uid, - -- prefer the new one over the old. - -- However, we might have added some targets to the old unit - -- (see special target), so preserve those - unionCIs new_ci old_ci = new_ci { componentTargets = componentTargets new_ci ++ componentTargets old_ci } - mkMap = Map.fromListWith unionCIs . map (\ci -> (componentUnitId ci, ci)) - let dfs = map componentDynFlags $ Map.elems cis - uids = Map.keys cis - logWith recorder Info $ LogMakingNewHscEnv uids - hscEnv' <- -- Set up a multi component session with the other units on GHC 9.4 - Compat.initUnits dfs hsc_env - - let closure_errs = maybeToList $ checkHomeUnitsClosed' (hsc_unit_env hscEnv') (hsc_all_home_unit_ids hscEnv') - closure_err_to_multi_err err = - ideErrorWithSource - (Just "cradle") (Just DiagnosticSeverity_Warning) _cfp - (T.pack (Compat.printWithoutUniques (singleMessage err))) - (Just (fmap GhcDriverMessage err)) - multi_errs = map closure_err_to_multi_err closure_errs - bad_units = OS.fromList $ concat $ do - x <- map errMsgDiagnostic closure_errs - DriverHomePackagesNotClosed us <- pure x - pure us - isBad ci = (homeUnitId_ (componentDynFlags ci)) `OS.member` bad_units - -- Whenever we spin up a session on Linux, dynamically load libm.so.6 - -- in. We need this in case the binary is statically linked, in which - -- case the interactive session will fail when trying to load - -- ghc-prim, which happens whenever Template Haskell is being - -- evaluated or haskell-language-server's eval plugin tries to run - -- some code. If the binary is dynamically linked, then this will have - -- no effect. - -- See https://github.com/haskell/haskell-language-server/issues/221 - -- We need to do this after the call to setSessionDynFlags initialises - -- the loader - when (os == "linux") $ do - initObjLinker hscEnv' - res <- loadDLL hscEnv' "libm.so.6" - case res of - Nothing -> pure () - Just err -> logWith recorder Error $ LogDLLLoadError err - - forM (Map.elems cis) $ \ci -> do - let df = componentDynFlags ci - thisEnv <- do - -- In GHC 9.4 we have multi component support, and we have initialised all the units - -- above. - -- We just need to set the current unit here - pure $ hscSetActiveUnitId (homeUnitId_ df) hscEnv' - henv <- newHscEnvEq thisEnv - let targetEnv = (if isBad ci then multi_errs else [], Just henv) - targetDepends = componentDependencyInfo ci - logWith recorder Debug $ LogNewComponentCache (targetEnv, targetDepends) - evaluate $ liftRnf rwhnf $ componentTargets ci - - let mk t = fromTargetId (importPaths df) exts (targetId t) targetEnv targetDepends - ctargets <- concatMapM mk (componentTargets ci) - - return (L.nubOrdOn targetTarget ctargets) - -{- Note [Avoiding bad interface files] -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Originally, we set the cache directory for the various components once -on the first occurrence of the component. -This works fine if these components have no references to each other, -but you have components that depend on each other, the interface files are -updated for each component. -After restarting the session and only opening the component that depended -on the other, suddenly the interface files of this component are stale. -However, from the point of view of `ghcide`, they do not look stale, -thus, not regenerated and the IDE shows weird errors such as: -``` -typecheckIface -Declaration for Rep_ClientRunFlags -Axiom branches Rep_ClientRunFlags: - Failed to load interface for ‘Distribution.Simple.Flag’ - Use -v to see a list of the files searched for. -``` -and -``` -expectJust checkFamInstConsistency -CallStack (from HasCallStack): - error, called at compiler\\utils\\Maybes.hs:55:27 in ghc:Maybes - expectJust, called at compiler\\typecheck\\FamInst.hs:461:30 in ghc:FamInst -``` - -and many more. - -To mitigate this, we set the cache directory for each component dependent -on the components of the current `HscEnv`, additionally to the component options -of the respective components. -Assume two components, c1, c2, where c2 depends on c1, and the options of the -respective components are co1, co2. -If we want to load component c2, followed by c1, we set the cache directory for -each component in this way: - - * Load component c2 - * (Cache Directory State) - - name of c2 + co2 - * Load component c1 - * (Cache Directory State) - - name of c2 + name of c1 + co2 - - name of c2 + name of c1 + co1 - -Overall, we created three cache directories. If we opened c1 first, then we -create a fourth cache directory. -This makes sure that interface files are always correctly updated. - -Since this causes a lot of recompilation, we only update the cache-directory, -if the dependencies of a component have really changed. -E.g. when you load two executables, they can not depend on each other. They -should be filtered out, such that we dont have to re-compile everything. --} - --- | Set the cache-directory based on the ComponentOptions and a list of --- internal packages. --- For the exact reason, see Note [Avoiding bad interface files]. -setCacheDirs :: MonadUnliftIO m => Recorder (WithPriority Log) -> CacheDirs -> DynFlags -> m DynFlags -setCacheDirs recorder CacheDirs{..} dflags = do - logWith recorder Info $ LogInterfaceFilesCacheDir (fromMaybe cacheDir hiCacheDir) - pure $ dflags - & maybe id setHiDir hiCacheDir - & maybe id setHieDir hieCacheDir - & maybe id setODir oCacheDir - -- See Note [Multi Cradle Dependency Info] -type DependencyInfo = Map.Map FilePath (Maybe UTCTime) type HieMap = Map.Map (Maybe FilePath) [RawComponentInfo] -- | Maps a "hie.yaml" location to all its Target Filepaths and options. -type FlagsMap = Map.Map (Maybe FilePath) (HM.HashMap NormalizedFilePath (IdeResult HscEnvEq, DependencyInfo)) +type FlagsMap = STM.Map (Maybe FilePath) (HM.HashMap NormalizedFilePath (IdeResult HscEnvEq, DependencyInfo)) -- | Maps a Filepath to its respective "hie.yaml" location. -- It aims to be the reverse of 'FlagsMap'. -type FilesMap = HM.HashMap NormalizedFilePath (Maybe FilePath) - --- This is pristine information about a component -data RawComponentInfo = RawComponentInfo - { rawComponentUnitId :: UnitId - -- | Unprocessed DynFlags. Contains inplace packages such as libraries. - -- We do not want to use them unprocessed. - , rawComponentDynFlags :: DynFlags - -- | All targets of this components. - , rawComponentTargets :: [GHC.Target] - -- | Filepath which caused the creation of this component - , rawComponentFP :: NormalizedFilePath - -- | Component Options used to load the component. - , rawComponentCOptions :: ComponentOptions - -- | Maps cradle dependencies, such as `stack.yaml`, or `.cabal` file - -- to last modification time. See Note [Multi Cradle Dependency Info]. - , rawComponentDependencyInfo :: DependencyInfo - } - --- This is processed information about the component, in particular the dynflags will be modified. -data ComponentInfo = ComponentInfo - { componentUnitId :: UnitId - -- | Processed DynFlags. Does not contain inplace packages such as local - -- libraries. Can be used to actually load this Component. - , componentDynFlags :: DynFlags - -- | All targets of this components. - , componentTargets :: [GHC.Target] - -- | Filepath which caused the creation of this component - , componentFP :: NormalizedFilePath - -- | Component Options used to load the component. - , componentCOptions :: ComponentOptions - -- | Maps cradle dependencies, such as `stack.yaml`, or `.cabal` file - -- to last modification time. See Note [Multi Cradle Dependency Info] - , componentDependencyInfo :: DependencyInfo - } - --- | Check if any dependency has been modified lately. -checkDependencyInfo :: DependencyInfo -> IO Bool -checkDependencyInfo old_di = do - di <- getDependencyInfo (Map.keys old_di) - return (di == old_di) - --- Note [Multi Cradle Dependency Info] --- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ --- Why do we implement our own file modification tracking here? --- The primary reason is that the custom caching logic is quite complicated and going into shake --- adds even more complexity and more indirection. I did try for about 5 hours to work out how to --- use shake rules rather than IO but eventually gave up. - --- | Computes a mapping from a filepath to its latest modification date. --- See Note [Multi Cradle Dependency Info] why we do this ourselves instead --- of letting shake take care of it. -getDependencyInfo :: [FilePath] -> IO DependencyInfo -getDependencyInfo fs = Map.fromList <$> mapM do_one fs - - where - safeTryIO :: IO a -> IO (Either IOException a) - safeTryIO = Safe.try - - do_one :: FilePath -> IO (FilePath, Maybe UTCTime) - do_one fp = (fp,) . eitherToMaybe <$> safeTryIO (getModificationTime fp) - --- | This function removes all the -package flags which refer to packages we --- are going to deal with ourselves. For example, if a executable depends --- on a library component, then this function will remove the library flag --- from the package flags for the executable --- --- There are several places in GHC (for example the call to hptInstances in --- tcRnImports) which assume that all modules in the HPT have the same unit --- ID. Therefore we create a fake one and give them all the same unit id. -_removeInplacePackages --Only used in ghc < 9.4 - :: UnitId -- ^ fake uid to use for our internal component - -> [UnitId] - -> DynFlags - -> (DynFlags, [UnitId]) -_removeInplacePackages fake_uid us df = (setHomeUnitId_ fake_uid $ - df { packageFlags = ps }, uids) - where - (uids, ps) = Compat.filterInplaceUnits us (packageFlags df) +type FilesMap = STM.Map NormalizedFilePath (Maybe FilePath) -- | Memoize an IO function, with the characteristics: -- @@ -1101,131 +1026,6 @@ memoIO op = do return (Map.insert k res mp, res) Just res -> return (mp, res) -unit_flags :: [Flag (CmdLineP [String])] -unit_flags = [defFlag "unit" (SepArg addUnit)] - -addUnit :: String -> EwM (CmdLineP [String]) () -addUnit unit_str = liftEwM $ do - units <- getCmdLineState - putCmdLineState (unit_str : units) - --- | Throws if package flags are unsatisfiable -setOptions :: GhcMonad m - => OptHaddockParse - -> NormalizedFilePath - -> ComponentOptions - -> DynFlags - -> FilePath -- ^ root dir, see Note [Root Directory] - -> m (NonEmpty (DynFlags, [GHC.Target])) -setOptions haddockOpt cfp (ComponentOptions theOpts compRoot _) dflags rootDir = do - ((theOpts',_errs,_warns),units) <- processCmdLineP unit_flags [] (map noLoc theOpts) - case NE.nonEmpty units of - Just us -> initMulti us - Nothing -> do - (df, targets) <- initOne (map unLoc theOpts') - -- A special target for the file which caused this wonderful - -- component to be created. In case the cradle doesn't list all the targets for - -- the component, in which case things will be horribly broken anyway. - -- - -- When we have a singleComponent that is caused to be loaded due to a - -- file, we assume the file is part of that component. This is useful - -- for bare GHC sessions, such as many of the ones used in the testsuite - -- - -- We don't do this when we have multiple components, because each - -- component better list all targets or there will be anarchy. - -- It is difficult to know which component to add our file to in - -- that case. - -- Multi unit arguments are likely to come from cabal, which - -- does list all targets. - -- - -- If we don't end up with a target for the current file in the end, then - -- we will report it as an error for that file - let abs_fp = toAbsolute rootDir (fromNormalizedFilePath cfp) - let special_target = Compat.mkSimpleTarget df abs_fp - pure $ (df, special_target : targets) :| [] - where - initMulti unitArgFiles = - forM unitArgFiles $ \f -> do - args <- liftIO $ expandResponse [f] - -- The reponse files may contain arguments like "+RTS", - -- and hie-bios doesn't expand the response files of @-unit@ arguments. - -- Thus, we need to do the stripping here. - initOne $ HieBios.removeRTS $ HieBios.removeVerbosityOpts args - initOne this_opts = do - (dflags', targets') <- addCmdOpts this_opts dflags - let dflags'' = - case unitIdString (homeUnitId_ dflags') of - -- cabal uses main for the unit id of all executable packages - -- This makes multi-component sessions confused about what - -- options to use for that component. - -- Solution: hash the options and use that as part of the unit id - -- This works because there won't be any dependencies on the - -- executable unit. - "main" -> - let hash = B.unpack $ B16.encode $ H.finalize $ H.updates H.init (map B.pack this_opts) - hashed_uid = Compat.toUnitId (Compat.stringToUnit ("main-"++hash)) - in setHomeUnitId_ hashed_uid dflags' - _ -> dflags' - - let targets = makeTargetsAbsolute root targets' - root = case workingDirectory dflags'' of - Nothing -> compRoot - Just wdir -> compRoot wdir - let dflags''' = - setWorkingDirectory root $ - disableWarningsAsErrors $ - -- disabled, generated directly by ghcide instead - flip gopt_unset Opt_WriteInterface $ - -- disabled, generated directly by ghcide instead - -- also, it can confuse the interface stale check - dontWriteHieFiles $ - setIgnoreInterfacePragmas $ - setBytecodeLinkerOptions $ - enableOptHaddock haddockOpt $ - disableOptimisation $ - Compat.setUpTypedHoles $ - makeDynFlagsAbsolute compRoot -- makeDynFlagsAbsolute already accounts for workingDirectory - dflags'' - return (dflags''', targets) - -setIgnoreInterfacePragmas :: DynFlags -> DynFlags -setIgnoreInterfacePragmas df = - gopt_set (gopt_set df Opt_IgnoreInterfacePragmas) Opt_IgnoreOptimChanges - -disableOptimisation :: DynFlags -> DynFlags -disableOptimisation df = updOptLevel 0 df - --- | We always compile with '-haddock' unless explicitly disabled. --- --- This avoids inconsistencies when doing recompilation checking which was --- observed in https://github.com/haskell/haskell-language-server/issues/4511 -enableOptHaddock :: OptHaddockParse -> DynFlags -> DynFlags -enableOptHaddock HaddockParse d = gopt_set d Opt_Haddock -enableOptHaddock NoHaddockParse d = d - -setHiDir :: FilePath -> DynFlags -> DynFlags -setHiDir f d = - -- override user settings to avoid conflicts leading to recompilation - d { hiDir = Just f} - -setODir :: FilePath -> DynFlags -> DynFlags -setODir f d = - -- override user settings to avoid conflicts leading to recompilation - d { objectDir = Just f} - -getCacheDirsDefault :: String -> [String] -> IO CacheDirs -getCacheDirsDefault prefix opts = do - dir <- Just <$> getXdgDirectory XdgCache (cacheDir prefix ++ "-" ++ opts_hash) - return $ CacheDirs dir dir dir - where - -- Create a unique folder per set of different GHC options, assuming that each different set of - -- GHC options will create incompatible interface files. - opts_hash = B.unpack $ B16.encode $ H.finalize $ H.updates H.init (map B.pack opts) - --- | Sub directory for the cache path -cacheDir :: String -cacheDir = "ghcide" - ---------------------------------------------------------------------------------------------------- data PackageSetupException @@ -1236,7 +1036,7 @@ data PackageSetupException { compileTime :: !Version , runTime :: !Version } - deriving (Eq, Show, Typeable) + deriving (Eq, Show) instance Exception PackageSetupException diff --git a/ghcide/session-loader/Development/IDE/Session/Dependency.hs b/ghcide/session-loader/Development/IDE/Session/Dependency.hs new file mode 100644 index 0000000000..deedf809b8 --- /dev/null +++ b/ghcide/session-loader/Development/IDE/Session/Dependency.hs @@ -0,0 +1,35 @@ +module Development.IDE.Session.Dependency where + +import Control.Exception.Safe as Safe +import Data.Either.Extra +import qualified Data.Map.Strict as Map +import Data.Time.Clock +import System.Directory + +type DependencyInfo = Map.Map FilePath (Maybe UTCTime) + +-- | Check if any dependency has been modified lately. +checkDependencyInfo :: DependencyInfo -> IO Bool +checkDependencyInfo old_di = do + di <- getDependencyInfo (Map.keys old_di) + return (di == old_di) + +-- Note [Multi Cradle Dependency Info] +-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-- Why do we implement our own file modification tracking here? +-- The primary reason is that the custom caching logic is quite complicated and going into shake +-- adds even more complexity and more indirection. I did try for about 5 hours to work out how to +-- use shake rules rather than IO but eventually gave up. + +-- | Computes a mapping from a filepath to its latest modification date. +-- See Note [Multi Cradle Dependency Info] why we do this ourselves instead +-- of letting shake take care of it. +getDependencyInfo :: [FilePath] -> IO DependencyInfo +getDependencyInfo fs = Map.fromList <$> mapM do_one fs + + where + safeTryIO :: IO a -> IO (Either IOException a) + safeTryIO = Safe.try + + do_one :: FilePath -> IO (FilePath, Maybe UTCTime) + do_one fp = (fp,) . eitherToMaybe <$> safeTryIO (getModificationTime fp) diff --git a/ghcide/session-loader/Development/IDE/Session/Ghc.hs b/ghcide/session-loader/Development/IDE/Session/Ghc.hs new file mode 100644 index 0000000000..76db75fabe --- /dev/null +++ b/ghcide/session-loader/Development/IDE/Session/Ghc.hs @@ -0,0 +1,540 @@ +{-# LANGUAGE CPP #-} +module Development.IDE.Session.Ghc where + +import Control.Monad +import Control.Monad.Extra as Extra +import Control.Monad.IO.Class +import qualified Crypto.Hash.SHA1 as H +import qualified Data.ByteString.Base16 as B16 +import qualified Data.ByteString.Char8 as B +import Data.Function +import Data.List +import Data.List.Extra as L +import Data.List.NonEmpty (NonEmpty (..)) +import qualified Data.List.NonEmpty as NE +import qualified Data.Map.Strict as Map +import Data.Maybe +import qualified Data.Text as T +import Development.IDE.Core.Shake hiding (Log, knownTargets, + withHieDb) +import qualified Development.IDE.GHC.Compat as Compat +import Development.IDE.GHC.Compat.CmdLine +import Development.IDE.GHC.Compat.Core hiding (Target, TargetFile, + TargetModule, Var, Warning, + getOptions) +import qualified Development.IDE.GHC.Compat.Core as GHC +import Development.IDE.GHC.Compat.Env hiding (Logger) +import Development.IDE.GHC.Compat.Units (UnitId) +import Development.IDE.GHC.Util +import Development.IDE.Types.Diagnostics +import Development.IDE.Types.HscEnvEq (HscEnvEq, newHscEnvEq) +import Development.IDE.Types.Location +import GHC.ResponseFile +import qualified HIE.Bios.Cradle.Utils as HieBios +import HIE.Bios.Environment hiding (getCacheDir) +import HIE.Bios.Types hiding (Log) +import Ide.Logger (Pretty (pretty), + Priority (Debug, Error, Info), + Recorder, WithPriority, + logWith, viaShow, (<+>)) +import System.Directory +import System.FilePath +import System.Info + + +import Control.DeepSeq +import Control.Exception (evaluate, mask_) +import Control.Monad.IO.Unlift (MonadUnliftIO) +import qualified Data.Set as OS +import qualified Development.IDE.GHC.Compat.Util as Compat +import Development.IDE.Session.Dependency +import Development.IDE.Types.Options +import GHC.Data.Graph.Directed +import Ide.PluginUtils (toAbsolute) + +import GHC.Driver.Env (hsc_all_home_unit_ids) +import GHC.Driver.Errors.Types +import GHC.Types.Error (errMsgDiagnostic, + singleMessage) +import GHC.Unit.State + +data Log + = LogInterfaceFilesCacheDir !FilePath + | LogMakingNewHscEnv ![UnitId] + | LogNewComponentCache !(([FileDiagnostic], Maybe HscEnvEq), DependencyInfo) + | LogDLLLoadError !String +deriving instance Show Log + +instance Pretty Log where + pretty = \case + LogInterfaceFilesCacheDir path -> + "Interface files cache directory:" <+> pretty path + LogMakingNewHscEnv inPlaceUnitIds -> + "Making new HscEnv. In-place unit ids:" <+> pretty (map show inPlaceUnitIds) + LogNewComponentCache componentCache -> + "New component cache HscEnvEq:" <+> viaShow componentCache + LogDLLLoadError errorString -> + "Error dynamically loading libm.so.6:" <+> pretty errorString + +-- This is pristine information about a component +data RawComponentInfo = RawComponentInfo + { rawComponentUnitId :: UnitId + -- | Unprocessed DynFlags. Contains inplace packages such as libraries. + -- We do not want to use them unprocessed. + , rawComponentDynFlags :: DynFlags + -- | All targets of this components. + , rawComponentTargets :: [GHC.Target] + -- | Filepath which caused the creation of this component + , rawComponentFP :: NormalizedFilePath + -- | Component Options used to load the component. + , rawComponentCOptions :: ComponentOptions + -- | Maps cradle dependencies, such as `stack.yaml`, or `.cabal` file + -- to last modification time. See Note [Multi Cradle Dependency Info]. + , rawComponentDependencyInfo :: DependencyInfo + } + +-- This is processed information about the component, in particular the dynflags will be modified. +data ComponentInfo = ComponentInfo + { componentUnitId :: UnitId + -- | Processed DynFlags. Does not contain inplace packages such as local + -- libraries. Can be used to actually load this Component. + , componentDynFlags :: DynFlags + -- | All targets of this components. + , componentTargets :: [GHC.Target] + -- | Filepath which caused the creation of this component + , componentFP :: NormalizedFilePath + -- | Component Options used to load the component. + , componentCOptions :: ComponentOptions + -- | Maps cradle dependencies, such as `stack.yaml`, or `.cabal` file + -- to last modification time. See Note [Multi Cradle Dependency Info] + , componentDependencyInfo :: DependencyInfo + } + + +unit_flags :: [Flag (CmdLineP [String])] +unit_flags = [defFlag "unit" (SepArg addUnit)] + +addUnit :: String -> EwM (CmdLineP [String]) () +addUnit unit_str = liftEwM $ do + units <- getCmdLineState + putCmdLineState (unit_str : units) + + +-- | Create a mapping from FilePaths to HscEnvEqs +-- This combines all the components we know about into +-- an appropriate session, which is a multi component +-- session on GHC 9.4+ +newComponentCache + :: Recorder (WithPriority Log) + -> [String] -- ^ File extensions to consider + -> NormalizedFilePath -- ^ Path to file that caused the creation of this component + -> HscEnv -- ^ An empty HscEnv + -> [ComponentInfo] -- ^ New components to be loaded + -> [ComponentInfo] -- ^ old, already existing components + -> IO [ [TargetDetails] ] +newComponentCache recorder exts _cfp hsc_env old_cis new_cis = do + let cis = Map.unionWith unionCIs (mkMap new_cis) (mkMap old_cis) + -- When we have multiple components with the same uid, + -- prefer the new one over the old. + -- However, we might have added some targets to the old unit + -- (see special target), so preserve those + unionCIs new_ci old_ci = new_ci { componentTargets = componentTargets new_ci ++ componentTargets old_ci } + mkMap = Map.fromListWith unionCIs . map (\ci -> (componentUnitId ci, ci)) + let dfs = map componentDynFlags $ Map.elems cis + uids = Map.keys cis + logWith recorder Info $ LogMakingNewHscEnv uids + hscEnv' <- -- Set up a multi component session with the other units on GHC 9.4 + Compat.initUnits dfs hsc_env + + let closure_errs = maybeToList $ checkHomeUnitsClosed' (hsc_unit_env hscEnv') (hsc_all_home_unit_ids hscEnv') + closure_err_to_multi_err err = + ideErrorWithSource + (Just "cradle") (Just DiagnosticSeverity_Warning) _cfp + (T.pack (Compat.printWithoutUniques (singleMessage err))) + (Just (fmap GhcDriverMessage err)) + multi_errs = map closure_err_to_multi_err closure_errs + bad_units = OS.fromList $ concat $ do + x <- map errMsgDiagnostic closure_errs + DriverHomePackagesNotClosed us <- pure x + pure us + isBad ci = (homeUnitId_ (componentDynFlags ci)) `OS.member` bad_units + -- Whenever we spin up a session on Linux, dynamically load libm.so.6 + -- in. We need this in case the binary is statically linked, in which + -- case the interactive session will fail when trying to load + -- ghc-prim, which happens whenever Template Haskell is being + -- evaluated or haskell-language-server's eval plugin tries to run + -- some code. If the binary is dynamically linked, then this will have + -- no effect. + -- See https://github.com/haskell/haskell-language-server/issues/221 + -- We need to do this after the call to setSessionDynFlags initialises + -- the loader + when (os == "linux") $ do + initObjLinker hscEnv' + res <- loadDLL hscEnv' "libm.so.6" + case res of + Nothing -> pure () + Just err -> logWith recorder Error $ LogDLLLoadError err + + forM (Map.elems cis) $ \ci -> do + let df = componentDynFlags ci + thisEnv <- do + -- In GHC 9.4 we have multi component support, and we have initialised all the units + -- above. + -- We just need to set the current unit here + pure $ hscSetActiveUnitId (homeUnitId_ df) hscEnv' + henv <- newHscEnvEq thisEnv + let targetEnv = (if isBad ci then multi_errs else [], Just henv) + targetDepends = componentDependencyInfo ci + logWith recorder Debug $ LogNewComponentCache (targetEnv, targetDepends) + evaluate $ liftRnf rwhnf $ componentTargets ci + + let mk t = fromTargetId (importPaths df) exts (targetId t) targetEnv targetDepends + ctargets <- concatMapM mk (componentTargets ci) + + return (L.nubOrdOn targetTarget ctargets) + +-- | Throws if package flags are unsatisfiable +setOptions :: GhcMonad m + => OptHaddockParse + -> NormalizedFilePath + -> ComponentOptions + -> DynFlags + -> FilePath -- ^ root dir, see Note [Root Directory] + -> m (NonEmpty (DynFlags, [GHC.Target])) +setOptions haddockOpt cfp (ComponentOptions theOpts compRoot _) dflags rootDir = do + ((theOpts',_errs,_warns),units) <- processCmdLineP unit_flags [] (map noLoc theOpts) + case NE.nonEmpty units of + Just us -> initMulti us + Nothing -> do + (df, targets) <- initOne (map unLoc theOpts') + -- A special target for the file which caused this wonderful + -- component to be created. In case the cradle doesn't list all the targets for + -- the component, in which case things will be horribly broken anyway. + -- + -- When we have a singleComponent that is caused to be loaded due to a + -- file, we assume the file is part of that component. This is useful + -- for bare GHC sessions, such as many of the ones used in the testsuite + -- + -- We don't do this when we have multiple components, because each + -- component better list all targets or there will be anarchy. + -- It is difficult to know which component to add our file to in + -- that case. + -- Multi unit arguments are likely to come from cabal, which + -- does list all targets. + -- + -- If we don't end up with a target for the current file in the end, then + -- we will report it as an error for that file + let abs_fp = toAbsolute rootDir (fromNormalizedFilePath cfp) + let special_target = Compat.mkSimpleTarget df abs_fp + pure $ (df, special_target : targets) :| [] + where + initMulti unitArgFiles = + forM unitArgFiles $ \f -> do + args <- liftIO $ expandResponse [f] + -- The reponse files may contain arguments like "+RTS", + -- and hie-bios doesn't expand the response files of @-unit@ arguments. + -- Thus, we need to do the stripping here. + initOne $ HieBios.removeRTS $ HieBios.removeVerbosityOpts args + initOne this_opts = do + (dflags', targets') <- addCmdOpts this_opts dflags + let dflags'' = + case unitIdString (homeUnitId_ dflags') of + -- cabal uses main for the unit id of all executable packages + -- This makes multi-component sessions confused about what + -- options to use for that component. + -- Solution: hash the options and use that as part of the unit id + -- This works because there won't be any dependencies on the + -- executable unit. + "main" -> + let hash = B.unpack $ B16.encode $ H.finalize $ H.updates H.init (map B.pack this_opts) + hashed_uid = Compat.toUnitId (Compat.stringToUnit ("main-"++hash)) + in setHomeUnitId_ hashed_uid dflags' + _ -> dflags' + + let targets = makeTargetsAbsolute root targets' + root = case workingDirectory dflags'' of + Nothing -> compRoot + Just wdir -> compRoot wdir + let dflags''' = + setWorkingDirectory root $ + disableWarningsAsErrors $ + -- disabled, generated directly by ghcide instead + flip gopt_unset Opt_WriteInterface $ + -- disabled, generated directly by ghcide instead + -- also, it can confuse the interface stale check + dontWriteHieFiles $ + setIgnoreInterfacePragmas $ + setBytecodeLinkerOptions $ + enableOptHaddock haddockOpt $ + disableOptimisation $ + Compat.setUpTypedHoles $ + makeDynFlagsAbsolute compRoot -- makeDynFlagsAbsolute already accounts for workingDirectory + dflags'' + return (dflags''', targets) + +addComponentInfo :: + MonadUnliftIO m => + Recorder (WithPriority Log) -> + (String -> [String] -> IO CacheDirs) -> + DependencyInfo -> + NonEmpty (DynFlags, [GHC.Target]) -> + (Maybe FilePath, NormalizedFilePath, ComponentOptions) -> + Map.Map (Maybe FilePath) [RawComponentInfo] -> + m (Map.Map (Maybe FilePath) [RawComponentInfo], ([ComponentInfo], [ComponentInfo])) +addComponentInfo recorder getCacheDirs dep_info newDynFlags (hieYaml, cfp, opts) m = do + -- Just deps if there's already an HscEnv + -- Nothing is it's the first time we are making an HscEnv + let oldDeps = Map.lookup hieYaml m + let -- Add the raw information about this component to the list + -- We will modify the unitId and DynFlags used for + -- compilation but these are the true source of + -- information. + new_deps = fmap (\(df, targets) -> RawComponentInfo (homeUnitId_ df) df targets cfp opts dep_info) newDynFlags + all_deps = new_deps `NE.appendList` fromMaybe [] oldDeps + -- Get all the unit-ids for things in this component + + all_deps' <- forM all_deps $ \RawComponentInfo{..} -> do + let prefix = show rawComponentUnitId + -- See Note [Avoiding bad interface files] + let cacheDirOpts = componentOptions opts + cacheDirs <- liftIO $ getCacheDirs prefix cacheDirOpts + processed_df <- setCacheDirs recorder cacheDirs rawComponentDynFlags + -- The final component information, mostly the same but the DynFlags don't + -- contain any packages which are also loaded + -- into the same component. + pure $ ComponentInfo + { componentUnitId = rawComponentUnitId + , componentDynFlags = processed_df + , componentTargets = rawComponentTargets + , componentFP = rawComponentFP + , componentCOptions = rawComponentCOptions + , componentDependencyInfo = rawComponentDependencyInfo + } + -- Modify the map so the hieYaml now maps to the newly updated + -- ComponentInfos + -- Returns + -- . The information for the new component which caused this cache miss + -- . The modified information (without -inplace flags) for + -- existing packages + let (new,old) = NE.splitAt (NE.length new_deps) all_deps' + pure (Map.insert hieYaml (NE.toList all_deps) m, (new,old)) + +setIgnoreInterfacePragmas :: DynFlags -> DynFlags +setIgnoreInterfacePragmas df = + gopt_set (gopt_set df Opt_IgnoreInterfacePragmas) Opt_IgnoreOptimChanges + +disableOptimisation :: DynFlags -> DynFlags +disableOptimisation df = updOptLevel 0 df + +-- | We always compile with '-haddock' unless explicitly disabled. +-- +-- This avoids inconsistencies when doing recompilation checking which was +-- observed in https://github.com/haskell/haskell-language-server/issues/4511 +enableOptHaddock :: OptHaddockParse -> DynFlags -> DynFlags +enableOptHaddock HaddockParse d = gopt_set d Opt_Haddock +enableOptHaddock NoHaddockParse d = d + +setHiDir :: FilePath -> DynFlags -> DynFlags +setHiDir f d = + -- override user settings to avoid conflicts leading to recompilation + d { hiDir = Just f} + +setODir :: FilePath -> DynFlags -> DynFlags +setODir f d = + -- override user settings to avoid conflicts leading to recompilation + d { objectDir = Just f} + +data CacheDirs = CacheDirs + { hiCacheDir, hieCacheDir, oCacheDir :: Maybe FilePath} + +{- Note [Avoiding bad interface files] +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Originally, we set the cache directory for the various components once +on the first occurrence of the component. +This works fine if these components have no references to each other, +but you have components that depend on each other, the interface files are +updated for each component. +After restarting the session and only opening the component that depended +on the other, suddenly the interface files of this component are stale. +However, from the point of view of `ghcide`, they do not look stale, +thus, not regenerated and the IDE shows weird errors such as: +``` +typecheckIface +Declaration for Rep_ClientRunFlags +Axiom branches Rep_ClientRunFlags: + Failed to load interface for ‘Distribution.Simple.Flag’ + Use -v to see a list of the files searched for. +``` +and +``` +expectJust checkFamInstConsistency +CallStack (from HasCallStack): + error, called at compiler\\utils\\Maybes.hs:55:27 in ghc:Maybes + expectJust, called at compiler\\typecheck\\FamInst.hs:461:30 in ghc:FamInst +``` + +and many more. + +To mitigate this, we set the cache directory for each component dependent +on the components of the current `HscEnv`, additionally to the component options +of the respective components. +Assume two components, c1, c2, where c2 depends on c1, and the options of the +respective components are co1, co2. +If we want to load component c2, followed by c1, we set the cache directory for +each component in this way: + + * Load component c2 + * (Cache Directory State) + - name of c2 + co2 + * Load component c1 + * (Cache Directory State) + - name of c2 + name of c1 + co2 + - name of c2 + name of c1 + co1 + +Overall, we created three cache directories. If we opened c1 first, then we +create a fourth cache directory. +This makes sure that interface files are always correctly updated. + +Since this causes a lot of recompilation, we only update the cache-directory, +if the dependencies of a component have really changed. +E.g. when you load two executables, they can not depend on each other. They +should be filtered out, such that we dont have to re-compile everything. +-} + + +-- | Set the cache-directory based on the ComponentOptions and a list of +-- internal packages. +-- For the exact reason, see Note [Avoiding bad interface files]. +setCacheDirs :: MonadUnliftIO m => Recorder (WithPriority Log) -> CacheDirs -> DynFlags -> m DynFlags +setCacheDirs recorder CacheDirs{..} dflags = do + logWith recorder Info $ LogInterfaceFilesCacheDir (fromMaybe cacheDir hiCacheDir) + pure $ dflags + & maybe id setHiDir hiCacheDir + & maybe id setHieDir hieCacheDir + & maybe id setODir oCacheDir + +getCacheDirsDefault :: String -> [String] -> IO CacheDirs +getCacheDirsDefault prefix opts = do + dir <- Just <$> getXdgDirectory XdgCache (cacheDir prefix ++ "-" ++ opts_hash) + return $ CacheDirs dir dir dir + where + -- Create a unique folder per set of different GHC options, assuming that each different set of + -- GHC options will create incompatible interface files. + opts_hash = B.unpack $ B16.encode $ H.finalize $ H.updates H.init (map B.pack opts) + +setNameCache :: NameCache -> HscEnv -> HscEnv +setNameCache nc hsc = hsc { hsc_NC = nc } + +-- | Sub directory for the cache path +cacheDir :: String +cacheDir = "ghcide" + +emptyHscEnv :: NameCache -> FilePath -> IO HscEnv +emptyHscEnv nc libDir = do + -- We call setSessionDynFlags so that the loader is initialised + -- We need to do this before we call initUnits. + env <- mask_ $ liftIO $ runGhc (Just libDir) $ + getSessionDynFlags >>= setSessionDynFlags >> getSession + pure $ setNameCache nc (hscSetFlags ((hsc_dflags env){useUnicode = True }) env) + +-- ---------------------------------------------------------------------------- +-- Target Details +-- ---------------------------------------------------------------------------- + +data TargetDetails = TargetDetails + { + targetTarget :: !Target, + targetEnv :: !(IdeResult HscEnvEq), + targetDepends :: !DependencyInfo, + targetLocations :: ![NormalizedFilePath] + } + +fromTargetId :: [FilePath] -- ^ import paths + -> [String] -- ^ extensions to consider + -> TargetId + -> IdeResult HscEnvEq + -> DependencyInfo + -> IO [TargetDetails] +-- For a target module we consider all the import paths +fromTargetId is exts (GHC.TargetModule modName) env dep = do + let fps = [i moduleNameSlashes modName -<.> ext <> boot + | ext <- exts + , i <- is + , boot <- ["", "-boot"] + ] + let locs = fmap toNormalizedFilePath' fps + return [TargetDetails (TargetModule modName) env dep locs] +-- For a 'TargetFile' we consider all the possible module names +fromTargetId _ _ (GHC.TargetFile f _) env deps = do + let nf = toNormalizedFilePath' f + let other + | "-boot" `isSuffixOf` f = toNormalizedFilePath' (L.dropEnd 5 $ fromNormalizedFilePath nf) + | otherwise = toNormalizedFilePath' (fromNormalizedFilePath nf ++ "-boot") + return [TargetDetails (TargetFile nf) env deps [nf, other]] + +-- ---------------------------------------------------------------------------- +-- Backwards compatibility +-- ---------------------------------------------------------------------------- + +#if MIN_VERSION_ghc(9,13,0) +-- Moved back to implementation in GHC. +checkHomeUnitsClosed' :: UnitEnv -> OS.Set UnitId -> [DriverMessages] +checkHomeUnitsClosed' ue _ = checkHomeUnitsClosed ue +#else +-- This function checks the important property that if both p and q are home units +-- then any dependency of p, which transitively depends on q is also a home unit. +-- GHC had an implementation of this function, but it was horribly inefficient +-- We should move back to the GHC implementation on compilers where +-- https://gitlab.haskell.org/ghc/ghc/-/merge_requests/12162 is included +checkHomeUnitsClosed' :: UnitEnv -> OS.Set UnitId -> Maybe (Compat.MsgEnvelope DriverMessage) +checkHomeUnitsClosed' ue home_id_set + | OS.null bad_unit_ids = Nothing + | otherwise = Just (GHC.mkPlainErrorMsgEnvelope rootLoc $ DriverHomePackagesNotClosed (OS.toList bad_unit_ids)) + where + bad_unit_ids = upwards_closure OS.\\ home_id_set + rootLoc = mkGeneralSrcSpan (Compat.fsLit "") + + graph :: Graph (Node UnitId UnitId) + graph = graphFromEdgedVerticesUniq graphNodes + + -- downwards closure of graph + downwards_closure + = graphFromEdgedVerticesUniq [ DigraphNode uid uid (OS.toList deps) + | (uid, deps) <- Map.toList (allReachable graph node_key)] + + inverse_closure = transposeG downwards_closure + + upwards_closure = OS.fromList $ map node_key $ reachablesG inverse_closure [DigraphNode uid uid [] | uid <- OS.toList home_id_set] + + all_unit_direct_deps :: UniqMap UnitId (OS.Set UnitId) + all_unit_direct_deps + = unitEnv_foldWithKey go emptyUniqMap $ ue_home_unit_graph ue + where + go rest this this_uis = + plusUniqMap_C OS.union + (addToUniqMap_C OS.union external_depends this (OS.fromList this_deps)) + rest + where + external_depends = mapUniqMap (OS.fromList . unitDepends) +#if !MIN_VERSION_ghc(9,7,0) + $ listToUniqMap $ Map.toList +#endif + + $ unitInfoMap this_units + this_units = homeUnitEnv_units this_uis + this_deps = [ Compat.toUnitId unit | (unit,Just _) <- explicitUnits this_units] + + graphNodes :: [Node UnitId UnitId] + graphNodes = go OS.empty home_id_set + where + go done todo + = case OS.minView todo of + Nothing -> [] + Just (uid, todo') + | OS.member uid done -> go done todo' + | otherwise -> case lookupUniqMap all_unit_direct_deps uid of + Nothing -> pprPanic "uid not found" (Compat.ppr (uid, all_unit_direct_deps)) + Just depends -> + let todo'' = (depends OS.\\ done) `OS.union` todo' + in DigraphNode uid uid (OS.toList depends) : go (OS.insert uid done) todo'' +#endif diff --git a/ghcide/session-loader/Development/IDE/Session/OrderedSet.hs b/ghcide/session-loader/Development/IDE/Session/OrderedSet.hs new file mode 100644 index 0000000000..630f1dc4fc --- /dev/null +++ b/ghcide/session-loader/Development/IDE/Session/OrderedSet.hs @@ -0,0 +1,54 @@ +module Development.IDE.Session.OrderedSet where + +import Control.Concurrent.STM (STM, TQueue, newTQueueIO) +import Control.Concurrent.STM.TQueue (readTQueue, writeTQueue) +import Control.Monad (when) +import Data.Hashable (Hashable) +import qualified Data.HashSet +import qualified Focus +import qualified ListT as LT +import qualified StmContainers.Set as S +import StmContainers.Set (Set) + + +data OrderedSet a = OrderedSet + { insertionOrder :: TQueue a + , elements :: Set a + } + +-- | Insert an element into the ordered set. +-- If the element is not already present, it is added to both the queue and set. +-- If the element already exists, ignore it +insert :: Hashable a => a -> OrderedSet a -> STM () +insert a (OrderedSet que s) = do + (_, inserted) <- S.focus (Focus.testingIfInserts $ Focus.insert ()) a s + -- if already in the set + when inserted $ writeTQueue que a + +newIO :: Hashable a => IO (OrderedSet a) +newIO = do + que <- newTQueueIO + s <- S.newIO + return (OrderedSet que s) + +-- | Read the first element from the queue. +-- If an element is not in the set, it means it has been deleted, +-- so we retry until we find a valid element that exists in the set. +readQueue :: Hashable a => OrderedSet a -> STM a +readQueue rs@(OrderedSet que s) = do + f <- readTQueue que + b <- S.lookup f s + -- retry if no files are left in the queue + if b then return f else readQueue rs + +lookup :: Hashable a => a -> OrderedSet a -> STM Bool +lookup a (OrderedSet _ s) = S.lookup a s + +-- | Delete an element from the set. +-- The queue is not modified directly; stale entries are filtered out lazily +-- during reading operations (see 'readQueue'). +delete :: Hashable a => a -> OrderedSet a -> STM () +delete a (OrderedSet _ s) = S.delete a s + +toHashSet :: Hashable a => OrderedSet a -> STM (Data.HashSet.HashSet a) +toHashSet (OrderedSet _ s) = Data.HashSet.fromList <$> LT.toList (S.listT s) diff --git a/ghcide/src/Development/IDE/Core/Compile.hs b/ghcide/src/Development/IDE/Core/Compile.hs index 48439e2ff3..2b25fb08c0 100644 --- a/ghcide/src/Development/IDE/Core/Compile.hs +++ b/ghcide/src/Development/IDE/Core/Compile.hs @@ -114,11 +114,15 @@ import qualified GHC.Runtime.Loader as Loader import GHC.Tc.Gen.Splice import GHC.Types.Error import GHC.Types.ForeignStubs -import GHC.Types.HpcInfo import GHC.Types.TypeEnv +import Development.IDE.WorkerThread (writeTaskQueue) -- See Note [Guidelines For Using CPP In GHCIDE Import Statements] +#if !MIN_VERSION_ghc(9,11,0) +import GHC.Types.HpcInfo +#endif + #if MIN_VERSION_ghc(9,7,0) import Data.Foldable (toList) import GHC.Unit.Module.Warnings @@ -793,7 +797,8 @@ atomicFileWrite se targetPath write = do let dir = takeDirectory targetPath createDirectoryIfMissing True dir (tempFilePath, cleanUp) <- newTempFileWithin dir - (write tempFilePath >>= \x -> renameFile tempFilePath targetPath >> atomically (resetInterfaceStore se (toNormalizedFilePath' targetPath)) >> pure x) + (write tempFilePath >>= \x -> renameFile tempFilePath targetPath >> + atomically (resetInterfaceStore se (toNormalizedFilePath' targetPath)) >> pure x) `onException` cleanUp generateHieAsts :: HscEnv -> TcModuleResult @@ -882,7 +887,7 @@ indexHieFile se mod_summary srcPath !hash hf = do -- hiedb doesn't use the Haskell src, so we clear it to avoid unnecessarily keeping it around let !hf' = hf{hie_hs_src = mempty} modifyTVar' indexPending $ HashMap.insert srcPath hash - writeTQueue indexQueue $ \withHieDb -> do + writeTaskQueue indexQueue $ \withHieDb -> do -- We are now in the worker thread -- Check if a newer index of this file has been scheduled, and if so skip this one newerScheduled <- atomically $ do diff --git a/ghcide/src/Development/IDE/Core/FileStore.hs b/ghcide/src/Development/IDE/Core/FileStore.hs index e545ec7b14..0bdec3874e 100644 --- a/ghcide/src/Development/IDE/Core/FileStore.hs +++ b/ghcide/src/Development/IDE/Core/FileStore.hs @@ -25,7 +25,6 @@ module Development.IDE.Core.FileStore( ) where import Control.Concurrent.STM.Stats (STM, atomically) -import Control.Concurrent.STM.TQueue (writeTQueue) import Control.Exception import Control.Lens ((^.)) import Control.Monad.Extra @@ -52,6 +51,7 @@ import Development.IDE.Types.Diagnostics import Development.IDE.Types.Location import Development.IDE.Types.Options import Development.IDE.Types.Shake (toKey) +import Development.IDE.WorkerThread (writeTaskQueue) import HieDb.Create (deleteMissingRealFiles) import Ide.Logger (Pretty (pretty), Priority (Info), @@ -109,7 +109,7 @@ addWatchedFileRule recorder isWatched = defineNoDiagnostics (cmapWithPrio LogSha getModificationTimeRule :: Recorder (WithPriority Log) -> Rules () -getModificationTimeRule recorder = defineEarlyCutoff (cmapWithPrio LogShake recorder) $ Rule $ \(GetModificationTime_ missingFileDiags) file -> +getModificationTimeRule recorder = defineEarlyCutoff (cmapWithPrio LogShake recorder) $ Rule $ \(GetModificationTime_ missingFileDiags) file -> do getModificationTimeImpl missingFileDiags file getModificationTimeImpl @@ -252,8 +252,8 @@ getVersionedTextDoc doc = do maybe (pure Nothing) getVirtualFile $ uriToNormalizedFilePath $ toNormalizedUri uri let ver = case mvf of - Just (VirtualFile lspver _ _) -> lspver - Nothing -> 0 + Just (VirtualFile lspver _ _ _) -> lspver + Nothing -> 0 return (VersionedTextDocumentIdentifier uri ver) fileStoreRules :: Recorder (WithPriority Log) -> (NormalizedFilePath -> Action Bool) -> Rules () @@ -279,11 +279,9 @@ setFileModified recorder vfs state saved nfp actionBefore = do AlwaysCheck -> True CheckOnSave -> saved _ -> False - restartShakeSession (shakeExtras state) vfs (fromNormalizedFilePath nfp ++ " (modified)") [] $ do + restartShakeSession (shakeExtras state) vfs (fromNormalizedFilePath nfp ++ " (modified)") ([mkDelayedAction "ParentTC" L.Debug (typecheckParentsAction recorder nfp) | checkParents]) $ do keys<-actionBefore return (toKey GetModificationTime nfp:keys) - when checkParents $ - typecheckParents recorder state nfp typecheckParents :: Recorder (WithPriority Log) -> IdeState -> NormalizedFilePath -> IO () typecheckParents recorder state nfp = void $ shakeEnqueue (shakeExtras state) parents @@ -295,7 +293,7 @@ typecheckParentsAction recorder nfp = do case revs of Nothing -> logWith recorder Info $ LogCouldNotIdentifyReverseDeps nfp Just rs -> do - logWith recorder Info $ LogTypeCheckingReverseDeps nfp revs + logWith recorder L.Debug $ LogTypeCheckingReverseDeps nfp revs void $ uses GetModIface rs -- | Note that some keys have been modified and restart the session @@ -304,7 +302,7 @@ typecheckParentsAction recorder nfp = do setSomethingModified :: VFSModified -> IdeState -> String -> IO [Key] -> IO () setSomethingModified vfs state reason actionBetweenSession = do -- Update database to remove any files that might have been renamed/deleted - atomically $ writeTQueue (indexQueue $ hiedbWriter $ shakeExtras state) (\withHieDb -> withHieDb deleteMissingRealFiles) + atomically $ writeTaskQueue (indexQueue $ hiedbWriter $ shakeExtras state) (\withHieDb -> withHieDb deleteMissingRealFiles) void $ restartShakeSession (shakeExtras state) vfs reason [] actionBetweenSession registerFileWatches :: [String] -> LSP.LspT Config IO Bool diff --git a/ghcide/src/Development/IDE/Core/ProgressReporting.hs b/ghcide/src/Development/IDE/Core/ProgressReporting.hs index 3d8a2bf989..4bf4b10ab5 100644 --- a/ghcide/src/Development/IDE/Core/ProgressReporting.hs +++ b/ghcide/src/Development/IDE/Core/ProgressReporting.hs @@ -23,24 +23,31 @@ import Control.Concurrent.STM (STM) import Control.Concurrent.STM.Stats (TVar, atomically, atomicallyNamed, modifyTVar', newTVarIO, readTVar, retry) -import Control.Concurrent.Strict (modifyVar_, newVar, - threadDelay) +import Control.Concurrent.Strict (modifyVar_, newBarrier, newVar, + signalBarrier, threadDelay, + waitBarrier) import Control.Monad.Extra hiding (loop) import Control.Monad.IO.Class import Control.Monad.Trans.Class (lift) +import qualified Data.Aeson as J import Data.Functor (($>)) import qualified Data.Text as T +import Data.Unique (hashUnique, newUnique) import Development.IDE.GHC.Orphans () import Development.IDE.Types.Location import Development.IDE.Types.Options import qualified Focus +import Language.LSP.Protocol.Message import Language.LSP.Protocol.Types -import Language.LSP.Server (ProgressAmount (..), +import qualified Language.LSP.Protocol.Types as L +import Language.LSP.Server (MonadLsp, ProgressAmount (..), ProgressCancellable (..), + sendNotification, sendRequest, withProgress) import qualified Language.LSP.Server as LSP import qualified StmContainers.Map as STM import UnliftIO (Async, async, bracket, cancel) +import qualified UnliftIO.Exception as UE data ProgressEvent = ProgressNewStarted @@ -168,7 +175,7 @@ progressReportingNoTrace todo done (Just lspEnv) title optProgressStyle = do let _progressUpdate event = liftIO $ updateStateVar $ Event event _progressStop = updateStateVar StopProgress updateStateVar = modifyVar_ progressState . updateState (progressCounter lspEnv title optProgressStyle todo done) - return ProgressReporting {..} + return ProgressReporting {_progressUpdate, _progressStop} -- | `progressReporting` initiates a new progress reporting session. -- It necessitates the active tracking of progress using the `inProgress` function. @@ -196,6 +203,25 @@ progressReporting (Just lspEnv) title optProgressStyle = do f = recordProgress inProgress file +withProgressDummy :: + forall c m a. + MonadLsp c m => + T.Text -> + Maybe ProgressToken -> + ProgressCancellable -> + ((ProgressAmount -> m ()) -> m a) -> + m a +withProgressDummy title _ _ f = do + t <- L.ProgressToken . L.InR . T.pack . show . hashUnique <$> liftIO newUnique + r <- liftIO newBarrier + _ <- sendRequest SMethod_WindowWorkDoneProgressCreate (WorkDoneProgressCreateParams t) $ + \_ -> liftIO $ signalBarrier r () + -- liftIO $ waitBarrier r + sendProgressReport t $ WorkDoneProgressBegin L.AString title Nothing Nothing Nothing + f (const $ return ()) `UE.finally` sendProgressReport t (WorkDoneProgressEnd L.AString Nothing) + where + sendProgressReport token report = sendNotification SMethod_Progress $ ProgressParams token $ J.toJSON report + -- Kill this to complete the progress session progressCounter :: LSP.LanguageContextEnv c -> @@ -205,8 +231,12 @@ progressCounter :: STM Int -> IO () progressCounter lspEnv title optProgressStyle getTodo getDone = - LSP.runLspT lspEnv $ withProgress title Nothing NotCancellable $ \update -> loop update 0 + LSP.runLspT lspEnv $ withProgressChoice title Nothing NotCancellable $ \update -> loop update 0 where + withProgressChoice = case optProgressStyle of + TestReporting -> withProgressDummy + _ -> withProgress + loop _ _ | optProgressStyle == NoProgress = forever $ liftIO $ threadDelay maxBound loop update prevPct = do (todo, done, nextPct) <- liftIO $ atomically $ do diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs index 964d6d379b..b3293ce468 100644 --- a/ghcide/src/Development/IDE/Core/Rules.hs +++ b/ghcide/src/Development/IDE/Core/Rules.hs @@ -175,6 +175,7 @@ import System.Info.Extra (isWindows) import qualified Data.IntMap as IM import GHC.Fingerprint +import Debug.Trace (traceEventIO) data Log = LogShake Shake.Log @@ -516,8 +517,8 @@ persistentHieFileRule recorder = addPersistentRule GetHieAst $ \file -> runMaybe vfsRef <- asks vfsVar vfsData <- liftIO $ _vfsMap <$> readTVarIO vfsRef (currentSource, ver) <- liftIO $ case M.lookup (filePathToUri' file) vfsData of - Nothing -> (,Nothing) . T.decodeUtf8 <$> BS.readFile (fromNormalizedFilePath file) - Just vf -> pure (virtualFileText vf, Just $ virtualFileVersion vf) + Just (Open vf) -> pure (virtualFileText vf, Just $ virtualFileVersion vf) + _ -> (,Nothing) . T.decodeUtf8 <$> BS.readFile (fromNormalizedFilePath file) let refmap = generateReferencesMap . getAsts . Compat.hie_asts $ res del = deltaFromDiff (T.decodeUtf8 $ Compat.hie_hs_src res) currentSource pure (HAR (Compat.hie_module res) (Compat.hie_asts res) refmap mempty (HieFromDisk res),del,ver) @@ -722,7 +723,7 @@ loadGhcSession recorder ghcSessionDepsConfig = do itExists <- getFileExists nfp when itExists $ void $ do use_ GetPhysicalModificationTime nfp - logWith recorder Logger.Info $ LogDependencies file deps + logWith recorder Logger.Debug $ LogDependencies file deps mapM_ addDependency deps let cutoffHash = LBS.toStrict $ B.encode (hash (snd val)) diff --git a/ghcide/src/Development/IDE/Core/Shake.hs b/ghcide/src/Development/IDE/Core/Shake.hs index 6fc9a4d00e..bdfea5402e 100644 --- a/ghcide/src/Development/IDE/Core/Shake.hs +++ b/ghcide/src/Development/IDE/Core/Shake.hs @@ -25,7 +25,9 @@ module Development.IDE.Core.Shake( IdeState, shakeSessionInit, shakeExtras, shakeDb, rootDir, ShakeExtras(..), getShakeExtras, getShakeExtrasRules, KnownTargets(..), Target(..), toKnownFiles, unionKnownTargets, mkKnownTargets, - IdeRule, IdeResult, + ShakeRestartArgs(..), + shakeRestart, + IdeRule, IdeResult, ShakeControlQueue, GetModificationTime(GetModificationTime, GetModificationTime_, missingFileDiagnostics), shakeOpen, shakeShut, shakeEnqueue, @@ -76,118 +78,142 @@ module Development.IDE.Core.Shake( Log(..), VFSModified(..), getClientConfigAction, ThreadQueue(..), - runWithSignal + runWithSignal, runRestartTask, runRestartTaskDyn, dynShakeRestart ) where import Control.Concurrent.Async import Control.Concurrent.STM -import Control.Concurrent.STM.Stats (atomicallyNamed) +import Control.Concurrent.STM.Stats (atomicallyNamed) import Control.Concurrent.Strict import Control.DeepSeq -import Control.Exception.Extra hiding (bracket_) -import Control.Lens ((%~), (&), (?~)) +import Control.Exception.Extra hiding (bracket_) +import Control.Lens ((%~), (&), (?~)) import Control.Monad.Extra import Control.Monad.IO.Class import Control.Monad.Reader import Control.Monad.Trans.Maybe -import Data.Aeson (Result (Success), - toJSON) -import qualified Data.Aeson.Types as A -import qualified Data.ByteString.Char8 as BS -import qualified Data.ByteString.Char8 as BS8 -import Data.Coerce (coerce) +import Data.Aeson (Result (Success), + toJSON) +import qualified Data.Aeson.Types as A +import qualified Data.ByteString.Char8 as BS +import qualified Data.ByteString.Char8 as BS8 +import Data.Coerce (coerce) import Data.Default import Data.Dynamic -import Data.EnumMap.Strict (EnumMap) -import qualified Data.EnumMap.Strict as EM -import Data.Foldable (find, for_) -import Data.Functor ((<&>)) +import Data.EnumMap.Strict (EnumMap) +import qualified Data.EnumMap.Strict as EM +import Data.Foldable (find, for_) +import Data.Functor ((<&>)) import Data.Functor.Identity import Data.Hashable -import qualified Data.HashMap.Strict as HMap -import Data.HashSet (HashSet) -import qualified Data.HashSet as HSet -import Data.List.Extra (foldl', partition, - takeEnd) -import qualified Data.Map.Strict as Map +import qualified Data.HashMap.Strict as HMap +import Data.HashSet (HashSet) +import qualified Data.HashSet as HSet +import Data.List.Extra (partition, takeEnd) +import qualified Data.Map.Strict as Map import Data.Maybe -import qualified Data.SortedList as SL -import Data.String (fromString) -import qualified Data.Text as T +import qualified Data.SortedList as SL +import Data.String (fromString) +import qualified Data.Text as T import Data.Time import Data.Traversable import Data.Tuple.Extra import Data.Typeable import Data.Unique -import Data.Vector (Vector) -import qualified Data.Vector as Vector +import Data.Vector (Vector) +import qualified Data.Vector as Vector import Development.IDE.Core.Debouncer -import Development.IDE.Core.FileUtils (getModTime) +import Development.IDE.Core.FileUtils (getModTime) import Development.IDE.Core.PositionMapping import Development.IDE.Core.ProgressReporting import Development.IDE.Core.RuleTypes -import Development.IDE.Types.Options as Options -import qualified Language.LSP.Protocol.Message as LSP -import qualified Language.LSP.Server as LSP - +import Development.IDE.Types.Options as Options +import qualified Language.LSP.Protocol.Message as LSP +import qualified Language.LSP.Server as LSP + +import Data.Either (isRight, lefts) +import Data.Int (Int64) +import Data.IORef.Extra (atomicModifyIORef'_) +import Data.Set (Set) +import qualified Data.Set as S import Development.IDE.Core.Tracing -import Development.IDE.Core.WorkerThread -import Development.IDE.GHC.Compat (NameCache, - NameCacheUpdater, - initNameCache, - knownKeyNames) -import Development.IDE.GHC.Orphans () -import Development.IDE.Graph hiding (ShakeValue, - action) -import qualified Development.IDE.Graph as Shake -import Development.IDE.Graph.Database (ShakeDatabase, - shakeGetBuildStep, - shakeGetDatabaseKeys, - shakeNewDatabase, - shakeProfileDatabase, - shakeRunDatabaseForKeys) +import Development.IDE.GHC.Compat (NameCache, + NameCacheUpdater, + initNameCache, + knownKeyNames) +import Development.IDE.GHC.Orphans () +import Development.IDE.Graph hiding (ShakeValue, + action) +import qualified Development.IDE.Graph as Shake +import Development.IDE.Graph.Database (ShakeDatabase, + shakeComputeToPreserve, + shakeGetActionQueueLength, + shakeGetBuildStep, + shakeGetDatabaseKeys, + shakeNewDatabase, + shakePeekAsyncsDelivers, + shakeProfileDatabase, + shakeRunDatabaseForKeysSep, + shakeShutDatabase, + shakedatabaseRuntimeDep) +import Development.IDE.Graph.Internal.Action (runActionInDbCb) +import Development.IDE.Graph.Internal.Database (AsyncParentKill (AsyncParentKill)) +import Development.IDE.Graph.Internal.Types (DBQue, Step (..), + getShakeQueue, + getShakeStep, + lockShakeDatabaseValues, + shakeDataBaseQueue, + unlockShakeDatabaseValues, + withShakeDatabaseValuesLock) import Development.IDE.Graph.Rule import Development.IDE.Types.Action import Development.IDE.Types.Diagnostics -import Development.IDE.Types.Exports hiding (exportsMapSize) -import qualified Development.IDE.Types.Exports as ExportsMap +import Development.IDE.Types.Exports hiding (exportsMapSize) +import qualified Development.IDE.Types.Exports as ExportsMap import Development.IDE.Types.KnownTargets import Development.IDE.Types.Location -import Development.IDE.Types.Monitoring (Monitoring (..)) +import Development.IDE.Types.Monitoring (Monitoring (..)) import Development.IDE.Types.Shake +import Development.IDE.WorkerThread import qualified Focus import GHC.Fingerprint -import GHC.Stack (HasCallStack) -import GHC.TypeLits (KnownSymbol) +import GHC.Stack (HasCallStack) +import GHC.TypeLits (KnownSymbol) import HieDb.Types -import Ide.Logger hiding (Priority) -import qualified Ide.Logger as Logger +import Ide.Logger hiding (Priority) +import qualified Ide.Logger as Logger import Ide.Plugin.Config -import qualified Ide.PluginUtils as HLS +import qualified Ide.PluginUtils as HLS import Ide.Types -import qualified Language.LSP.Protocol.Lens as L +import qualified Language.LSP.Protocol.Lens as L import Language.LSP.Protocol.Message import Language.LSP.Protocol.Types -import qualified Language.LSP.Protocol.Types as LSP -import Language.LSP.VFS hiding (start) +import qualified Language.LSP.Protocol.Types as LSP +import Language.LSP.VFS hiding (start) import qualified "list-t" ListT -import OpenTelemetry.Eventlog hiding (addEvent) -import qualified Prettyprinter as Pretty -import qualified StmContainers.Map as STM -import System.FilePath hiding (makeRelative) -import System.IO.Unsafe (unsafePerformIO) +import OpenTelemetry.Eventlog hiding (addEvent) +import qualified Prettyprinter as Pretty +import qualified StmContainers.Map as STM +import System.FilePath hiding (makeRelative) +import System.IO.Unsafe (unsafePerformIO) import System.Time.Extra -import UnliftIO (MonadUnliftIO (withRunInIO)) +import UnliftIO (MonadUnliftIO (withRunInIO), + newIORef, readIORef) + +#if !MIN_VERSION_ghc(9,9,0) +import Data.Foldable (foldl') +#endif data Log = LogCreateHieDbExportsMapStart | LogCreateHieDbExportsMapFinish !Int - | LogBuildSessionRestart !String ![DelayedActionInternal] !KeySet !Seconds !(Maybe FilePath) + | LogBuildSessionRestart !ShakeRestartArgs ![DelayedActionInternal] !KeySet !Seconds !(Maybe FilePath) !Int ![DeliverStatus] | LogBuildSessionRestartTakingTooLong !Seconds | LogDelayedAction !(DelayedAction ()) !Seconds - | LogBuildSessionFinish !(Maybe SomeException) + | LogBuildSessionFinish !Step !(Either SomeException [Either SomeException ()]) | LogDiagsDiffButNoLspEnv ![FileDiagnostic] + | LogDiagsPublishLog !Key ![FileDiagnostic] ![FileDiagnostic] | LogDefineEarlyCutoffRuleNoDiagHasDiag !FileDiagnostic | LogDefineEarlyCutoffRuleCustomNewnessHasDiag !FileDiagnostic | LogCancelledAction !T.Text @@ -196,19 +222,43 @@ data Log | LogShakeGarbageCollection !T.Text !Int !Seconds -- * OfInterest Log messages | LogSetFilesOfInterest ![(NormalizedFilePath, FileOfInterestStatus)] + | LogShakeText !T.Text + | LogMonitering !T.Text !Int64 + | LogPreserveKeys ![Key] ![Key] ![Key] ![(Key, KeySet)] deriving Show instance Pretty Log where pretty = \case + LogPreserveKeys kvs ks allRunnings reverseKs -> + vcat [ + "LogPreserveKeys" + , "dirty keys:" <+> pretty (map show ks) + , "Preserving keys: " <+> pretty (map show kvs) + , "All running: " <+> pretty (map show allRunnings) + , "Reverse deps: " <+> pretty reverseKs + ] + LogMonitering name value -> + "Monitoring:" <+> pretty name <+> "value:" <+> pretty value + LogDiagsPublishLog key lastDiags diags -> + vcat + [ "Publishing diagnostics for" <+> pretty (show key) + , "Last published:" <+> pretty (showDiagnosticsColored lastDiags) <+> "diagnostics" + , "New:" <+> pretty (showDiagnosticsColored diags) <+> "diagnostics" + ] + LogShakeText msg -> pretty msg LogCreateHieDbExportsMapStart -> "Initializing exports map from hiedb" LogCreateHieDbExportsMapFinish exportsMapSize -> "Done initializing exports map from hiedb. Size:" <+> pretty exportsMapSize - LogBuildSessionRestart reason actionQueue keyBackLog abortDuration shakeProfilePath -> + LogBuildSessionRestart restartArgs actionQueue keyBackLog abortDuration shakeProfilePath step delivers -> vcat - [ "Restarting build session due to" <+> pretty reason + [ "Restarting build session due to" <+> pretty (sraReason restartArgs) + , "Restarts num:" <+> pretty (sraCount $ restartArgs) , "Action Queue:" <+> pretty (map actionName actionQueue) - , "Keys:" <+> pretty (map show $ toListKeySet keyBackLog) + -- , "Keys:" <+> pretty (map show $ toListKeySet keyBackLog) + , "Keys:" <+> pretty (length $ toListKeySet keyBackLog) + , "Deliveries still alive:" <+> pretty delivers + , "Current step:" <+> pretty (show step) , "Aborting previous build session took" <+> pretty (showDuration abortDuration) <+> pretty shakeProfilePath ] LogBuildSessionRestartTakingTooLong seconds -> "Build restart is taking too long (" <> pretty seconds <> " seconds)" @@ -216,10 +266,18 @@ instance Pretty Log where hsep [ "Finished:" <+> pretty (actionName delayedAct) , "Took:" <+> pretty (showDuration seconds) ] - LogBuildSessionFinish e -> + LogBuildSessionFinish step e -> vcat [ "Finished build session" - , pretty (fmap displayException e) ] + , "Step:" <+> pretty (show step) + , "Result:" <+> case e of + Left ex -> "Exception:" <+> pretty (show ex) + Right rs -> + if all isRight rs then + "Success" + else + "Exceptions in actions:" <+> pretty (fmap displayException $ lefts rs) + ] LogDiagsDiffButNoLspEnv fileDiagnostics -> "updateFileDiagnostics published different from new diagnostics - file diagnostics:" <+> pretty (showDiagnosticsColored fileDiagnostics) @@ -254,12 +312,17 @@ data HieDbWriter -- | Actions to queue up on the index worker thread -- The inner `(HieDb -> IO ()) -> IO ()` wraps `HieDb -> IO ()` -- with (currently) retry functionality -type IndexQueue = TQueue (((HieDb -> IO ()) -> IO ()) -> IO ()) +type IndexQueue = TaskQueue (((HieDb -> IO ()) -> IO ()) -> IO ()) +-- type ShakeControlQueue = TaskQueue ShakeRestartArgs +type ShakeQueue = DBQue +type ShakeControlQueue = ShakeQueue +type LoaderQueue = TaskQueue (IO ()) + data ThreadQueue = ThreadQueue { - tIndexQueue :: IndexQueue - , tRestartQueue :: TQueue (IO ()) - , tLoaderQueue :: TQueue (IO ()) + tIndexQueue :: IndexQueue + , tShakeControlQueue :: ShakeControlQueue + , tLoaderQueue :: LoaderQueue } -- Note [Semantic Tokens Cache Location] @@ -279,7 +342,7 @@ data ShakeExtras = ShakeExtras ,globals :: TVar (HMap.HashMap TypeRep Dynamic) -- ^ Registry of global state used by rules. -- Small and immutable after startup, so not worth using an STM.Map. - ,state :: Values + ,stateValues :: Values ,diagnostics :: STMDiagnosticStore ,hiddenDiagnostics :: STMDiagnosticStore ,publishedDiagnostics :: STM.Map NormalizedUri [FileDiagnostic] @@ -330,9 +393,9 @@ data ShakeExtras = ShakeExtras -- ^ Default HLS config, only relevant if the client does not provide any Config , dirtyKeys :: TVar KeySet -- ^ Set of dirty rule keys since the last Shake run - , restartQueue :: TQueue (IO ()) + , shakeControlQueue :: ShakeControlQueue -- ^ Queue of restart actions to be run. - , loaderQueue :: TQueue (IO ()) + , loaderQueue :: LoaderQueue -- ^ Queue of loader actions to be run. } @@ -390,11 +453,17 @@ addPersistentRule k getVal = do class Typeable a => IsIdeGlobal a where +-- data VirtualFileEntry = Open VirtualFile | Closed ClosedVirtualFile +-- | Read a virtual file from the current snapshot +getOpenFile :: VirtualFileEntry -> Maybe VirtualFile +getOpenFile (Open vf) = Just vf +getOpenFile _ = Nothing -- | Read a virtual file from the current snapshot getVirtualFile :: NormalizedFilePath -> Action (Maybe VirtualFile) getVirtualFile nf = do vfs <- fmap _vfsMap . liftIO . readTVarIO . vfsVar =<< getShakeExtras - pure $! Map.lookup (filePathToUri' nf) vfs -- Don't leak a reference to the entire map + let file = getOpenFile =<< Map.lookup (filePathToUri' nf) vfs + pure $! file -- Don't leak a reference to the entire map -- Take a snapshot of the current LSP VFS vfsSnapshot :: Maybe (LSP.LanguageContextEnv a) -> IO VFS @@ -452,7 +521,7 @@ getIdeOptionsIO ide = do -- | Return the most recent, potentially stale, value and a PositionMapping -- for the version of that value. lastValueIO :: IdeRule k v => ShakeExtras -> k -> NormalizedFilePath -> IO (Maybe (v, PositionMapping)) -lastValueIO s@ShakeExtras{positionMapping,persistentKeys,state} k file = do +lastValueIO s@ShakeExtras{positionMapping,persistentKeys,stateValues} k file = do let readPersistent | IdeTesting testing <- ideTesting s -- Don't read stale persistent values in tests @@ -466,7 +535,7 @@ lastValueIO s@ShakeExtras{positionMapping,persistentKeys,state} k file = do MaybeT $ pure $ (,del,ver) <$> fromDynamic dv case mv of Nothing -> atomicallyNamed "lastValueIO 1" $ do - STM.focus (Focus.alter (alterValue $ Failed True)) (toKey k file) state + STM.focus (Focus.alter (alterValue $ Failed True)) (toKey k file) stateValues return Nothing Just (v,del,mbVer) -> do actual_version <- case mbVer of @@ -474,7 +543,7 @@ lastValueIO s@ShakeExtras{positionMapping,persistentKeys,state} k file = do Nothing -> (Just . ModificationTime <$> getModTime (fromNormalizedFilePath file)) `catch` (\(_ :: IOException) -> pure Nothing) atomicallyNamed "lastValueIO 2" $ do - STM.focus (Focus.alter (alterValue $ Stale (Just del) actual_version (toDyn v))) (toKey k file) state + STM.focus (Focus.alter (alterValue $ Stale (Just del) actual_version (toDyn v))) (toKey k file) stateValues Just . (v,) . addOldDelta del <$> mappingForVersion positionMapping file actual_version -- We got a new stale value from the persistent rule, insert it in the map without affecting diagnostics @@ -485,7 +554,7 @@ lastValueIO s@ShakeExtras{positionMapping,persistentKeys,state} k file = do -- Something already succeeded before, leave it alone _ -> old - atomicallyNamed "lastValueIO 4" (STM.lookup (toKey k file) state) >>= \case + atomicallyNamed "lastValueIO 4" (STM.lookup (toKey k file) stateValues) >>= \case Nothing -> readPersistent Just (ValueWithDiagnostics value _) -> case value of Succeeded ver (fromDynamic -> Just v) -> @@ -523,7 +592,7 @@ type IdeRule k v = -- | A live Shake session with the ability to enqueue Actions for running. -- Keeps the 'ShakeDatabase' open, so at most one 'ShakeSession' per database. newtype ShakeSession = ShakeSession - { cancelShakeSession :: IO () + { cancelShakeSession :: Set (Async ()) -> IO () -- ^ Closes the Shake session } @@ -599,8 +668,8 @@ deleteValue -> k -> NormalizedFilePath -> STM [Key] -deleteValue ShakeExtras{state} key file = do - STM.delete (toKey key file) state +deleteValue ShakeExtras{stateValues} key file = do + STM.delete (toKey key file) stateValues return [toKey key file] @@ -659,40 +728,40 @@ shakeOpen :: Recorder (WithPriority Log) shakeOpen recorder lspEnv defaultConfig idePlugins debouncer shakeProfileDir (IdeReportProgress reportProgress) ideTesting - withHieDb threadQueue opts monitoring rules rootDir = mdo + withHieDb threadQueue opts argMonitoring rules rootDir = mdo -- see Note [Serializing runs in separate thread] let indexQueue = tIndexQueue threadQueue - restartQueue = tRestartQueue threadQueue + shakeControlQueue = tShakeControlQueue threadQueue loaderQueue = tLoaderQueue threadQueue ideNc <- initNameCache 'r' knownKeyNames shakeExtras <- do globals <- newTVarIO HMap.empty - state <- STM.newIO + stateValues <- STM.newIO diagnostics <- STM.newIO hiddenDiagnostics <- STM.newIO publishedDiagnostics <- STM.newIO semanticTokensCache <- STM.newIO positionMapping <- STM.newIO knownTargetsVar <- newTVarIO $ hashed emptyKnownTargets - let restartShakeSession = shakeRestart recorder ideState + restartVersion <- newTVarIO 0 + let restartShakeSession = shakeRestart restartVersion shakeDb persistentKeys <- newTVarIO mempty indexPending <- newTVarIO HMap.empty indexCompleted <- newTVarIO 0 semanticTokensId <- newTVarIO 0 indexProgressReporting <- progressReportingNoTrace - (liftM2 (+) (length <$> readTVar indexPending) (readTVar indexCompleted)) - (readTVar indexCompleted) - lspEnv "Indexing" optProgressStyle + (liftM2 (+) (length <$> readTVar indexPending) (readTVar indexCompleted) ) + (readTVar indexCompleted) lspEnv "Indexing" optProgressStyle let hiedbWriter = HieDbWriter{..} exportsMap <- newTVarIO mempty -- lazily initialize the exports map with the contents of the hiedb - -- TODO: exceptions can be swallowed here? - _ <- async $ do + async <- async $ do logWith recorder Debug LogCreateHieDbExportsMapStart em <- createExportsMapHieDb withHieDb atomically $ modifyTVar' exportsMap (<> em) logWith recorder Debug $ LogCreateHieDbExportsMapFinish (ExportsMap.size em) + link async progress <- if reportProgress @@ -707,6 +776,8 @@ shakeOpen recorder lspEnv defaultConfig idePlugins debouncer pure ShakeExtras{shakeRecorder = recorder, ..} shakeDb <- shakeNewDatabase + (\logText -> logWith recorder Debug (LogShakeText $ T.pack logText)) + shakeControlQueue opts { shakeExtra = newShakeExtra shakeExtras } rules shakeSession <- newEmptyMVar @@ -719,13 +790,17 @@ shakeOpen recorder lspEnv defaultConfig idePlugins debouncer checkParents <- optCheckParents + + -- logMonitoring <- newLogMonitoring recorder + let monitoring = argMonitoring -- monitoring let readValuesCounter = fromIntegral . countRelevantKeys checkParents <$> getStateKeys shakeExtras - readDirtyKeys = fromIntegral . countRelevantKeys checkParents . toListKeySet <$> readTVarIO(dirtyKeys shakeExtras) + readDirtyKeys = fromIntegral . countRelevantKeys checkParents . toListKeySet <$> readTVarIO (dirtyKeys shakeExtras) readIndexPending = fromIntegral . HMap.size <$> readTVarIO (indexPending $ hiedbWriter shakeExtras) readExportsMap = fromIntegral . ExportsMap.exportsMapSize <$> readTVarIO (exportsMap shakeExtras) readDatabaseCount = fromIntegral . countRelevantKeys checkParents . map fst <$> shakeGetDatabaseKeys shakeDb readDatabaseStep = fromIntegral <$> shakeGetBuildStep shakeDb + readDatabaseActionQueueCount = fromIntegral <$> shakeGetActionQueueLength shakeDb registerGauge monitoring "ghcide.values_count" readValuesCounter registerGauge monitoring "ghcide.dirty_keys_count" readDirtyKeys @@ -733,15 +808,32 @@ shakeOpen recorder lspEnv defaultConfig idePlugins debouncer registerGauge monitoring "ghcide.exports_map_count" readExportsMap registerGauge monitoring "ghcide.database_count" readDatabaseCount registerCounter monitoring "ghcide.num_builds" readDatabaseStep + registerCounter monitoring "ghcide.database_action_queue_count" readDatabaseActionQueueCount stopMonitoring <- start monitoring let ideState = IdeState{..} return ideState +newLogMonitoring :: MonadIO m => Recorder (WithPriority Log) -> m Monitoring +newLogMonitoring logger = do + actions <- newIORef [] + let registerCounter name readA = do + let update = do + val <- readA + logWith logger Info $ LogMonitering name (fromIntegral val) + atomicModifyIORef'_ actions (update :) + registerGauge = registerCounter + let start = do + a <- regularly 10 $ sequence_ =<< readIORef actions + return (cancel a) + return Monitoring{..} + where + regularly :: Seconds -> IO () -> IO (Async ()) + regularly delay act = async $ forever (act >> sleep delay) getStateKeys :: ShakeExtras -> IO [Key] -getStateKeys = (fmap.fmap) fst . atomically . ListT.toList . STM.listT . state +getStateKeys = (fmap.fmap) fst . atomically . ListT.toList . STM.listT . stateValues -- | Must be called in the 'Initialized' handler and only once shakeSessionInit :: Recorder (WithPriority Log) -> IdeState -> IO () @@ -758,7 +850,8 @@ shakeShut IdeState{..} = do runner <- tryReadMVar shakeSession -- Shake gets unhappy if you try to close when there is a running -- request so we first abort that. - for_ runner cancelShakeSession + for_ runner (flip cancelShakeSession mempty) + shakeShutDatabase mempty shakeDb void $ shakeDatabaseProfile shakeDb progressStop $ progress shakeExtras progressStop $ indexProgressReporting $ hiedbWriter shakeExtras @@ -788,37 +881,105 @@ delayedAction a = do liftIO $ shakeEnqueue extras a +data ShakeRestartArgs = ShakeRestartArgs + { sraVfs :: !VFSModified + , sraReason :: !String + , sraActions :: ![DelayedAction ()] + , sraBetweenSessions :: IO [Key] + , sraCount :: !Int + -- ^ Just for debugging, how many restarts have been requested so far + , sraWaitMVars :: ![MVar ()] + , sraVersion :: !Int + } + +instance Show ShakeRestartArgs where + show ShakeRestartArgs{..} = + "ShakeRestartArgs { sraReason = " ++ show sraReason + ++ ", sraActions = " ++ show (map actionName sraActions) + ++ ", sraCount = " ++ show sraCount + ++ " }" + +instance Semigroup ShakeRestartArgs where + a <> b = + -- the larger the version, the later it was requested + -- prefer the later one + let (new, old) = if sraVersion a >= sraVersion b then (a, b) else (b, a) + in ShakeRestartArgs + { sraVfs = sraVfs old <> sraVfs new + , sraReason = sraReason old ++ "; " ++ sraReason new + , sraActions = sraActions old ++ sraActions new + , sraBetweenSessions = (++) <$> sraBetweenSessions old <*> sraBetweenSessions new + , sraCount = sraCount old + sraCount new + , sraWaitMVars = sraWaitMVars old ++ sraWaitMVars new + , sraVersion = sraVersion new + } + -- | Restart the current 'ShakeSession' with the given system actions. -- Any actions running in the current session will be aborted, -- but actions added via 'shakeEnqueue' will be requeued. -shakeRestart :: Recorder (WithPriority Log) -> IdeState -> VFSModified -> String -> [DelayedAction ()] -> IO [Key] -> IO () -shakeRestart recorder IdeState{..} vfs reason acts ioActionBetweenShakeSession = - void $ awaitRunInThread (restartQueue shakeExtras) $ do - withMVar' - shakeSession - (\runner -> do - (stopTime,()) <- duration $ logErrorAfter 10 $ cancelShakeSession runner - keys <- ioActionBetweenShakeSession - -- it is every important to update the dirty keys after we enter the critical section - -- see Note [Housekeeping rule cache and dirty key outside of hls-graph] - atomically $ modifyTVar' (dirtyKeys shakeExtras) $ \x -> foldl' (flip insertKeySet) x keys - res <- shakeDatabaseProfile shakeDb - backlog <- readTVarIO $ dirtyKeys shakeExtras - queue <- atomicallyNamed "actionQueue - peek" $ peekInProgress $ actionQueue shakeExtras - - -- this log is required by tests - logWith recorder Debug $ LogBuildSessionRestart reason queue backlog stopTime res - ) - -- It is crucial to be masked here, otherwise we can get killed - -- between spawning the new thread and updating shakeSession. - -- See https://github.com/haskell/ghcide/issues/79 - (\() -> do - (,()) <$> newSession recorder shakeExtras vfs shakeDb acts reason) - where - logErrorAfter :: Seconds -> IO () -> IO () - logErrorAfter seconds action = flip withAsync (const action) $ do - sleep seconds - logWith recorder Error (LogBuildSessionRestartTakingTooLong seconds) +shakeRestart :: TVar Int -> ShakeDatabase -> VFSModified -> String -> [DelayedAction ()] -> IO [Key] -> IO () +shakeRestart version db vfs reason acts ioActionBetweenShakeSession = do + -- lockShakeDatabaseValues db + v <- atomically $ do + modifyTVar' version (+1) + readTVar version + let rts = shakeDataBaseQueue db + waitMVar <- newEmptyMVar + -- submit at the head of the queue, + -- prefer restart request over any pending actions + void $ submitWorkAtHead rts $ Left $ + toDyn $ ShakeRestartArgs vfs reason acts ioActionBetweenShakeSession 1 [waitMVar] v + -- Wait until the restart is done + takeMVar waitMVar + + +runRestartTaskDyn :: Recorder (WithPriority Log) -> MVar IdeState -> Dynamic -> IO () +runRestartTaskDyn recorder ideStateVar dy = runRestartTask recorder ideStateVar (dynShakeRestart dy) + +dynShakeRestart :: Dynamic -> ShakeRestartArgs +dynShakeRestart dy = case fromDynamic dy of + Just shakeRestartArgs -> shakeRestartArgs + Nothing -> error "Internal error, dynShakeRestart, got invalid dynamic type" + +runRestartTask :: Recorder (WithPriority Log) -> MVar IdeState -> ShakeRestartArgs -> IO () +runRestartTask recorder ideStateVar shakeRestartArgs = do + IdeState {shakeDb, shakeSession, shakeExtras, shakeDatabaseProfile} <- readMVar ideStateVar + withShakeDatabaseValuesLock shakeDb $ do + withMVar' + shakeSession + ( \runner -> do + newDirtyKeys <- sraBetweenSessions shakeRestartArgs + reverseMap <- shakedatabaseRuntimeDep shakeDb + (preservekvs, allRunning2) <- shakeComputeToPreserve shakeDb $ fromListKeySet newDirtyKeys + logWith recorder Debug $ LogPreserveKeys (map fst preservekvs) newDirtyKeys allRunning2 reverseMap + (stopTime, ()) <- duration $ logErrorAfter 10 $ cancelShakeSession runner $ S.fromList $ map snd preservekvs + survivedDelivers <- shakePeekAsyncsDelivers shakeDb + -- it is every important to update the dirty keys after we enter the critical section + -- see Note [Housekeeping rule cache and dirty key outside of hls-graph] + atomically $ modifyTVar' (dirtyKeys shakeExtras) $ \x -> foldl' (flip insertKeySet) x newDirtyKeys + + queue <- atomicallyNamed "actionQueue - peek" $ peekInProgress $ actionQueue shakeExtras + res <- shakeDatabaseProfile shakeDb + backlog <- readTVarIO $ dirtyKeys shakeExtras + -- this log is required by tests + step <- shakeGetBuildStep shakeDb + + logWith recorder Info $ LogBuildSessionRestart shakeRestartArgs queue backlog stopTime res step survivedDelivers + return shakeRestartArgs + ) + -- It is crucial to be masked here, otherwise we can get killed + -- between spawning the new thread and updating shakeSession. + -- See https://github.com/haskell/ghcide/issues/79 + ( \(ShakeRestartArgs {..}) -> + do + (,()) <$> newSession recorder shakeExtras sraVfs shakeDb sraActions sraReason + `finally` for_ sraWaitMVars (`putMVar` ()) + ) + where + logErrorAfter :: Seconds -> IO () -> IO () + logErrorAfter seconds action = flip withAsync (const action) $ do + sleep seconds + logWith recorder Error (LogBuildSessionRestartTakingTooLong seconds) -- | Enqueue an action in the existing 'ShakeSession'. -- Returns a computation to block until the action is run, propagating exceptions. @@ -829,12 +990,13 @@ shakeEnqueue :: ShakeExtras -> DelayedAction a -> IO (IO a) shakeEnqueue ShakeExtras{actionQueue, shakeRecorder} act = do (b, dai) <- instantiateDelayedAction act atomicallyNamed "actionQueue - push" $ pushQueue dai actionQueue + logWith shakeRecorder Debug $ LogShakeText (T.pack $ "Enqueued action: " <> actionName act) let wait' barrier = waitBarrier barrier `catches` - [ Handler(\BlockedIndefinitelyOnMVar -> + [ Handler (\BlockedIndefinitelyOnMVar -> fail $ "internal bug: forever blocked on MVar for " <> actionName act) - , Handler (\e@AsyncCancelled -> do + , Handler (\e@(SomeAsyncException _) -> do logWith shakeRecorder Debug $ LogCancelledAction (T.pack $ actionName act) atomicallyNamed "actionQueue - abort" $ abortQueue dai actionQueue @@ -844,6 +1006,10 @@ shakeEnqueue ShakeExtras{actionQueue, shakeRecorder} act = do data VFSModified = VFSUnmodified | VFSModified !VFS +instance Semigroup VFSModified where + x <> VFSUnmodified = x + _ <> x = x + -- | Set up a new 'ShakeSession' with a set of initial actions -- Will crash if there is an existing 'ShakeSession' running. newSession @@ -862,7 +1028,9 @@ newSession recorder extras@ShakeExtras{..} vfsMod shakeDb acts reason = do VFSModified vfs -> atomically $ writeTVar vfsVar vfs IdeOptions{optRunSubset} <- getIdeOptionsIO extras + reenqueued <- atomicallyNamed "actionQueue - peek" $ peekInProgress actionQueue + step <- getShakeStep shakeDb allPendingKeys <- if optRunSubset then Just <$> readTVarIO dirtyKeys @@ -870,12 +1038,20 @@ newSession recorder extras@ShakeExtras{..} vfsMod shakeDb acts reason = do let -- A daemon-like action used to inject additional work -- Runs actions from the work queue sequentially - pumpActionThread otSpan = do - d <- liftIO $ atomicallyNamed "action queue - pop" $ popQueue actionQueue - actionFork (run otSpan d) $ \_ -> pumpActionThread otSpan + logResult :: Show a => String -> [Either SomeException a] -> IO () + logResult label results = for_ results $ \case + Left e | Just (AsyncParentKill _ _) <- fromException e -> logWith recorder Debug $ LogShakeText (T.pack $ label ++ " failed: " ++ show e) + Left e -> logWith recorder Error $ LogShakeText (T.pack $ label ++ " failed: " ++ show e) + Right r -> logWith recorder Debug $ LogShakeText (T.pack $ label ++ " finished: " ++ show r) + pumpActionThread = do + logWith recorder Debug $ LogShakeText (T.pack $ "Starting action" ++ "(step: " <> show step) + d <- runActionInDbCb actionName run (popQueue actionQueue) (logResult "pumpActionThread" . return) + step <- getShakeStep shakeDb + logWith recorder Debug $ LogShakeText (T.pack $ "started action" ++ "(step: " <> show step <> "): " <> actionName d) + pumpActionThread -- TODO figure out how to thread the otSpan into defineEarlyCutoff - run _otSpan d = do + run d = do start <- liftIO offsetTime getAction d liftIO $ atomicallyNamed "actionQueue - done" $ doneQueue d actionQueue @@ -883,34 +1059,33 @@ newSession recorder extras@ShakeExtras{..} vfsMod shakeDb acts reason = do logWith recorder (actionPriority d) $ LogDelayedAction d runTime -- The inferred type signature doesn't work in ghc >= 9.0.1 - workRun :: (forall b. IO b -> IO b) -> IO (IO ()) - workRun restore = withSpan "Shake session" $ \otSpan -> do + -- workRun :: (forall b. IO b -> IO b) -> IO () + workRun start restore = withSpan "Shake session" $ \otSpan -> do setTag otSpan "reason" (fromString reason) setTag otSpan "queue" (fromString $ unlines $ map actionName reenqueued) whenJust allPendingKeys $ \kk -> setTag otSpan "keys" (BS8.pack $ unlines $ map show $ toListKeySet kk) - let keysActs = pumpActionThread otSpan : map (run otSpan) (reenqueued ++ acts) - res <- try @SomeException $ - restore $ shakeRunDatabaseForKeys (toListKeySet <$> allPendingKeys) shakeDb keysActs - return $ do - let exception = - case res of - Left e -> Just e - _ -> Nothing - logWith recorder Debug $ LogBuildSessionFinish exception + res <- try @SomeException $ restore start + logWith recorder Info $ LogBuildSessionFinish step res - -- Do the work in a background thread - workThread <- asyncWithUnmask workRun - -- run the wrap up in a separate thread since it contains interruptible - -- commands (and we are not using uninterruptible mask) - -- TODO: can possibly swallow exceptions? - _ <- async $ join $ wait workThread + let keysActs = pumpActionThread : map run (reenqueued ++ acts) + -- first we increase the step, so any actions started from here on + startDatabase <- shakeRunDatabaseForKeysSep (toListKeySet <$> allPendingKeys) shakeDb keysActs + -- Do the work in a background thread + workThread <- asyncWithUnmask $ \x -> do + workRun startDatabase x -- Cancelling is required to flush the Shake database when either -- the filesystem or the Ghc configuration have changed - let cancelShakeSession :: IO () - cancelShakeSession = cancel workThread + let cancelShakeSession :: Set (Async ()) -> IO () + cancelShakeSession preserve = do + logWith recorder Info $ LogShakeText ("Starting shake cancellation: " <> " (" <> T.pack (show reason) <> ")") + tid <- myThreadId + cancelWith workThread $ AsyncParentKill tid step + shakeShutDatabase preserve shakeDb + + -- should wait until the step has increased pure (ShakeSession{..}) instantiateDelayedAction @@ -959,9 +1134,9 @@ garbageCollectDirtyKeysOlderThan maxAge checkParents = otTracedGarbageCollection garbageCollectKeys :: String -> Int -> CheckParents -> [(Key, Int)] -> Action [Key] garbageCollectKeys label maxAge checkParents agedKeys = do start <- liftIO offsetTime - ShakeExtras{state, dirtyKeys, lspEnv, shakeRecorder, ideTesting} <- getShakeExtras + ShakeExtras{stateValues, dirtyKeys, lspEnv, shakeRecorder, ideTesting} <- getShakeExtras (n::Int, garbage) <- liftIO $ - foldM (removeDirtyKey dirtyKeys state) (0,[]) agedKeys + foldM (removeDirtyKey dirtyKeys stateValues) (0,[]) agedKeys t <- liftIO start when (n>0) $ liftIO $ do logWith shakeRecorder Debug $ LogShakeGarbageCollection (T.pack label) n t @@ -975,7 +1150,7 @@ garbageCollectKeys label maxAge checkParents agedKeys = do removeDirtyKey dk values st@(!counter, keys) (k, age) | age > maxAge , Just (kt,_) <- fromKeyType k - , not(kt `HSet.member` preservedKeys checkParents) + , not (kt `HSet.member` preservedKeys checkParents) = atomicallyNamed "GC" $ do gotIt <- STM.focus (Focus.member <* Focus.delete) k values when gotIt $ @@ -1090,8 +1265,8 @@ useWithStaleFast' key file = do -- keep updating the value in the key. waitValue <- delayedAction $ mkDelayedAction ("C:" ++ show key ++ ":" ++ fromNormalizedFilePath file) Debug $ use key file - s@ShakeExtras{state} <- askShake - r <- liftIO $ atomicallyNamed "useStateFast" $ getValues state key file + s@ShakeExtras{stateValues} <- askShake + r <- liftIO $ atomicallyNamed "useStateFast" $ getValues stateValues key file liftIO $ case r of -- block for the result if we haven't computed before Nothing -> do @@ -1235,13 +1410,13 @@ defineEarlyCutoff' -> (Value v -> Action (Maybe BS.ByteString, IdeResult v)) -> Action (RunResult (A (RuleResult k))) defineEarlyCutoff' doDiagnostics cmp key file mbOld mode action = do - ShakeExtras{state, progress, dirtyKeys} <- getShakeExtras + ShakeExtras{stateValues, progress, dirtyKeys} <- getShakeExtras options <- getIdeOptions let trans g x = withRunInIO $ \run -> g (run x) (if optSkipProgress options key then id else trans (inProgress progress file)) $ do val <- case mbOld of Just old | mode == RunDependenciesSame -> do - mbValue <- liftIO $ atomicallyNamed "define - read 1" $ getValues state key file + mbValue <- liftIO $ atomicallyNamed "define - read 1" $ getValues stateValues key file case mbValue of -- No changes in the dependencies and we have -- an existing successful result. @@ -1257,7 +1432,7 @@ defineEarlyCutoff' doDiagnostics cmp key file mbOld mode action = do res <- case val of Just res -> return res Nothing -> do - staleV <- liftIO $ atomicallyNamed "define -read 3" $ getValues state key file <&> \case + staleV <- liftIO $ atomicallyNamed "define -read 3" $ getValues stateValues key file <&> \case Nothing -> Failed False Just (Succeeded ver v, _) -> Stale Nothing ver v Just (Stale d ver v, _) -> Stale d ver v @@ -1285,7 +1460,7 @@ defineEarlyCutoff' doDiagnostics cmp key file mbOld mode action = do (A res) $ do -- this hook needs to be run in the same transaction as the key is marked clean -- see Note [Housekeeping rule cache and dirty key outside of hls-graph] - setValues state key file res (Vector.fromList diags) + setValues stateValues key file res (Vector.fromList diags) modifyTVar' dirtyKeys (deleteKeySet $ toKey key file) return res where @@ -1350,12 +1525,12 @@ updateFileDiagnostics :: MonadIO m -> [FileDiagnostic] -- ^ current results -> m () updateFileDiagnostics recorder fp ver k ShakeExtras{diagnostics, hiddenDiagnostics, publishedDiagnostics, debouncer, lspEnv, ideTesting} current0 = do - liftIO $ withTrace ("update diagnostics " <> fromString(fromNormalizedFilePath fp)) $ \ addTag -> do + liftIO $ withTrace ("update diagnostics " <> fromString (fromNormalizedFilePath fp)) $ \ addTag -> do addTag "key" (show k) let (currentShown, currentHidden) = partition ((== ShowDiag) . fdShouldShowDiagnostic) current uri = filePathToUri' fp addTagUnsafe :: String -> String -> String -> a -> a - addTagUnsafe msg t x v = unsafePerformIO(addTag (msg <> t) x) `seq` v + addTagUnsafe msg t x v = unsafePerformIO (addTag (msg <> t) x) `seq` v update :: (forall a. String -> String -> a -> a) -> [FileDiagnostic] -> STMDiagnosticStore -> STM [FileDiagnostic] update addTagUnsafeMethod new store = addTagUnsafeMethod "count" (show $ Prelude.length new) $ setStageDiagnostics addTagUnsafeMethod uri ver (renderKey k) new store current = map (fdLspDiagnosticL %~ diagsFromRule) current0 @@ -1378,6 +1553,7 @@ updateFileDiagnostics recorder fp ver k ShakeExtras{diagnostics, hiddenDiagnosti Just env -> LSP.runLspT env $ do liftIO $ tag "count" (show $ Prelude.length newDiags) liftIO $ tag "key" (show k) + -- logWith recorder Debug $ LogDiagsPublishLog k lastPublish newDiags LSP.sendNotification SMethod_TextDocumentPublishDiagnostics $ LSP.PublishDiagnosticsParams (fromNormalizedUri uri') (fmap fromIntegral ver) (map fdLspDiagnostic newDiags) return action @@ -1481,3 +1657,4 @@ runWithSignal msgStart msgEnd files rule = do kickSignal testing lspEnv files msgStart void $ uses rule files kickSignal testing lspEnv files msgEnd + diff --git a/ghcide/src/Development/IDE/Core/WorkerThread.hs b/ghcide/src/Development/IDE/Core/WorkerThread.hs deleted file mode 100644 index 6d141c7ef3..0000000000 --- a/ghcide/src/Development/IDE/Core/WorkerThread.hs +++ /dev/null @@ -1,59 +0,0 @@ -{- -Module : Development.IDE.Core.WorkerThread -Author : @soulomoon -SPDX-License-Identifier: Apache-2.0 - -Description : This module provides an API for managing worker threads in the IDE. -see Note [Serializing runs in separate thread] --} -module Development.IDE.Core.WorkerThread - (withWorkerQueue, awaitRunInThread) - where - -import Control.Concurrent.Async (AsyncCancelled (AsyncCancelled), - withAsync) -import Control.Concurrent.STM -import Control.Concurrent.Strict (newBarrier, signalBarrier, - waitBarrier) -import Control.Exception.Safe (Exception (fromException), - SomeException, throwIO, try) -import Control.Monad (forever) -import Control.Monad.Cont (ContT (ContT)) - -{- -Note [Serializing runs in separate thread] -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -We often want to take long-running actions using some resource that cannot be shared. -In this instance it is useful to have a queue of jobs to run using the resource. -Like the db writes, session loading in session loader, shake session restarts. - -Originally we used various ways to implement this, but it was hard to maintain and error prone. -Moreover, we can not stop these threads uniformly when we are shutting down the server. --} - --- | 'withWorkerQueue' creates a new 'TQueue', and launches a worker --- thread which polls the queue for requests and runs the given worker --- function on them. -withWorkerQueue :: (t -> IO a) -> ContT () IO (TQueue t) -withWorkerQueue workerAction = ContT $ \mainAction -> do - q <- newTQueueIO - withAsync (writerThread q) $ \_ -> mainAction q - where - writerThread q = - forever $ do - l <- atomically $ readTQueue q - workerAction l - --- | 'awaitRunInThread' queues up an 'IO' action to be run by a worker thread, --- and then blocks until the result is computed. If the action throws an --- non-async exception, it is rethrown in the calling thread. -awaitRunInThread :: TQueue (IO ()) -> IO result -> IO result -awaitRunInThread q act = do - -- Take an action from TQueue, run it and - -- use barrier to wait for the result - barrier <- newBarrier - atomically $ writeTQueue q $ try act >>= signalBarrier barrier - resultOrException <- waitBarrier barrier - case resultOrException of - Left e -> throwIO (e :: SomeException) - Right r -> return r diff --git a/ghcide/src/Development/IDE/LSP/LanguageServer.hs b/ghcide/src/Development/IDE/LSP/LanguageServer.hs index 918e024a4f..e6c9845042 100644 --- a/ghcide/src/Development/IDE/LSP/LanguageServer.hs +++ b/ghcide/src/Development/IDE/LSP/LanguageServer.hs @@ -12,6 +12,8 @@ module Development.IDE.LSP.LanguageServer , ThreadQueue , runWithWorkerThreads , Setup (..) + , InitializationContext (..) + , untilMVar' ) where import Control.Concurrent.STM @@ -35,32 +37,62 @@ import UnliftIO.Directory import UnliftIO.Exception import qualified Colog.Core as Colog +import Control.Concurrent.Extra (newBarrier, + signalBarrier, + waitBarrier) import Control.Monad.IO.Unlift (MonadUnliftIO) -import Control.Monad.Trans.Cont (evalContT) +import Control.Monad.Trans.Cont (ContT, evalContT) import Development.IDE.Core.IdeConfiguration +import Development.IDE.Core.Service (shutdown) import Development.IDE.Core.Shake hiding (Log) +import qualified Development.IDE.Core.Shake as Shake import Development.IDE.Core.Tracing -import Development.IDE.Core.WorkerThread (withWorkerQueue) +import Development.IDE.Graph.Internal.Types (DBQue) import qualified Development.IDE.Session as Session import Development.IDE.Types.Shake (WithHieDb, WithHieDbShield (..)) +import Development.IDE.WorkerThread import Ide.Logger import Language.LSP.Server (LanguageContextEnv, LspServerLog, type (<~>)) +import System.Time.Extra (Seconds) +import System.Timeout (timeout) data Log = LogRegisteringIdeConfig !IdeConfiguration | LogReactorThreadException !SomeException | LogReactorMessageActionException !SomeException - | LogReactorThreadStopped + | LogReactorThreadStopped Int | LogCancelledRequest !SomeLspId | LogSession Session.Log + | LogShake Shake.Log | LogLspServer LspServerLog - | LogServerShutdownMessage + | LogReactorShutdownRequested Bool + | LogShutDownTimeout Int + | LogServerExitWith (Either () Int) + | LogReactorShutdownConfirmed !T.Text + | LogInitializeIdeStateTookTooLong Seconds + | LogText !T.Text deriving Show instance Pretty Log where pretty = \case + LogText msg -> pretty msg + LogShake msg -> pretty msg + LogInitializeIdeStateTookTooLong seconds -> + "Building the initial session took more than" <+> pretty seconds <+> "seconds" + LogReactorShutdownRequested b -> + "Requested reactor shutdown; stop signal posted: " <+> pretty b + LogReactorShutdownConfirmed msg -> + "Reactor shutdown confirmed: " <+> pretty msg + LogServerExitWith (Right 0) -> + "Server exited successfully" + LogServerExitWith (Right code) -> + "Server exited with failure code" <+> pretty code + LogServerExitWith (Left _) -> + "Server forcefully exited due to exception in reactor thread" + LogShutDownTimeout seconds -> + "Shutdown timeout, the server will exit now after waiting for" <+> pretty seconds <+> "seconds" LogRegisteringIdeConfig ideConfig -> -- This log is also used to identify if HLS starts successfully in vscode-haskell, -- don't forget to update the corresponding test in vscode-haskell if the text in @@ -74,13 +106,38 @@ instance Pretty Log where vcat [ "ReactorMessageActionException" , pretty $ displayException e ] - LogReactorThreadStopped -> - "Reactor thread stopped" + LogReactorThreadStopped i -> + "Reactor thread stopped" <+> pretty i LogCancelledRequest requestId -> "Cancelled request" <+> viaShow requestId LogSession msg -> pretty msg LogLspServer msg -> pretty msg - LogServerShutdownMessage -> "Received shutdown message" + +-- | Context for initializing the LSP language server. +-- This record encapsulates all the configuration and callback functions +-- needed to set up and run the language server initialization process. +data InitializationContext config = InitializationContext + { ctxRecorder :: Recorder (WithPriority Log) + -- ^ Logger for recording server events and diagnostics + , ctxDefaultRoot :: FilePath + -- ^ Default root directory for the workspace, see Note [Root Directory] + , ctxGetHieDbLoc :: FilePath -> IO FilePath + -- ^ Function to determine the HIE database location for a given root path + , ctxGetIdeState :: LSP.LanguageContextEnv config -> FilePath -> WithHieDb -> ThreadQueue -> IO IdeState + -- ^ Function to create and initialize the IDE state with the given environment + , ctxUntilReactorStopSignal :: IO () -> IO () + -- ^ Lifetime control: MVar to signal reactor shutdown + , ctxconfirmReactorShutdown :: T.Text -> IO () + -- ^ Callback to log/confirm reactor shutdown with a reason + , ctxForceShutdown :: IO () + -- ^ Action to forcefully exit the server when exception occurs + , ctxClearReqId :: SomeLspId -> IO () + -- ^ Function to clear/cancel a request by its ID + , ctxWaitForCancel :: SomeLspId -> IO () + -- ^ Function to wait for a request cancellation by its ID + , ctxClientMsgChan :: Chan ReactorMessage + -- ^ Channel for communicating with the reactor message loop + } data Setup config m a = MkSetup @@ -136,8 +193,8 @@ runLanguageServer recorder options inH outH defaultConfig parseConfig onConfigCh outH serverDefinition - untilMVar clientMsgVar $ - runServer `finally` sequence_ onExit + untilMVar' clientMsgVar runServer `finally` sequence_ onExit + >>= logWith recorder Info . LogServerExitWith setupLSP :: forall config. @@ -155,8 +212,21 @@ setupLSP recorder defaultRoot getHieDbLoc userHandlers getIdeState clientMsgVar -- An MVar to control the lifetime of the reactor loop. -- The loop will be stopped and resources freed when it's full - reactorLifetime <- newEmptyMVar - let stopReactorLoop = void $ tryPutMVar reactorLifetime () + reactorStopSignal <- newEmptyMVar + reactorConfirmBarrier <- newBarrier + let + untilReactorStopSignal = untilMVar reactorStopSignal + confirmReactorShutdown reason = do + logWith recorder Debug $ LogReactorShutdownConfirmed reason + signalBarrier reactorConfirmBarrier () + requestReactorShutdown = do + k <- tryPutMVar reactorStopSignal () + logWith recorder Info $ LogReactorShutdownRequested k + let timeOutSeconds = 10 + timeout (timeOutSeconds * 1_000_000) (waitBarrier reactorConfirmBarrier) >>= \case + Just () -> pure () + -- If we don't get confirmation within 2 seconds, we log a warning and shutdown anyway. + Nothing -> logWith recorder Warning $ LogShutDownTimeout timeOutSeconds -- Forcefully exit let exit = void $ tryPutMVar clientMsgVar () @@ -185,49 +255,64 @@ setupLSP recorder defaultRoot getHieDbLoc userHandlers getIdeState clientMsgVar let staticHandlers = mconcat [ userHandlers , cancelHandler cancelRequest - , exitHandler exit - , shutdownHandler recorder stopReactorLoop + , shutdownHandler recorder requestReactorShutdown + , exitHandler recorder exit ] -- Cancel requests are special since they need to be handled -- out of order to be useful. Existing handlers are run afterwards. - let doInitialize = handleInit recorder defaultRoot getHieDbLoc getIdeState reactorLifetime exit clearReqId waitForCancel clientMsgChan + let initParams = InitializationContext + { ctxRecorder = recorder + , ctxDefaultRoot = defaultRoot + , ctxGetHieDbLoc = getHieDbLoc + , ctxGetIdeState = getIdeState + , ctxUntilReactorStopSignal = untilReactorStopSignal + , ctxconfirmReactorShutdown = confirmReactorShutdown + , ctxForceShutdown = exit + , ctxClearReqId = clearReqId + , ctxWaitForCancel = waitForCancel + , ctxClientMsgChan = clientMsgChan + } + + let doInitialize = handleInit initParams let interpretHandler (env, st) = LSP.Iso (LSP.runLspT env . flip (runReaderT . unServerM) (clientMsgChan,st)) liftIO - - let onExit = [stopReactorLoop, exit] + let onExit = [void $ tryPutMVar reactorStopSignal ()] pure MkSetup {doInitialize, staticHandlers, interpretHandler, onExit} handleInit - :: Recorder (WithPriority Log) - -> FilePath -- ^ root directory, see Note [Root Directory] - -> (FilePath -> IO FilePath) - -> (LSP.LanguageContextEnv config -> FilePath -> WithHieDb -> ThreadQueue -> IO IdeState) - -> MVar () - -> IO () - -> (SomeLspId -> IO ()) - -> (SomeLspId -> IO ()) - -> Chan ReactorMessage + :: InitializationContext config -> LSP.LanguageContextEnv config -> TRequestMessage Method_Initialize -> IO (Either err (LSP.LanguageContextEnv config, IdeState)) -handleInit recorder defaultRoot getHieDbLoc getIdeState lifetime exitClientMsg clearReqId waitForCancel clientMsgChan env (TRequestMessage _ _ m params) = otTracedHandler "Initialize" (show m) $ \sp -> do +handleInit initParams env (TRequestMessage _ _ m params) = otTracedHandler "Initialize" (show m) $ \sp -> do traceWithSpan sp params - -- only shift if lsp root is different from the rootDir - -- see Note [Root Directory] + -- only shift if lsp root is different from the rootDir + -- see Note [Root Directory] + let + recorder = ctxRecorder initParams + defaultRoot = ctxDefaultRoot initParams + untilReactorStopSignal = ctxUntilReactorStopSignal initParams + lifetimeConfirm = ctxconfirmReactorShutdown initParams root <- case LSP.resRootPath env of - Just lspRoot | lspRoot /= defaultRoot -> setCurrentDirectory lspRoot >> return lspRoot - _ -> pure defaultRoot - dbLoc <- getHieDbLoc root + Just lspRoot | lspRoot /= defaultRoot -> setCurrentDirectory lspRoot >> return lspRoot + _ -> pure defaultRoot + dbLoc <- ctxGetHieDbLoc initParams root let initConfig = parseConfiguration params logWith recorder Info $ LogRegisteringIdeConfig initConfig - dbMVar <- newEmptyMVar - - - let handleServerException (Left e) = do - logWith recorder Error $ LogReactorThreadException e - exitClientMsg - handleServerException (Right _) = pure () + ideMVar <- newEmptyMVar + + let handleServerExceptionOrShutDown me = do + -- shutdown shake + tryReadMVar ideMVar >>= mapM_ shutdown + case me of + Left e -> do + lifetimeConfirm "due to exception in reactor thread" + logWith recorder Error $ LogReactorThreadException e + ctxForceShutdown initParams + _ -> do + lifetimeConfirm "due to shutdown message" + return () exceptionInHandler e = do logWith recorder Error $ LogReactorMessageActionException e @@ -235,13 +320,13 @@ handleInit recorder defaultRoot getHieDbLoc getIdeState lifetime exitClientMsg c checkCancelled :: forall m . LspId m -> IO () -> (TResponseError m -> IO ()) -> IO () checkCancelled _id act k = let sid = SomeLspId _id - in flip finally (clearReqId sid) $ + in flip finally (ctxClearReqId initParams sid) $ catch (do -- We could optimize this by first checking if the id -- is in the cancelled set. However, this is unlikely to be a -- bottleneck and the additional check might hide -- issues with async exceptions that need to be fixed. - cancelOrRes <- race (waitForCancel sid) act + cancelOrRes <- race (ctxWaitForCancel initParams sid) act case cancelOrRes of Left () -> do logWith recorder Debug $ LogCancelledRequest sid @@ -250,32 +335,40 @@ handleInit recorder defaultRoot getHieDbLoc getIdeState lifetime exitClientMsg c ) $ \(e :: SomeException) -> do exceptionInHandler e k $ TResponseError (InR ErrorCodes_InternalError) (T.pack $ show e) Nothing - _ <- flip forkFinally handleServerException $ do - untilMVar lifetime $ runWithWorkerThreads (cmapWithPrio LogSession recorder) dbLoc $ \withHieDb' threadQueue' -> do - putMVar dbMVar (WithHieDbShield withHieDb',threadQueue') - forever $ do - msg <- readChan clientMsgChan - -- We dispatch notifications synchronously and requests asynchronously - -- This is to ensure that all file edits and config changes are applied before a request is handled - case msg of - ReactorNotification act -> handle exceptionInHandler act - ReactorRequest _id act k -> void $ async $ checkCancelled _id act k - logWith recorder Info LogReactorThreadStopped - - (WithHieDbShield withHieDb, threadQueue) <- takeMVar dbMVar - ide <- getIdeState env root withHieDb threadQueue + _ <- flip forkFinally handleServerExceptionOrShutDown $ do + runWithWorkerThreads recorder ideMVar dbLoc $ \withHieDb' threadQueue' -> + do + ide <- ctxGetIdeState initParams env root withHieDb' threadQueue' + putMVar ideMVar ide + -- We might be blocked indefinitly at initialization if reactorStop is signaled + -- before we putMVar. + untilReactorStopSignal $ forever $ do + msg <- readChan $ ctxClientMsgChan initParams + -- We dispatch notifications synchronously and requests asynchronously + -- This is to ensure that all file edits and config changes are applied before a request is handled + case msg of + ReactorNotification act -> handle exceptionInHandler act + ReactorRequest _id act k -> void $ async $ checkCancelled _id act k + + ide <- readMVar ideMVar registerIdeConfiguration (shakeExtras ide) initConfig pure $ Right (env,ide) +runShakeThread :: Recorder (WithPriority Log) -> MVar IdeState -> ContT () IO DBQue +runShakeThread recorder mide = + withWorkerQueue + (logWith (cmapWithPrio (LogSession . Session.LogSessionWorkerThread) recorder) Debug) + "ShakeShakeControlQueue" + (eitherWorker (runRestartTaskDyn (cmapWithPrio LogShake recorder) mide) id) -- | runWithWorkerThreads -- create several threads to run the session, db and session loader -- see Note [Serializing runs in separate thread] -runWithWorkerThreads :: Recorder (WithPriority Session.Log) -> FilePath -> (WithHieDb -> ThreadQueue -> IO ()) -> IO () -runWithWorkerThreads recorder dbLoc f = evalContT $ do - sessionRestartTQueue <- withWorkerQueue id - sessionLoaderTQueue <- withWorkerQueue id - (WithHieDbShield hiedb, threadQueue) <- runWithDb recorder dbLoc +runWithWorkerThreads :: Recorder (WithPriority Log) -> MVar IdeState -> FilePath -> (WithHieDb -> ThreadQueue -> IO ()) -> IO () +runWithWorkerThreads recorder mide dbLoc f = evalContT $ do + (WithHieDbShield hiedb, threadQueue) <- runWithDb (cmapWithPrio LogSession recorder) dbLoc + sessionRestartTQueue <- runShakeThread recorder mide + sessionLoaderTQueue <- withWorkerQueueSimple (logWith (cmapWithPrio (LogSession . Session.LogSessionWorkerThread) recorder) Debug) "SessionLoaderTQueue" liftIO $ f hiedb (ThreadQueue threadQueue sessionRestartTQueue sessionLoaderTQueue) -- | Runs the action until it ends or until the given MVar is put. @@ -286,6 +379,9 @@ runWithWorkerThreads recorder dbLoc f = evalContT $ do untilMVar :: MonadUnliftIO m => MVar () -> m a -> m () untilMVar mvar io = race_ (readMVar mvar) io +untilMVar' :: MonadUnliftIO m => MVar a -> m b -> m (Either a b) +untilMVar' mvar io = race (readMVar mvar) io + cancelHandler :: (SomeLspId -> IO ()) -> LSP.Handlers (ServerM c) cancelHandler cancelRequest = LSP.notificationHandler SMethod_CancelRequest $ \TNotificationMessage{_params=CancelParams{_id}} -> liftIO $ cancelRequest (SomeLspId (toLspId _id)) @@ -294,17 +390,16 @@ cancelHandler cancelRequest = LSP.notificationHandler SMethod_CancelRequest $ \T toLspId (InR y) = IdString y shutdownHandler :: Recorder (WithPriority Log) -> IO () -> LSP.Handlers (ServerM c) -shutdownHandler recorder stopReactor = LSP.requestHandler SMethod_Shutdown $ \_ resp -> do - (_, ide) <- ask - liftIO $ logWith recorder Debug LogServerShutdownMessage - -- stop the reactor to free up the hiedb connection - liftIO stopReactor - -- flush out the Shake session to record a Shake profile if applicable - liftIO $ shakeShut ide +shutdownHandler _recorder requestReactorShutdown = LSP.requestHandler SMethod_Shutdown $ \_ resp -> do + -- stop the reactor to free up the hiedb connection and shut down shake + logWith _recorder Info $ LogText "Shutdown requested" + liftIO requestReactorShutdown resp $ Right Null -exitHandler :: IO () -> LSP.Handlers (ServerM c) -exitHandler exit = LSP.notificationHandler SMethod_Exit $ const $ liftIO exit +exitHandler :: Recorder (WithPriority Log) -> IO () -> LSP.Handlers (ServerM c) +exitHandler _recorder exit = LSP.notificationHandler SMethod_Exit $ \_ -> do + -- stop the reactor to free up the hiedb connection and shut down shake + liftIO exit modifyOptions :: LSP.Options -> LSP.Options modifyOptions x = x{ LSP.optTextDocumentSync = Just $ tweakTDS origTDS diff --git a/ghcide/src/Development/IDE/Main.hs b/ghcide/src/Development/IDE/Main.hs index ad4a36327a..6b791acd5e 100644 --- a/ghcide/src/Development/IDE/Main.hs +++ b/ghcide/src/Development/IDE/Main.hs @@ -77,8 +77,9 @@ import Development.IDE.Types.Location (NormalizedUri, toNormalizedFilePath') import Development.IDE.Types.Monitoring (Monitoring) import Development.IDE.Types.Options (IdeGhcSession, - IdeOptions (optCheckParents, optCheckProject, optReportProgress, optRunSubset), + IdeOptions (..), IdeTesting (IdeTesting), + ProgressReportingStyle (TestReporting), clientSupportsProgress, defaultIdeOptions, optModifyDynFlags, @@ -276,7 +277,10 @@ testing recorder projectRoot plugins = let defOptions = argsIdeOptions config sessionLoader in - defOptions{ optTesting = IdeTesting True } + defOptions{ + optTesting = IdeTesting True + , optProgressStyle = TestReporting + } lspOptions = argsLspOptions { LSP.optProgressStartDelay = 0, LSP.optProgressUpdateDelay = 0 } in arguments @@ -374,7 +378,8 @@ defaultMain recorder Arguments{..} = withHeapStats (cmapWithPrio LogHeapStats re Check argFiles -> do let dir = argsProjectRoot dbLoc <- getHieDbLoc dir - runWithWorkerThreads (cmapWithPrio LogSession recorder) dbLoc $ \hiedb threadQueue -> do + ideMVar <- newEmptyMVar + runWithWorkerThreads (cmapWithPrio LogLanguageServer recorder) ideMVar dbLoc $ \hiedb threadQueue -> do -- GHC produces messages with UTF8 in them, so make sure the terminal doesn't error hSetEncoding stdout utf8 hSetEncoding stderr utf8 @@ -403,6 +408,7 @@ defaultMain recorder Arguments{..} = withHeapStats (cmapWithPrio LogHeapStats re , optModifyDynFlags = optModifyDynFlags def_options <> pluginModifyDynflags plugins } ide <- initialise (cmapWithPrio LogService recorder) argsDefaultHlsConfig argsHlsPlugins rules Nothing debouncer ideOptions hiedb threadQueue mempty dir + putMVar ideMVar ide shakeSessionInit (cmapWithPrio LogShake recorder) ide registerIdeConfiguration (shakeExtras ide) $ IdeConfiguration mempty (hashed Nothing) @@ -432,7 +438,8 @@ defaultMain recorder Arguments{..} = withHeapStats (cmapWithPrio LogHeapStats re Custom (IdeCommand c) -> do let root = argsProjectRoot dbLoc <- getHieDbLoc root - runWithWorkerThreads (cmapWithPrio LogSession recorder) dbLoc $ \hiedb threadQueue -> do + ideMVar <- newEmptyMVar + runWithWorkerThreads (cmapWithPrio LogLanguageServer recorder) ideMVar dbLoc $ \hiedb threadQueue -> do sessionLoader <- loadSessionWithOptions (cmapWithPrio LogSession recorder) argsSessionLoadingOptions "." (tLoaderQueue threadQueue) let def_options = argsIdeOptions argsDefaultHlsConfig sessionLoader ideOptions = def_options @@ -441,6 +448,7 @@ defaultMain recorder Arguments{..} = withHeapStats (cmapWithPrio LogHeapStats re , optModifyDynFlags = optModifyDynFlags def_options <> pluginModifyDynflags plugins } ide <- initialise (cmapWithPrio LogService recorder) argsDefaultHlsConfig argsHlsPlugins rules Nothing debouncer ideOptions hiedb threadQueue mempty root + putMVar ideMVar ide shakeSessionInit (cmapWithPrio LogShake recorder) ide registerIdeConfiguration (shakeExtras ide) $ IdeConfiguration mempty (hashed Nothing) c ide diff --git a/ghcide/src/Development/IDE/Plugin/Completions/Logic.hs b/ghcide/src/Development/IDE/Plugin/Completions/Logic.hs index 0a5cecaca8..6c59a5ffe5 100644 --- a/ghcide/src/Development/IDE/Plugin/Completions/Logic.hs +++ b/ghcide/src/Development/IDE/Plugin/Completions/Logic.hs @@ -857,7 +857,7 @@ mergeListsBy cmp all_lists = merge_lists all_lists -- |From the given cursor position, gets the prefix module or record for autocompletion getCompletionPrefix :: Position -> VFS.VirtualFile -> PosPrefixInfo -getCompletionPrefix pos (VFS.VirtualFile _ _ ropetext) = getCompletionPrefixFromRope pos ropetext +getCompletionPrefix pos (VFS.VirtualFile _ _ ropetext _) = getCompletionPrefixFromRope pos ropetext getCompletionPrefixFromRope :: Position -> Rope.Rope -> PosPrefixInfo getCompletionPrefixFromRope pos@(Position l c) ropetext = diff --git a/ghcide/src/Development/IDE/Plugin/Test.hs b/ghcide/src/Development/IDE/Plugin/Test.hs index e24bcfeee9..8c0733b22f 100644 --- a/ghcide/src/Development/IDE/Plugin/Test.hs +++ b/ghcide/src/Development/IDE/Plugin/Test.hs @@ -39,7 +39,7 @@ import Development.IDE.Graph.Database (ShakeDatabase, shakeGetBuildStep, shakeGetCleanKeys) import Development.IDE.Graph.Internal.Types (Result (resultBuilt, resultChanged, resultVisited), - Step (Step)) + Step (..)) import qualified Development.IDE.Graph.Internal.Types as Graph import Development.IDE.Types.Action import Development.IDE.Types.HscEnvEq (HscEnvEq (hscEnv)) @@ -53,7 +53,6 @@ import qualified "list-t" ListT import qualified StmContainers.Map as STM import System.Time.Extra -type Age = Int data TestRequest = BlockSeconds Seconds -- ^ :: Null | GetInterfaceFilesDir Uri -- ^ :: String @@ -64,7 +63,6 @@ data TestRequest | GetBuildKeysBuilt -- ^ :: [(String] | GetBuildKeysChanged -- ^ :: [(String] | GetBuildEdgesCount -- ^ :: Int - | GarbageCollectDirtyKeys CheckParents Age -- ^ :: [String] (list of keys collected) | GetStoredKeys -- ^ :: [String] (list of keys in store) | GetFilesOfInterest -- ^ :: [FilePath] | GetRebuildsCount -- ^ :: Int (number of times we recompiled with GHC) @@ -126,11 +124,8 @@ testRequestHandler s GetBuildKeysVisited = liftIO $ do testRequestHandler s GetBuildEdgesCount = liftIO $ do count <- shakeGetBuildEdges $ shakeDb s return $ Right $ toJSON count -testRequestHandler s (GarbageCollectDirtyKeys parents age) = do - res <- liftIO $ runAction "garbage collect dirty" s $ garbageCollectDirtyKeysOlderThan age parents - return $ Right $ toJSON $ map show res testRequestHandler s GetStoredKeys = do - keys <- liftIO $ atomically $ map fst <$> ListT.toList (STM.listT $ state $ shakeExtras s) + keys <- liftIO $ atomically $ map fst <$> ListT.toList (STM.listT $ stateValues $ shakeExtras s) return $ Right $ toJSON $ map show keys testRequestHandler s GetFilesOfInterest = do ff <- liftIO $ getFilesOfInterest s diff --git a/ghcide/src/Development/IDE/Types/Action.hs b/ghcide/src/Development/IDE/Types/Action.hs index 0aedd1d0da..225f5b603d 100644 --- a/ghcide/src/Development/IDE/Types/Action.hs +++ b/ghcide/src/Development/IDE/Types/Action.hs @@ -7,7 +7,9 @@ module Development.IDE.Types.Action popQueue, doneQueue, peekInProgress, - abortQueue,countQueue) + abortQueue, + countQueue, + isActionQueueEmpty) where import Control.Concurrent.STM @@ -86,3 +88,9 @@ countQueue ActionQueue{..} = do peekInProgress :: ActionQueue -> STM [DelayedActionInternal] peekInProgress ActionQueue {..} = Set.toList <$> readTVar inProgress + +isActionQueueEmpty :: ActionQueue -> STM Bool +isActionQueueEmpty ActionQueue {..} = do + emptyQueue <- isEmptyTQueue newActions + inProg <- Set.null <$> readTVar inProgress + return (emptyQueue && inProg) diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs index 1c2ed1732f..e14ab56847 100644 --- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs +++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs @@ -15,7 +15,6 @@ import Control.Exception (evaluate, mask, throwIO) import Control.Monad.Extra (eitherM, join, mapMaybeM) import Data.Either (fromRight) import Data.IORef -import qualified Data.Map as M import Data.Unique (Unique) import qualified Data.Unique as Unique import Development.IDE.GHC.Compat hiding (newUnique) @@ -25,9 +24,7 @@ import Development.IDE.GHC.Util (lookupPackageConfig) import Development.IDE.Graph.Classes import Development.IDE.Types.Exports (ExportsMap, createExportsMap) import GHC.Driver.Env (hsc_all_home_unit_ids) -import Ide.PluginUtils (toAbsolute) import OpenTelemetry.Eventlog (withSpan) -import System.Directory (makeAbsolute) -- | An 'HscEnv' with equality. Two values are considered equal @@ -54,7 +51,6 @@ newHscEnvEq :: HscEnv -> IO HscEnvEq newHscEnvEq hscEnv' = do mod_cache <- newIORef emptyInstalledModuleEnv - file_cache <- newIORef M.empty -- This finder cache is for things which are outside of things which are tracked -- by HLS. For example, non-home modules, dependent object files etc #if MIN_VERSION_ghc(9,11,0) diff --git a/ghcide/src/Development/IDE/Types/Options.hs b/ghcide/src/Development/IDE/Types/Options.hs index 8d4d91e166..124e7a9469 100644 --- a/ghcide/src/Development/IDE/Types/Options.hs +++ b/ghcide/src/Development/IDE/Types/Options.hs @@ -107,6 +107,7 @@ newtype IdeOTMemoryProfiling = IdeOTMemoryProfiling Bool data ProgressReportingStyle = Percentage -- ^ Report using the LSP @_percentage@ field | Explicit -- ^ Report using explicit 123/456 text + | TestReporting -- ^ Special mode for testing, reports only start/stop | NoProgress -- ^ Do not report any percentage deriving Eq diff --git a/hlint.eventlog b/hlint.eventlog new file mode 100644 index 0000000000..501382a694 Binary files /dev/null and b/hlint.eventlog differ diff --git a/hls-graph/hls-graph.cabal b/hls-graph/hls-graph.cabal index 5eccb4d75e..b1553580d3 100644 --- a/hls-graph/hls-graph.cabal +++ b/hls-graph/hls-graph.cabal @@ -65,11 +65,14 @@ library Development.IDE.Graph.KeyMap Development.IDE.Graph.KeySet Development.IDE.Graph.Rule + Development.IDE.WorkerThread Paths_hls_graph autogen-modules: Paths_hls_graph hs-source-dirs: src build-depends: + , mtl ^>=2.3.1 + , safe-exceptions ^>=0.1.7.4 , aeson , async >=2.0 , base >=4.12 && <5 @@ -92,6 +95,7 @@ library , transformers , unliftio , unordered-containers + , prettyprinter if flag(embed-files) cpp-options: -DFILE_EMBED @@ -129,6 +133,7 @@ test-suite tests -threaded -rtsopts -with-rtsopts=-N -fno-ignore-asserts build-depends: + , transformers , base , extra , hls-graph diff --git a/hls-graph/src/Development/IDE/Graph.hs b/hls-graph/src/Development/IDE/Graph.hs index 81ad3b3dfd..bb973c6130 100644 --- a/hls-graph/src/Development/IDE/Graph.hs +++ b/hls-graph/src/Development/IDE/Graph.hs @@ -5,7 +5,7 @@ module Development.IDE.Graph( Action, action, pattern Key, newKey, renderKey, - actionFinally, actionBracket, actionCatch, actionFork, + actionFinally, actionBracket, actionCatch, -- * Configuration ShakeOptions(shakeAllowRedefineRules, shakeExtra), getShakeExtra, getShakeExtraRules, newShakeExtra, @@ -18,6 +18,7 @@ module Development.IDE.Graph( -- * Actions for inspecting the keys in the database getDirtySet, getKeysAndVisitedAge, + module Development.IDE.Graph.KeyMap, module Development.IDE.Graph.KeySet, ) where diff --git a/hls-graph/src/Development/IDE/Graph/Database.hs b/hls-graph/src/Development/IDE/Graph/Database.hs index bd8601cd16..0c942e9074 100644 --- a/hls-graph/src/Development/IDE/Graph/Database.hs +++ b/hls-graph/src/Development/IDE/Graph/Database.hs @@ -4,15 +4,27 @@ module Development.IDE.Graph.Database( shakeNewDatabase, shakeRunDatabase, shakeRunDatabaseForKeys, + shakeRunDatabaseForKeysSep, shakeProfileDatabase, shakeGetBuildStep, shakeGetDatabaseKeys, shakeGetDirtySet, shakeGetCleanKeys - ,shakeGetBuildEdges) where -import Control.Concurrent.STM.Stats (readTVarIO) + ,shakeGetBuildEdges, + shakeShutDatabase, + shakeGetActionQueueLength, + shakeComputeToPreserve, + shakedatabaseRuntimeDep, + shakePeekAsyncsDelivers) where +import Control.Concurrent.Async (Async) +import Control.Concurrent.STM.Stats (atomically, + readTVarIO) +import Control.Exception (SomeException) +import Control.Monad (join) import Data.Dynamic +import Data.HashMap.Strict (toList) import Data.Maybe +import Data.Set (Set) import Development.IDE.Graph.Classes () import Development.IDE.Graph.Internal.Action import Development.IDE.Graph.Internal.Database @@ -21,20 +33,24 @@ import Development.IDE.Graph.Internal.Options import Development.IDE.Graph.Internal.Profile (writeProfile) import Development.IDE.Graph.Internal.Rules import Development.IDE.Graph.Internal.Types +import Development.IDE.WorkerThread (DeliverStatus) -- Placeholder to be the 'extra' if the user doesn't set it data NonExportedType = NonExportedType -shakeNewDatabase :: ShakeOptions -> Rules () -> IO ShakeDatabase -shakeNewDatabase opts rules = do +shakeShutDatabase :: Set (Async ()) -> ShakeDatabase -> IO () +shakeShutDatabase preserve (ShakeDatabase _ _ db) = shutDatabase preserve db + +shakeNewDatabase :: (String -> IO ()) -> DBQue -> ShakeOptions -> Rules () -> IO ShakeDatabase +shakeNewDatabase l que opts rules = do let extra = fromMaybe (toDyn NonExportedType) $ shakeExtra opts (theRules, actions) <- runRules extra rules - db <- newDatabase extra theRules + db <- newDatabase l que extra theRules pure $ ShakeDatabase (length actions) actions db -shakeRunDatabase :: ShakeDatabase -> [Action a] -> IO [a] -shakeRunDatabase = shakeRunDatabaseForKeys Nothing +shakeRunDatabase :: ShakeDatabase -> [Action a] -> IO [Either SomeException a] +shakeRunDatabase s xs = shakeRunDatabaseForKeys Nothing s xs -- | Returns the set of dirty keys annotated with their age (in # of builds) shakeGetDirtySet :: ShakeDatabase -> IO [(Key, Int)] @@ -52,15 +68,39 @@ unvoid :: Functor m => m () -> m a unvoid = fmap undefined -- | Assumes that the database is not running a build -shakeRunDatabaseForKeys +-- The nested IO is to +-- seperate incrementing the step from running the build +shakeRunDatabaseForKeysSep :: Maybe [Key] -- ^ Set of keys changed since last run. 'Nothing' means everything has changed -> ShakeDatabase -> [Action a] - -> IO [a] -shakeRunDatabaseForKeys keysChanged (ShakeDatabase lenAs1 as1 db) as2 = do + -> IO (IO [Either SomeException a]) +shakeRunDatabaseForKeysSep keysChanged (ShakeDatabase lenAs1 as1 db) as2 = do incDatabase db keysChanged - fmap (drop lenAs1) $ runActions db $ map unvoid as1 ++ as2 + return $ drop lenAs1 <$> runActions (newKey "root") db (map unvoid as1 ++ as2) + +shakedatabaseRuntimeDep :: ShakeDatabase -> IO [(Key, KeySet)] +shakedatabaseRuntimeDep (ShakeDatabase _ _ db) = + atomically $ toList <$> computeReverseRuntimeMap db + + +shakeComputeToPreserve :: ShakeDatabase -> KeySet -> IO ([(Key, Async ())], [Key]) +shakeComputeToPreserve (ShakeDatabase _ _ db) ks = atomically (computeToPreserve db ks) + +--a dsfds +-- fds make it possible to do al ot of jobs +shakeRunDatabaseForKeys + :: Maybe [Key] + -- ^ Set of keys changed since last run. 'Nothing' means everything has changed + -> ShakeDatabase + -> [Action a] + -> IO [Either SomeException a] +shakeRunDatabaseForKeys keysChanged sdb as2 = join $ shakeRunDatabaseForKeysSep keysChanged sdb as2 + + +shakePeekAsyncsDelivers :: ShakeDatabase -> IO [DeliverStatus] +shakePeekAsyncsDelivers (ShakeDatabase _ _ db) = peekAsyncsDelivers db -- | Given a 'ShakeDatabase', write an HTML profile to the given file about the latest run. shakeProfileDatabase :: ShakeDatabase -> FilePath -> IO () @@ -83,3 +123,7 @@ shakeGetBuildEdges (ShakeDatabase _ _ db) = do -- annotated with how long ago (in # builds) they were visited shakeGetDatabaseKeys :: ShakeDatabase -> IO [(Key, Int)] shakeGetDatabaseKeys (ShakeDatabase _ _ db) = getKeysAndVisitAge db + +shakeGetActionQueueLength :: ShakeDatabase -> IO Int +shakeGetActionQueueLength (ShakeDatabase _ _ db) = + atomically $ databaseGetActionQueueLength db diff --git a/hls-graph/src/Development/IDE/Graph/Internal/Action.hs b/hls-graph/src/Development/IDE/Graph/Internal/Action.hs index 6d47d9b511..adac90f3b9 100644 --- a/hls-graph/src/Development/IDE/Graph/Internal/Action.hs +++ b/hls-graph/src/Development/IDE/Graph/Internal/Action.hs @@ -2,7 +2,6 @@ module Development.IDE.Graph.Internal.Action ( ShakeValue -, actionFork , actionBracket , actionCatch , actionFinally @@ -14,14 +13,17 @@ module Development.IDE.Graph.Internal.Action , runActions , Development.IDE.Graph.Internal.Action.getDirtySet , getKeysAndVisitedAge +, runActionInDbCb ) where import Control.Concurrent.Async +import Control.Concurrent.STM.Stats (atomicallyNamed) import Control.DeepSeq (force) import Control.Exception import Control.Monad.IO.Class +import Control.Monad.RWS (MonadReader (ask), + asks) import Control.Monad.Trans.Class -import Control.Monad.Trans.Reader import Data.Foldable (toList) import Data.Functor.Identity import Data.IORef @@ -31,66 +33,71 @@ import Development.IDE.Graph.Internal.Key import Development.IDE.Graph.Internal.Rules (RuleResult) import Development.IDE.Graph.Internal.Types import System.Exit +import UnliftIO (STM, atomically, + newEmptyTMVarIO, + putTMVar, readTMVar) type ShakeValue a = (Show a, Typeable a, Eq a, Hashable a, NFData a) -- | Always rerun this rule when dirty, regardless of the dependencies. alwaysRerun :: Action () alwaysRerun = do - ref <- Action $ asks actionDeps + ref <- asks actionDeps liftIO $ modifyIORef' ref (AlwaysRerunDeps mempty <>) -parallel :: [Action a] -> Action [a] -parallel [] = pure [] -parallel [x] = fmap (:[]) x +parallel :: [Action a] -> Action [Either SomeException a] +parallel [] = return [] parallel xs = do - a <- Action ask + a <- ask deps <- liftIO $ readIORef $ actionDeps a case deps of UnknownDeps -> -- if we are already in the rerun mode, nothing we do is going to impact our state - liftIO $ mapConcurrently (ignoreState a) xs - deps -> do - (newDeps, res) <- liftIO $ unzip <$> mapConcurrently (usingState a) xs - liftIO $ writeIORef (actionDeps a) $ mconcat $ deps : newDeps - pure res - where - usingState a x = do - ref <- newIORef mempty - res <- runReaderT (fromAction x) a{actionDeps=ref} - deps <- readIORef ref - pure (deps, res) + runActionInDb "parallel" xs + deps -> error $ "parallel not supported when we have precise dependencies: " ++ show deps + -- (newDeps, res) <- liftIO $ unzip <$> runActionInDb usingState xs + -- liftIO $ writeIORef (actionDeps a) $ mconcat $ deps : newDeps + -- return () + +-- non-blocking version of runActionInDb +runActionInDbCb :: (a -> String) -> (a -> Action result) -> STM a -> (Either SomeException result -> IO ()) -> Action a +runActionInDbCb getTitle work getAct handler = do + a <- ask + liftIO $ atomicallyNamed "action queue - pop" $ do + act <- getAct + runInDataBase (getTitle act) (actionDatabase a) [(ignoreState a $ work act, handler)] + return act + +runActionInDb :: String -> [Action a] -> Action [Either SomeException a] +runActionInDb title acts = do + a <- ask + xs <- mapM (\x -> do + barrier <- newEmptyTMVarIO + return (x, barrier)) acts + liftIO $ atomically $ runInDataBase title (actionDatabase a) + (map (\(x, b) -> (ignoreState a x, atomically . putTMVar b)) xs) + results <- liftIO $ mapM (atomically . readTMVar) $ fmap snd xs + return results ignoreState :: SAction -> Action b -> IO b ignoreState a x = do ref <- newIORef mempty - runReaderT (fromAction x) a{actionDeps=ref} - -actionFork :: Action a -> (Async a -> Action b) -> Action b -actionFork act k = do - a <- Action ask - deps <- liftIO $ readIORef $ actionDeps a - let db = actionDatabase a - case deps of - UnknownDeps -> do - -- if we are already in the rerun mode, nothing we do is going to impact our state - [res] <- liftIO $ withAsync (ignoreState a act) $ \as -> runActions db [k as] - return res - _ -> - error "please help me" + runActionMonad x a{actionDeps=ref} isAsyncException :: SomeException -> Bool isAsyncException e + | Just (_ :: SomeAsyncException) <- fromException e = True | Just (_ :: AsyncCancelled) <- fromException e = True | Just (_ :: AsyncException) <- fromException e = True + | Just (_ :: AsyncParentKill) <- fromException e = True | Just (_ :: ExitCode) <- fromException e = True | otherwise = False actionCatch :: Exception e => Action a -> (e -> Action a) -> Action a actionCatch a b = do - v <- Action ask - Action $ lift $ catchJust f (runReaderT (fromAction a) v) (\x -> runReaderT (fromAction (b x)) v) + v <- ask + liftIO $ catchJust f (runActionMonad a v) (\x -> runActionMonad (b x) v) where -- Catch only catches exceptions that were caused by this code, not those that -- are a result of program termination @@ -99,23 +106,24 @@ actionCatch a b = do actionBracket :: IO a -> (a -> IO b) -> (a -> Action c) -> Action c actionBracket a b c = do - v <- Action ask - Action $ lift $ bracket a b (\x -> runReaderT (fromAction (c x)) v) + v <- ask + liftIO $ bracket a b (\x -> runActionMonad (c x) v) actionFinally :: Action a -> IO b -> Action a actionFinally a b = do v <- Action ask - Action $ lift $ finally (runReaderT (fromAction a) v) b + Action $ lift $ finally (runActionMonad a v) b apply1 :: (RuleResult key ~ value, ShakeValue key, Typeable value) => key -> Action value apply1 k = runIdentity <$> apply (Identity k) apply :: (Traversable f, RuleResult key ~ value, ShakeValue key, Typeable value) => f key -> Action (f value) apply ks = do - db <- Action $ asks actionDatabase - stack <- Action $ asks actionStack - (is, vs) <- liftIO $ build db stack ks - ref <- Action $ asks actionDeps + db <- asks actionDatabase + stack <- asks actionStack + pk <- getActionKey + (is, vs) <- liftIO $ build pk db stack ks + ref <- asks actionDeps let !ks = force $ fromListKeySet $ toList is liftIO $ modifyIORef' ref (ResultDeps [ks] <>) pure vs @@ -123,15 +131,16 @@ apply ks = do -- | Evaluate a list of keys without recording any dependencies. applyWithoutDependency :: (Traversable f, RuleResult key ~ value, ShakeValue key, Typeable value) => f key -> Action (f value) applyWithoutDependency ks = do - db <- Action $ asks actionDatabase - stack <- Action $ asks actionStack - (_, vs) <- liftIO $ build db stack ks + db <- asks actionDatabase + stack <- asks actionStack + pk <- getActionKey + (_, vs) <- liftIO $ build pk db stack ks pure vs -runActions :: Database -> [Action a] -> IO [a] -runActions db xs = do +runActions :: Key -> Database -> [Action a] -> IO [Either SomeException a] +runActions pk db xs = do deps <- newIORef mempty - runReaderT (fromAction $ parallel xs) $ SAction db deps emptyStack + runActionMonad (parallel xs) $ SAction pk db deps emptyStack -- | Returns the set of dirty keys annotated with their age (in # of builds) getDirtySet :: Action [(Key, Int)] diff --git a/hls-graph/src/Development/IDE/Graph/Internal/Database.hs b/hls-graph/src/Development/IDE/Graph/Internal/Database.hs index 359e5ceb6a..9102881299 100644 --- a/hls-graph/src/Development/IDE/Graph/Internal/Database.hs +++ b/hls-graph/src/Development/IDE/Graph/Internal/Database.hs @@ -8,16 +8,16 @@ {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE TypeFamilies #-} -module Development.IDE.Graph.Internal.Database (compute, newDatabase, incDatabase, build, getDirtySet, getKeysAndVisitAge) where +module Development.IDE.Graph.Internal.Database (compute, newDatabase, incDatabase, build, getDirtySet, getKeysAndVisitAge, AsyncParentKill(..)) where import Prelude hiding (unzip) -import Control.Concurrent.Async -import Control.Concurrent.Extra -import Control.Concurrent.STM.Stats (STM, atomically, - atomicallyNamed, - modifyTVar', newTVarIO, - readTVarIO) +import Control.Concurrent.STM.Stats (STM, atomicallyNamed, + check, modifyTVar', + newEmptyTMVarIO, + newTVarIO, putTMVar, + readTMVar, readTVar, + readTVarIO, retry) import Control.Exception import Control.Monad import Control.Monad.IO.Class (MonadIO (liftIO)) @@ -25,22 +25,24 @@ import Control.Monad.Trans.Class (lift) import Control.Monad.Trans.Reader import qualified Control.Monad.Trans.State.Strict as State import Data.Dynamic -import Data.Either import Data.Foldable (for_, traverse_) import Data.IORef.Extra import Data.Maybe import Data.Traversable (for) import Data.Tuple.Extra -import Debug.Trace (traceM) +import Debug.Trace (traceEvent) import Development.IDE.Graph.Classes import Development.IDE.Graph.Internal.Key import Development.IDE.Graph.Internal.Rules import Development.IDE.Graph.Internal.Types +import Development.IDE.WorkerThread (DeliverStatus (DeliverStatus)) import qualified Focus import qualified ListT import qualified StmContainers.Map as SMap -import System.IO.Unsafe -import System.Time.Extra (duration, sleep) +import System.Time.Extra (duration) +import UnliftIO (async, atomically, + newEmptyMVar, putMVar, + readMVar) #if MIN_VERSION_base(4,19,0) import Data.Functor (unzip) @@ -49,10 +51,13 @@ import Data.List.NonEmpty (unzip) #endif -newDatabase :: Dynamic -> TheRules -> IO Database -newDatabase databaseExtra databaseRules = do +newDatabase :: (String -> IO ()) -> DBQue -> Dynamic -> TheRules -> IO Database +newDatabase dataBaseLogger databaseQueue databaseExtra databaseRules = do databaseStep <- newTVarIO $ Step 0 + databaseThreads <- newTVarIO [] + databaseValuesLock <- newTVarIO True databaseValues <- atomically SMap.new + databaseRuntimeDep <- atomically SMap.new pure Database{..} -- | Increment the step and mark dirty. @@ -67,78 +72,103 @@ incDatabase db (Just kk) = do -- since we assume that no build is mutating the db. -- Therefore run one transaction per key to minimise contention. atomicallyNamed "incDatabase" $ SMap.focus updateDirty k (databaseValues db) - -- all keys are dirty incDatabase db Nothing = do atomically $ modifyTVar' (databaseStep db) $ \(Step i) -> Step $ i + 1 let list = SMap.listT (databaseValues db) + -- all running keys are also dirty atomicallyNamed "incDatabase - all " $ flip ListT.traverse_ list $ \(k,_) -> SMap.focus updateDirty k (databaseValues db) updateDirty :: Monad m => Focus.Focus KeyDetails m () updateDirty = Focus.adjust $ \(KeyDetails status rdeps) -> let status' - | Running _ _ _ x <- status = Dirty x + | Running _ x _ _ <- status = Dirty x | Clean x <- status = Dirty (Just x) | otherwise = status in KeyDetails status' rdeps -- | Unwrap and build a list of keys in parallel -build - :: forall f key value . (Traversable f, RuleResult key ~ value, Typeable key, Show key, Hashable key, Eq key, Typeable value) - => Database -> Stack -> f key -> IO (f Key, f value) +build :: + forall f key value. + (Traversable f, RuleResult key ~ value, Typeable key, Show key, Hashable key, Eq key, Typeable value) => + Key -> Database -> Stack -> f key -> IO (f Key, f value) -- build _ st k | traceShow ("build", st, k) False = undefined -build db stack keys = do - built <- runAIO $ do - built <- builder db stack (fmap newKey keys) - case built of - Left clean -> return clean - Right dirty -> liftIO dirty - let (ids, vs) = unzip built - pure (ids, fmap (asV . resultValue) vs) - where - asV :: Value -> value - asV (Value x) = unwrapDynamic x +build pk db stack keys = do + -- step <- readTVarIO $ databaseStep db + -- built <- mapConcurrently (builderOne db stack) (fmap newKey keys) + built <- builder pk db stack (fmap newKey keys) + let (ids, vs) = unzip built + pure (ids, fmap (asV . resultValue) vs) + where + asV :: Value -> value + asV (Value x) = unwrapDynamic x + -- | Build a list of keys and return their results. -- If none of the keys are dirty, we can return the results immediately. -- Otherwise, a blocking computation is returned *which must be evaluated asynchronously* to avoid deadlock. -builder - :: Traversable f => Database -> Stack -> f Key -> AIO (Either (f (Key, Result)) (IO (f (Key, Result)))) +builder :: (Traversable f) => Key -> Database -> Stack -> f Key -> IO (f (Key, Result)) -- builder _ st kk | traceShow ("builder", st,kk) False = undefined -builder db@Database{..} stack keys = withRunInIO $ \(RunInIO run) -> do - -- Things that I need to force before my results are ready - toForce <- liftIO $ newTVarIO [] - current <- liftIO $ readTVarIO databaseStep - results <- liftIO $ for keys $ \id -> - -- Updating the status of all the dependencies atomically is not necessary. - -- Therefore, run one transaction per dep. to avoid contention - atomicallyNamed "builder" $ do - -- Spawn the id if needed - status <- SMap.lookup id databaseValues - val <- case viewDirty current $ maybe (Dirty Nothing) keyStatus status of - Clean r -> pure r - Running _ force val _ - | memberStack id stack -> throw $ StackException stack - | otherwise -> do - modifyTVar' toForce (Wait force :) - pure val - Dirty s -> do - let act = run (refresh db stack id s) - (force, val) = splitIO (join act) - SMap.focus (updateStatus $ Running current force val s) id databaseValues - modifyTVar' toForce (Spawn force:) - pure val - - pure (id, val) - - toForceList <- liftIO $ readTVarIO toForce - let waitAll = run $ waitConcurrently_ toForceList - case toForceList of - [] -> return $ Left results - _ -> return $ Right $ do - waitAll - pure results - +builder pk db stack keys = do + waits <- for keys (\k -> builderOne pk db stack k) + for waits interpreBuildContinue + +-- the first run should not block +data BuildContinue = BCContinue (IO (Key, Result)) | BCStop Key Result + +interpreBuildContinue :: BuildContinue -> IO (Key, Result) +interpreBuildContinue (BCStop k v) = return (k, v) +interpreBuildContinue (BCContinue ioR) = ioR + +builderOne :: Key -> Database -> Stack -> Key -> IO BuildContinue +builderOne parentKey db@Database {..} stack id = do + traceEvent ("builderOne: " ++ show id) return () + barrier <- newEmptyMVar + liftIO $ atomicallyNamed "builder" $ do + -- Spawn the id if needed + dbNotLocked db + insertdatabaseRuntimeDep id parentKey db + -- if a build is running, wait + -- it will either be killed or continue + -- depending on wether it is marked as dirty + status <- SMap.lookup id databaseValues + current <- readTVar databaseStep + case viewDirty current $ maybe (Dirty Nothing) keyStatus status of + Dirty s -> do + -- we need to run serially to avoid summiting run but killed in the middle + let wait = readMVar barrier + runOneInDataBase + ( do + status <- atomically (SMap.lookup id databaseValues) + let cur = fromIntegral $ case keyStatus <$> status of + -- this is ensure that we get an bumped up step when not dirty + -- after an restart to skipped an rerun + Just (Running entryStep _s _wait RunningStage1) -> entryStep + _ -> current + return $ DeliverStatus cur (show (parentKey, id)) (newKey id) + ) + db + ( \adyncH -> + -- it is safe from worker thread + atomically $ SMap.focus (updateStatus $ Running current s wait (RunningStage2 adyncH)) id databaseValues + ) + (refresh db stack id s >>= putMVar barrier . (id,)) + $ \e -> do + atomically $ SMap.focus (updateStatus $ Exception current e s) id databaseValues + putMVar barrier (throw e) + SMap.focus (updateStatus $ Running current s wait RunningStage1) id databaseValues + return $ BCContinue $ readMVar barrier + Clean r -> return $ BCStop id r + Running _step _s wait _ + | memberStack id stack -> throw $ StackException stack + | otherwise -> return $ BCContinue wait + Exception _ e _s -> throw e + where + warpLog title a = + bracket_ + (dataBaseLogger ("Starting async action: " ++ title)) + (dataBaseLogger $ "Finished async action: " ++ title) + a -- | isDirty -- only dirty when it's build time is older than the changed time of one of its dependencies @@ -152,44 +182,38 @@ isDirty me = any (\(_,dep) -> resultBuilt me < resultChanged dep) -- * If no dirty dependencies and we have evaluated the key previously, then we refresh it in the current thread. -- This assumes that the implementation will be a lookup -- * Otherwise, we spawn a new thread to refresh the dirty deps (if any) and the key itself -refreshDeps :: KeySet -> Database -> Stack -> Key -> Result -> [KeySet] -> AIO Result +refreshDeps :: KeySet -> Database -> Stack -> Key -> Result -> [KeySet] -> IO Result refreshDeps visited db stack key result = \case -- no more deps to refresh - [] -> liftIO $ compute db stack key RunDependenciesSame (Just result) + [] -> compute db stack key RunDependenciesSame (Just result) (dep:deps) -> do let newVisited = dep <> visited - res <- builder db stack (toListKeySet (dep `differenceKeySet` visited)) - case res of - Left res -> if isDirty result res + res <- builder key db stack (toListKeySet (dep `differenceKeySet` visited)) + if isDirty result res -- restart the computation if any of the deps are dirty - then liftIO $ compute db stack key RunDependenciesChanged (Just result) + then compute db stack key RunDependenciesChanged (Just result) -- else kick the rest of the deps else refreshDeps newVisited db stack key result deps - Right iores -> do - res <- liftIO iores - if isDirty result res - then liftIO $ compute db stack key RunDependenciesChanged (Just result) - else refreshDeps newVisited db stack key result deps --- | Refresh a key: -refresh :: Database -> Stack -> Key -> Maybe Result -> AIO (IO Result) + +-- refresh :: Database -> Stack -> Key -> Maybe Result -> IO Result -- refresh _ st k _ | traceShow ("refresh", st, k) False = undefined +refresh :: Database -> Stack -> Key -> Maybe Result -> IO Result refresh db stack key result = case (addStack key stack, result) of (Left e, _) -> throw e - (Right stack, Just me@Result{resultDeps = ResultDeps deps}) -> asyncWithCleanUp $ refreshDeps mempty db stack key me (reverse deps) - (Right stack, _) -> - asyncWithCleanUp $ liftIO $ compute db stack key RunDependenciesChanged result - + (Right stack, Just me@Result{resultDeps = ResultDeps deps}) -> refreshDeps mempty db stack key me (reverse deps) + (Right stack, _) -> compute db stack key RunDependenciesChanged result -- | Compute a key. compute :: Database -> Stack -> Key -> RunMode -> Maybe Result -> IO Result -- compute _ st k _ _ | traceShow ("compute", st, k) False = undefined compute db@Database{..} stack key mode result = do let act = runRule databaseRules key (fmap resultData result) mode - deps <- newIORef UnknownDeps + deps <- liftIO $ newIORef UnknownDeps + curStep <- liftIO $ readTVarIO databaseStep + dataBaseLogger $ "Computing key: " ++ show key ++ " at step " ++ show curStep (execution, RunResult{..}) <- - duration $ runReaderT (fromAction act) $ SAction db deps stack - curStep <- readTVarIO databaseStep - deps <- readIORef deps + liftIO $ duration $ runReaderT (fromAction act) $ SAction key db deps stack + deps <- liftIO $ readIORef deps let lastChanged = maybe curStep resultChanged result let lastBuild = maybe curStep resultBuilt result -- changed time is always older than or equal to build time @@ -212,12 +236,13 @@ compute db@Database{..} stack key mode result = do -- If an async exception strikes before the deps have been recorded, -- we won't be able to accurately propagate dirtiness for this key -- on the next build. - void $ + liftIO $ void $ updateReverseDeps key db (getResultDepsDefault mempty previousDeps) deps _ -> pure () - atomicallyNamed "compute and run hook" $ do + liftIO $ atomicallyNamed "compute and run hook" $ do + dbNotLocked db runHook SMap.focus (updateStatus $ Clean res) key databaseValues pure res @@ -247,18 +272,6 @@ getKeysAndVisitAge db = do getAge Result{resultVisited = Step s} = curr - s return keysWithVisitAge -------------------------------------------------------------------------------- --- Lazy IO trick - -data Box a = Box {fromBox :: a} - --- | Split an IO computation into an unsafe lazy value and a forcing computation -splitIO :: IO a -> (IO (), a) -splitIO act = do - let act2 = Box <$> act - let res = unsafePerformIO act2 - (void $ evaluate res, fromBox res) - --------------------------------------------------------------------------------- -- Reverse dependencies -- | Update the reverse dependencies of an Id @@ -296,84 +309,5 @@ transitiveDirtySet database = flip State.execStateT mempty . traverse_ loop next <- lift $ atomically $ getReverseDependencies database x traverse_ loop (maybe mempty toListKeySet next) --------------------------------------------------------------------------------- --- Asynchronous computations with cancellation - --- | A simple monad to implement cancellation on top of 'Async', --- generalizing 'withAsync' to monadic scopes. -newtype AIO a = AIO { unAIO :: ReaderT (IORef [Async ()]) IO a } - deriving newtype (Applicative, Functor, Monad, MonadIO) - --- | Run the monadic computation, cancelling all the spawned asyncs if an exception arises -runAIO :: AIO a -> IO a -runAIO (AIO act) = do - asyncs <- newIORef [] - runReaderT act asyncs `onException` cleanupAsync asyncs - --- | Like 'async' but with built-in cancellation. --- Returns an IO action to wait on the result. -asyncWithCleanUp :: AIO a -> AIO (IO a) -asyncWithCleanUp act = do - st <- AIO ask - io <- unliftAIO act - -- mask to make sure we keep track of the spawned async - liftIO $ uninterruptibleMask $ \restore -> do - a <- async $ restore io - atomicModifyIORef'_ st (void a :) - return $ wait a - -unliftAIO :: AIO a -> AIO (IO a) -unliftAIO act = do - st <- AIO ask - return $ runReaderT (unAIO act) st - -newtype RunInIO = RunInIO (forall a. AIO a -> IO a) - -withRunInIO :: (RunInIO -> AIO b) -> AIO b -withRunInIO k = do - st <- AIO ask - k $ RunInIO (\aio -> runReaderT (unAIO aio) st) - -cleanupAsync :: IORef [Async a] -> IO () --- mask to make sure we interrupt all the asyncs -cleanupAsync ref = uninterruptibleMask $ \unmask -> do - asyncs <- atomicModifyIORef' ref ([],) - -- interrupt all the asyncs without waiting - mapM_ (\a -> throwTo (asyncThreadId a) AsyncCancelled) asyncs - -- Wait until all the asyncs are done - -- But if it takes more than 10 seconds, log to stderr - unless (null asyncs) $ do - let warnIfTakingTooLong = unmask $ forever $ do - sleep 10 - traceM "cleanupAsync: waiting for asyncs to finish" - withAsync warnIfTakingTooLong $ \_ -> - mapM_ waitCatch asyncs - -data Wait - = Wait {justWait :: !(IO ())} - | Spawn {justWait :: !(IO ())} - -fmapWait :: (IO () -> IO ()) -> Wait -> Wait -fmapWait f (Wait io) = Wait (f io) -fmapWait f (Spawn io) = Spawn (f io) -waitOrSpawn :: Wait -> IO (Either (IO ()) (Async ())) -waitOrSpawn (Wait io) = pure $ Left io -waitOrSpawn (Spawn io) = Right <$> async io -waitConcurrently_ :: [Wait] -> AIO () -waitConcurrently_ [] = pure () -waitConcurrently_ [one] = liftIO $ justWait one -waitConcurrently_ many = do - ref <- AIO ask - -- spawn the async computations. - -- mask to make sure we keep track of all the asyncs. - (asyncs, syncs) <- liftIO $ uninterruptibleMask $ \unmask -> do - waits <- liftIO $ traverse (waitOrSpawn . fmapWait unmask) many - let (syncs, asyncs) = partitionEithers waits - liftIO $ atomicModifyIORef'_ ref (asyncs ++) - return (asyncs, syncs) - -- work on the sync computations - liftIO $ sequence_ syncs - -- wait for the async computations before returning - liftIO $ traverse_ wait asyncs diff --git a/hls-graph/src/Development/IDE/Graph/Internal/Key.hs b/hls-graph/src/Development/IDE/Graph/Internal/Key.hs index 85cebeb110..71760586cc 100644 --- a/hls-graph/src/Development/IDE/Graph/Internal/Key.hs +++ b/hls-graph/src/Development/IDE/Graph/Internal/Key.hs @@ -31,6 +31,7 @@ module Development.IDE.Graph.Internal.Key , fromListKeySet , deleteKeySet , differenceKeySet + , unionKyeSet ) where --import Control.Monad.IO.Class () @@ -47,15 +48,20 @@ import Data.Text (Text) import qualified Data.Text as T import Data.Typeable import Development.IDE.Graph.Classes +import Prettyprinter import System.IO.Unsafe newtype Key = UnsafeMkKey Int + pattern Key :: () => (Typeable a, Hashable a, Show a) => a -> Key pattern Key a <- (lookupKeyValue -> KeyValue a _) {-# COMPLETE Key #-} +instance Pretty Key where + pretty = pretty . renderKey + data KeyValue = forall a . (Typeable a, Hashable a, Show a) => KeyValue a Text instance Eq KeyValue where @@ -111,6 +117,9 @@ renderKey (lookupKeyValue -> KeyValue _ t) = t newtype KeySet = KeySet IntSet deriving newtype (Eq, Ord, Semigroup, Monoid, NFData) +instance Pretty KeySet where + pretty (KeySet is) = pretty (coerce (IS.toList is) :: [Key]) + instance Show KeySet where showsPrec p (KeySet is)= showParen (p > 10) $ showString "fromList " . shows ks @@ -131,6 +140,10 @@ nullKeySet = coerce IS.null differenceKeySet :: KeySet -> KeySet -> KeySet differenceKeySet = coerce IS.difference + +unionKyeSet :: KeySet -> KeySet -> KeySet +unionKyeSet = coerce IS.union + deleteKeySet :: Key -> KeySet -> KeySet deleteKeySet = coerce IS.delete diff --git a/hls-graph/src/Development/IDE/Graph/Internal/Types.hs b/hls-graph/src/Development/IDE/Graph/Internal/Types.hs index 34bed42391..447a9f9e8f 100644 --- a/hls-graph/src/Development/IDE/Graph/Internal/Types.hs +++ b/hls-graph/src/Development/IDE/Graph/Internal/Types.hs @@ -1,15 +1,18 @@ {-# LANGUAGE CPP #-} {-# LANGUAGE DeriveAnyClass #-} {-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE RecordWildCards #-} module Development.IDE.Graph.Internal.Types where -import Control.Concurrent.STM (STM) -import Control.Monad ((>=>)) +import Control.Concurrent.STM (STM, check, modifyTVar') +import Control.Monad (forM, forM_, forever, + unless, when) import Control.Monad.Catch import Control.Monad.IO.Class -import Control.Monad.Trans.Reader +import Control.Monad.RWS (MonadReader (local), asks) +import Control.Monad.Trans.Reader (ReaderT (..)) import Data.Aeson (FromJSON, ToJSON) import Data.Bifunctor (second) import qualified Data.ByteString as BS @@ -18,17 +21,35 @@ import Data.Foldable (fold) import qualified Data.HashMap.Strict as Map import Data.IORef import Data.List (intercalate) -import Data.Maybe +import Data.Maybe (fromMaybe, isJust, + isNothing) +import Data.Set (Set) +import qualified Data.Set as S import Data.Typeable +import Debug.Trace (traceEventIO) import Development.IDE.Graph.Classes import Development.IDE.Graph.Internal.Key +import Development.IDE.WorkerThread (DeliverStatus (..), + TaskQueue (..), + awaitRunInThread, + counTaskQueue, + writeTaskQueue) +import qualified Focus import GHC.Conc (TVar, atomically) import GHC.Generics (Generic) import qualified ListT import qualified StmContainers.Map as SMap import StmContainers.Map (Map) -import System.Time.Extra (Seconds) -import UnliftIO (MonadUnliftIO) +import System.Time.Extra (Seconds, sleep) +import UnliftIO (Async (asyncThreadId), + MonadUnliftIO, async, + asyncExceptionFromException, + asyncExceptionToException, + poll, readTVar, readTVarIO, + throwTo, waitCatch, + withAsync) +import UnliftIO.Concurrent (ThreadId, myThreadId) +import qualified UnliftIO.Exception as UE #if !MIN_VERSION_base(4,18,0) import Control.Applicative (liftA2) @@ -68,20 +89,30 @@ data SRules = SRules { -- 'Development.IDE.Graph.Internal.Action.actionCatch'. In particular, it is -- permissible to use the 'MonadFail' instance, which will lead to an 'IOException'. newtype Action a = Action {fromAction :: ReaderT SAction IO a} - deriving newtype (Monad, Applicative, Functor, MonadIO, MonadFail, MonadThrow, MonadCatch, MonadMask, MonadUnliftIO) + deriving newtype (Monad, Applicative, Functor, MonadIO, MonadFail, MonadThrow, MonadCatch, MonadMask, MonadUnliftIO, MonadReader SAction) + +runActionMonad :: Action a -> SAction -> IO a +runActionMonad (Action r) s = runReaderT r s data SAction = SAction { + actionKey :: !Key, actionDatabase :: !Database, actionDeps :: !(IORef ResultDeps), actionStack :: !Stack } getDatabase :: Action Database -getDatabase = Action $ asks actionDatabase +getDatabase = asks actionDatabase + +getActionKey :: Action Key +getActionKey = asks actionKey + +setActionKey :: Key -> Action a -> Action a +setActionKey k act = local (\s' -> s'{actionKey = k}) act -- | waitForDatabaseRunningKeysAction waits for all keys in the database to finish running. -waitForDatabaseRunningKeysAction :: Action () -waitForDatabaseRunningKeysAction = getDatabase >>= liftIO . waitForDatabaseRunningKeys +-- waitForDatabaseRunningKeysAction :: Action () +-- waitForDatabaseRunningKeysAction = getDatabase >>= liftIO . waitForDatabaseRunningKeys --------------------------------------------------------------------- -- DATABASE @@ -89,14 +120,36 @@ waitForDatabaseRunningKeysAction = getDatabase >>= liftIO . waitForDatabaseRunni data ShakeDatabase = ShakeDatabase !Int [Action ()] Database newtype Step = Step Int - deriving newtype (Eq,Ord,Hashable,Show) + deriving newtype (Eq,Ord,Hashable,Show,Num,Enum,Real,Integral) ---------------------------------------------------------------------- --- Keys +getShakeStep :: MonadIO m => ShakeDatabase -> m Step +getShakeStep (ShakeDatabase _ _ db) = do + s <- readTVarIO $ databaseStep db + return s + +lockShakeDatabaseValues :: MonadIO m => ShakeDatabase -> m () +lockShakeDatabaseValues (ShakeDatabase _ _ db) = do + liftIO $ atomically $ modifyTVar' (databaseValuesLock db) (const False) +unlockShakeDatabaseValues :: MonadIO m => ShakeDatabase -> m () +unlockShakeDatabaseValues (ShakeDatabase _ _ db) = do + liftIO $ atomically $ modifyTVar' (databaseValuesLock db) (const True) +withShakeDatabaseValuesLock :: ShakeDatabase -> IO c -> IO c +withShakeDatabaseValuesLock sdb act = do + UE.bracket_ (lockShakeDatabaseValues sdb) (unlockShakeDatabaseValues sdb) act +dbNotLocked :: Database -> STM () +dbNotLocked db = do + check =<< readTVar (databaseValuesLock db) + + + +getShakeQueue :: ShakeDatabase -> DBQue +getShakeQueue (ShakeDatabase _ _ db) = databaseQueue db +--------------------------------------------------------------------- +-- Keys newtype Value = Value Dynamic data KeyDetails = KeyDetails { @@ -108,15 +161,221 @@ onKeyReverseDeps :: (KeySet -> KeySet) -> KeyDetails -> KeyDetails onKeyReverseDeps f it@KeyDetails{..} = it{keyReverseDeps = f keyReverseDeps} + +type DBQue = TaskQueue (Either Dynamic (IO ())) data Database = Database { - databaseExtra :: Dynamic, - databaseRules :: TheRules, - databaseStep :: !(TVar Step), - databaseValues :: !(Map Key KeyDetails) + databaseExtra :: Dynamic, + + databaseThreads :: TVar [(DeliverStatus, Async ())], + + databaseRuntimeDep :: SMap.Map Key KeySet, + -- it is used to compute the transitive reverse deps, so + -- if not in any of the transitive reverse deps of a dirty node, it is clean + -- we can skip clean the threads. + -- this is update right before we query the database for the key result. + dataBaseLogger :: String -> IO (), + + databaseQueue :: DBQue, + + databaseRules :: TheRules, + databaseStep :: !(TVar Step), + + databaseValuesLock :: !(TVar Bool), + -- when we restart a build, we set this to False to block any other + -- threads from reading databaseValues + databaseValues :: !(Map Key KeyDetails) + } +--------------------------------------------------------------------- +-- | Remove finished asyncs from 'databaseThreads' (non-blocking). +-- Uses 'poll' to check completion without waiting. +pruneFinished :: Database -> IO () +pruneFinished db@Database{..} = do + threads <- readTVarIO databaseThreads + statuses <- forM threads $ \(d,a) -> do + p <- poll a + return (d,a,p) + let still = [ (d,a) | (d,a,p) <- statuses, isNothing p ] + -- deleteDatabaseRuntimeDep of finished async keys + forM_ statuses $ \(d,_,p) -> when (isJust p) $ do + let k = deliverKey d + when (k /= newKey "root") $ atomically $ deleteDatabaseRuntimeDep k db + atomically $ modifyTVar' databaseThreads (const still) + +deleteDatabaseRuntimeDep :: Key -> Database -> STM () +deleteDatabaseRuntimeDep k db = do + SMap.delete k (databaseRuntimeDep db) + +computeReverseRuntimeMap :: Database -> STM (Map.HashMap Key KeySet) +computeReverseRuntimeMap db = do + -- Create a fresh snapshot (pure Data.Map) of the current runtime reverse deps. + pairs <- ListT.toList $ SMap.listT (databaseRuntimeDep db) + -- 'pairs' is a map from parent -> set of children (dependencies recorded at runtime). + -- We need to invert this to child -> set of parents (reverse dependencies). + let addParent acc (parent, children) = + foldr (\child m -> Map.insertWith (\new old -> unionKyeSet new old) child (singletonKeySet parent) m) acc (toListKeySet children) + m = foldl addParent Map.empty pairs + return m + +-- compute to preserve asyncs +-- only the running stage 2 keys are actually running +-- so we only need to preserve them if they are not affected by the dirty set + +-- to acompany with this, +-- all non-dirty running need to have an updated step, +-- so it won't be view as dirty when we restart the build +-- computeToPreserve :: Database -> KeySet -> STM [(Key, Async ())] +computeToPreserve :: Database -> KeySet -> STM ([(Key, Async ())], [Key]) +computeToPreserve db dirtySet = do + -- All keys that depend (directly or transitively) on any dirty key + affected <- computeTransitiveReverseDeps db dirtySet + allRunings <- ListT.toList $ SMap.listT (databaseValues db) + let allRuningkeys = map fst allRunings + let running2UnAffected = [ (k, async) | (k, v) <- allRunings, not (k `memberKeySet` affected), Running _ _ _ (RunningStage2 async) <- [keyStatus v] ] + forM_ allRuningkeys $ \k -> do + -- if not dirty, bump its step + unless (memberKeySet k affected) $ do + SMap.focus + ( Focus.adjust $ \case + kd@KeyDetails {keyStatus = Running {runningStep, runningPrev, runningWait, runningStage}} -> + (kd {keyStatus = Running (runningStep + 1) runningPrev runningWait runningStage}) + kd -> kd + ) + k + (databaseValues db) + -- Keep only those whose key is NOT affected by the dirty set + pure ([kv | kv@(k, _async) <- running2UnAffected, not (memberKeySet k affected)], allRuningkeys) + +-- compute the transitive reverse dependencies of a set of keys +-- using databaseRuntimeDep in the Database +-- compute the transitive reverse dependencies of a set of keys +-- using databaseRuntimeDep in the Database +computeTransitiveReverseDeps :: Database -> KeySet -> STM KeySet +computeTransitiveReverseDeps db seeds = do + rev <- computeReverseRuntimeMap db + let -- BFS worklist starting from all seed keys. + -- visited contains everything we've already enqueued (including seeds). + go :: KeySet -> [Key] -> STM KeySet + go visited [] = pure visited + go visited (k:todo) = do + let mDeps = Map.lookup k rev + case mDeps of + Nothing -> go visited todo + Just direct -> + -- new keys = direct dependents not seen before + let newKs = filter (\x -> not (memberKeySet x visited)) (toListKeySet direct) + visited' = foldr insertKeySet visited newKs + in go visited' (newKs ++ todo) + + -- Start with seeds already marked visited to prevent self-revisit. + go seeds (toListKeySet seeds) + + +insertdatabaseRuntimeDep :: Key -> Key -> Database -> STM () +insertdatabaseRuntimeDep k pk db = do + SMap.focus (Focus.alter (Just . maybe (singletonKeySet k) (insertKeySet k))) pk (databaseRuntimeDep db) + +--------------------------------------------------------------------- -waitForDatabaseRunningKeys :: Database -> IO () -waitForDatabaseRunningKeys = getDatabaseValues >=> mapM_ (waitRunning . snd) +shakeDataBaseQueue :: ShakeDatabase -> DBQue +shakeDataBaseQueue = databaseQueue . (\(ShakeDatabase _ _ db) -> db) + +awaitRunInDb :: Database -> IO result -> IO result +awaitRunInDb db act = awaitRunInThread (databaseQueue db) act + + +databaseGetActionQueueLength :: Database -> STM Int +databaseGetActionQueueLength db = do + counTaskQueue (databaseQueue db) + +runInDataBase :: String -> Database -> [(IO result, Either SomeException result -> IO ())] -> STM () +runInDataBase title db acts = do + s <- getDataBaseStepInt db + let actWithEmptyHook = map (\(x, y) -> (const $ return (), x, y)) acts + runInThreadStmInNewThreads db (return $ DeliverStatus s title (newKey "root")) actWithEmptyHook + +runInThreadStmInNewThreads :: Database -> IO DeliverStatus -> [(Async () -> IO (), IO result, Either SomeException result -> IO ())] -> STM () +runInThreadStmInNewThreads db mkDeliver acts = do + -- Take an action from TQueue, run it and + -- use barrier to wait for the result + let log prefix title = dataBaseLogger db (prefix ++ title) + writeTaskQueue (databaseQueue db) $ Right $ do + uninterruptibleMask $ \restore -> do + do + deliver <- mkDeliver + log "runInThreadStmInNewThreads submit begin " (deliverName deliver) + curStep <- atomically $ getDataBaseStepInt db + when (curStep == deliverStep deliver) $ do + syncs <- mapM (\(preHook, act, handler) -> do + a <- async (handler =<< (restore $ Right <$> act) `catch` \e@(SomeException _) -> return (Left e)) + preHook a + return (deliver, a) + ) acts + atomically $ modifyTVar' (databaseThreads db) (syncs++) + log "runInThreadStmInNewThreads submit end " (deliverName deliver) + +runOneInDataBase :: IO DeliverStatus -> Database -> (Async () -> IO ()) -> IO result -> (SomeException -> IO ()) -> STM () +runOneInDataBase mkDelivery db registerAsync act handler = do + runInThreadStmInNewThreads + db + mkDelivery + [ ( registerAsync, act, + \case + Left e -> handler e + Right _ -> return () + ) + ] + + +getDataBaseStepInt :: Database -> STM Int +getDataBaseStepInt db = do + Step s <- readTVar $ databaseStep db + return s + +data AsyncParentKill = AsyncParentKill ThreadId Step + deriving (Show, Eq) + +instance Exception AsyncParentKill where + toException = asyncExceptionToException + fromException = asyncExceptionFromException + +shutDatabase ::Set (Async ()) -> Database -> IO () +shutDatabase preserve db@Database{..} = uninterruptibleMask $ \unmask -> do + -- wait for all threads to finish + asyncs <- readTVarIO databaseThreads + step <- readTVarIO databaseStep + tid <- myThreadId + -- traceEventIO ("shutDatabase: cancelling " ++ show (length asyncs) ++ " asyncs, step " ++ show step) + -- traceEventIO ("shutDatabase: async entries: " ++ show (map (deliverName . fst) asyncs)) + let remains = filter (\(_, s) -> s `S.member` preserve) asyncs + let toCancel = filter (\(_, s) -> s `S.notMember` preserve) asyncs + -- traceEventIO ("shutDatabase: remains count: " ++ show (length remains) ++ ", names: " ++ show (map (deliverName . fst) remains)) + -- traceEventIO ("shutDatabase: toCancel count: " ++ show (length toCancel) ++ ", names: " ++ show (map (deliverName . fst) toCancel)) + mapM_ (\(_, a) -> throwTo (asyncThreadId a) $ AsyncParentKill tid step) toCancel + atomically $ modifyTVar' databaseThreads (const remains) + -- Wait until all the asyncs are done + -- But if it takes more than 10 seconds, log to stderr + unless (null asyncs) $ do + let warnIfTakingTooLong = unmask $ forever $ do + sleep 5 + as <- readTVarIO databaseThreads + -- poll each async: Nothing => still running + statuses <- forM as $ \(d,a) -> do + p <- poll a + return (d, a, p) + let still = [ (deliverName d, show (asyncThreadId a)) | (d,a,p) <- statuses, isNothing p ] + traceEventIO $ "cleanupAsync: waiting for asyncs to finish; total=" ++ show (length as) ++ ", stillRunning=" ++ show (length still) + traceEventIO $ "cleanupAsync: still running (deliverName, threadId) = " ++ show still + withAsync warnIfTakingTooLong $ \_ -> + mapM_ waitCatch $ map snd toCancel + pruneFinished db + +peekAsyncsDelivers :: Database -> IO [DeliverStatus] +peekAsyncsDelivers db = do + asyncs <- readTVarIO (databaseThreads db) + return (map fst asyncs) +-- waitForDatabaseRunningKeys :: Database -> IO () +-- waitForDatabaseRunningKeys = getDatabaseValues >=> mapM_ (waitRunning . snd) getDatabaseValues :: Database -> IO [(Key, Status)] getDatabaseValues = atomically @@ -125,28 +384,34 @@ getDatabaseValues = atomically . SMap.listT . databaseValues +data RunningStage = RunningStage1 | RunningStage2 (Async ()) + deriving (Eq, Ord) data Status = Clean !Result | Dirty (Maybe Result) + | Exception !Step !SomeException !(Maybe Result) | Running { - runningStep :: !Step, - runningWait :: !(IO ()), - runningResult :: Result, -- LAZY - runningPrev :: !(Maybe Result) + runningStep :: !Step, + -- runningResult :: Result, -- LAZY + runningPrev :: !(Maybe Result), + runningWait :: !(IO (Key, Result)), + runningStage :: !RunningStage } viewDirty :: Step -> Status -> Status -viewDirty currentStep (Running s _ _ re) | currentStep /= s = Dirty re +viewDirty currentStep (Running s re _ _) | currentStep /= s = Dirty re +viewDirty currentStep (Exception s _ re) | currentStep /= s = Dirty re viewDirty _ other = other getResult :: Status -> Maybe Result getResult (Clean re) = Just re getResult (Dirty m_re) = m_re -getResult (Running _ _ _ m_re) = m_re -- watch out: this returns the previous result +getResult (Running _ m_re _ _) = m_re -- watch out: this returns the previous result +getResult (Exception _ _ m_re) = m_re -waitRunning :: Status -> IO () -waitRunning Running{..} = runningWait -waitRunning _ = return () +-- waitRunning :: Status -> IO () +-- waitRunning Running{..} = runningWait +-- waitRunning _ = return () data Result = Result { resultValue :: !Value, diff --git a/hls-graph/src/Development/IDE/WorkerThread.hs b/hls-graph/src/Development/IDE/WorkerThread.hs new file mode 100644 index 0000000000..39783b220a --- /dev/null +++ b/hls-graph/src/Development/IDE/WorkerThread.hs @@ -0,0 +1,194 @@ +{- +Module : Development.IDE.WorkerThread +Author : @soulomoon +SPDX-License-Identifier: Apache-2.0 + +Description : This module provides an API for managing worker threads in the IDE. +see Note [Serializing runs in separate thread] +-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeFamilies #-} + +module Development.IDE.WorkerThread + ( LogWorkerThread (..), + DeliverStatus(..), + withWorkerQueue, + TaskQueue(..), + writeTaskQueue, + withWorkerQueueSimple, + isEmptyTaskQueue, + counTaskQueue, + submitWork, + eitherWorker, + Worker, + tryReadTaskQueue, + withWorkerQueueSimpleRight, + submitWorkAtHead, + awaitRunInThread, + withAsyncs + ) where + +import Control.Concurrent.Async (withAsync) +import Control.Concurrent.STM +import Control.Exception.Safe (SomeException, finally, + throw, try) +import Control.Monad.Cont (ContT (ContT)) +import qualified Data.Text as T + +import Control.Concurrent +import Data.Dynamic (Dynamic) +import Development.IDE.Graph.Internal.Key (Key) +import Prettyprinter + +data LogWorkerThread + = LogThreadEnding !T.Text + | LogThreadEnded !T.Text + | LogSingleWorkStarting !T.Text + | LogSingleWorkEnded !T.Text + | LogMainThreadId !T.Text !ThreadId + deriving (Show) + +instance Pretty LogWorkerThread where + pretty = \case + LogThreadEnding t -> "Worker thread ending:" <+> pretty t + LogThreadEnded t -> "Worker thread ended:" <+> pretty t + LogSingleWorkStarting t -> "Worker starting a unit of work: " <+> pretty t + LogSingleWorkEnded t -> "Worker ended a unit of work: " <+> pretty t + LogMainThreadId t tid -> "Main thread for" <+> pretty t <+> "is" <+> pretty (show tid) + + +{- +Note [Serializing runs in separate thread] +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +We often want to take long-running actions using some resource that cannot be shared. +In this instance it is useful to have a queue of jobs to run using the resource. +Like the db writes, session loading in session loader, shake session restarts. + +Originally we used various ways to implement this, but it was hard to maintain and error prone. +Moreover, we can not stop these threads uniformly when we are shutting down the server. +-} +data TaskQueue a = TaskQueue (TQueue a) +newTaskQueueIO :: IO (TaskQueue a) +newTaskQueueIO = TaskQueue <$> newTQueueIO +data ExitOrTask t = Exit | Task t +type Logger = LogWorkerThread -> IO () + +-- | 'withWorkerQueue' creates a new 'TQueue', and launches a worker +-- thread which polls the queue for requests and runs the given worker +-- function on them. +withWorkerQueueSimple :: Logger -> T.Text -> ContT () IO (TaskQueue (IO ())) +withWorkerQueueSimple log title = withWorkerQueue log title id + +withWorkerQueueSimpleRight :: Logger -> T.Text -> ContT () IO (TaskQueue (Either Dynamic (IO ()))) +withWorkerQueueSimpleRight log title = withWorkerQueue log title $ eitherWorker (const $ return ()) id + + +withWorkerQueue :: Logger -> T.Text -> (t -> IO ()) -> ContT () IO (TaskQueue t) +withWorkerQueue = withWorkersQueue 1 +withWorkersQueue :: Int -> Logger -> T.Text -> (t -> IO ()) -> ContT () IO (TaskQueue t) +withWorkersQueue n log title workerAction = ContT $ \mainAction -> do + tid <- myThreadId + log (LogMainThreadId title tid) + q <- newTaskQueueIO + -- Use a TMVar as a stop flag to coordinate graceful shutdown. + -- The worker thread checks this flag before dequeuing each job; if set, it exits immediately, + -- ensuring that no new work is started after shutdown is requested. + -- This mechanism is necessary because some downstream code may swallow async exceptions, + -- making 'cancel' unreliable for stopping the thread in all cases. + -- If 'cancel' does interrupt the thread (e.g., while blocked in STM or in a cooperative job), + -- the thread exits immediately and never checks the TMVar; in such cases, the stop flag is redundant. + b <- newEmptyTMVarIO + withAsyncs (replicate n (writerThread q b)) $ do + mainAction q + -- if we want to debug the exact location the worker swallows an async exception, we can + -- temporarily comment out the `finally` clause. + `finally` atomically (putTMVar b ()) + log (LogThreadEnding title) + log (LogThreadEnded title) + where + -- writerThread :: TaskQueue t -> TMVar () -> (forall a. IO a -> IO a) -> IO () + writerThread q b = + -- See above: check stop flag before dequeuing, exit if set, otherwise run next job. + do + task <- atomically $ do + task <- tryReadTaskQueue q + isEm <- isEmptyTMVar b + case (isEm, task) of + (False, _) -> return Exit -- stop flag set, exit + (_, Just t) -> return $ Task t -- got a task, run it + (_, Nothing) -> retry -- no task, wait + case task of + Exit -> return () + Task t -> do + log $ LogSingleWorkStarting title + workerAction t + log $ LogSingleWorkEnded title + writerThread q b + +withAsyncs :: [IO ()] -> IO () -> IO () +withAsyncs ios mainAction = go ios + where + go [] = mainAction + go (x:xs) = withAsync x $ \_ -> go xs + +-- | 'awaitRunInThread' queues up an 'IO' action to be run by a worker thread, +-- and then blocks until the result is computed. If the action throws an +-- non-async exception, it is rethrown in the calling thread. + +data DeliverStatus = DeliverStatus + { deliverStep :: Int + , deliverName :: String + , deliverKey :: Key + } deriving (Show) + +instance Pretty DeliverStatus where + pretty (DeliverStatus step _name key) = + "Step:" <+> pretty step <> "," <+> "Key:" <+> pretty (show key) + + +type Worker arg = arg -> IO () + +eitherWorker :: Worker a -> Worker b -> Worker (Either a b) +eitherWorker w1 w2 = \case + Left a -> w1 a + Right b -> w2 b + +awaitRunInThread :: TaskQueue (Either Dynamic (IO ())) -> IO result -> IO result +awaitRunInThread (TaskQueue q) act = do + barrier <- newEmptyTMVarIO + -- Take an action from TQueue, run it and + -- use barrier to wait for the result + atomically $ writeTQueue q (Right $ try act >>= atomically . putTMVar barrier) + resultOrException <- atomically $ takeTMVar barrier + case resultOrException of + Left e -> throw (e :: SomeException) + Right r -> return r + + +-- submitWork without waiting for the result +submitWork :: TaskQueue arg -> arg -> IO () +submitWork (TaskQueue q) arg = atomically $ writeTQueue q arg + +-- submit work at the head of the queue, so it will be executed next +submitWorkAtHead :: TaskQueue arg -> arg -> IO () +submitWorkAtHead (TaskQueue q) arg = do + atomically $ unGetTQueue q arg + +writeTaskQueue :: TaskQueue a -> a -> STM () +writeTaskQueue (TaskQueue q) = writeTQueue q + +tryReadTaskQueue :: TaskQueue a -> STM (Maybe a) +tryReadTaskQueue (TaskQueue q) = tryReadTQueue q + +isEmptyTaskQueue :: TaskQueue a -> STM Bool +isEmptyTaskQueue (TaskQueue q) = isEmptyTQueue q + +-- look and count the number of items in the queue +-- do not remove them +counTaskQueue :: TaskQueue a -> STM Int +counTaskQueue (TaskQueue q) = do + xs <- flushTQueue q + mapM_ (unGetTQueue q) (reverse xs) + return $ length xs + diff --git a/hls-graph/test/ActionSpec.hs b/hls-graph/test/ActionSpec.hs index 97ab5555ac..865dcfb36f 100644 --- a/hls-graph/test/ActionSpec.hs +++ b/hls-graph/test/ActionSpec.hs @@ -1,5 +1,6 @@ -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ImpredicativeTypes #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} module ActionSpec where @@ -7,7 +8,12 @@ import Control.Concurrent (MVar, readMVar) import qualified Control.Concurrent as C import Control.Concurrent.STM import Control.Monad.IO.Class (MonadIO (..)) -import Development.IDE.Graph (shakeOptions) +import Control.Monad.Trans.Cont (evalContT) +import Data.Typeable (Typeable) +import Development.IDE.Graph (RuleResult, + ShakeOptions, + shakeOptions) +import Development.IDE.Graph.Classes (Hashable) import Development.IDE.Graph.Database (shakeNewDatabase, shakeRunDatabase, shakeRunDatabaseForKeys) @@ -15,15 +21,33 @@ import Development.IDE.Graph.Internal.Database (build, incDatabase) import Development.IDE.Graph.Internal.Key import Development.IDE.Graph.Internal.Types import Development.IDE.Graph.Rule +import Development.IDE.WorkerThread import Example import qualified StmContainers.Map as STM import Test.Hspec +buildWithRoot :: forall f key value . (Traversable f, RuleResult key ~ value, Typeable key, Show key, Hashable key, Typeable value) => Database -> Stack -> f key -> IO (f Key, f value) +buildWithRoot = build (newKey ("root" :: [Char])) +shakeNewDatabaseWithLogger :: DBQue -> ShakeOptions -> Rules () -> IO ShakeDatabase +shakeNewDatabaseWithLogger = shakeNewDatabase (const $ return ()) + +itInThread :: String -> (DBQue -> IO ()) -> SpecWith () +itInThread name ex = it name $ evalContT $ do + -- thread <- withWorkerQueueSimpleRight (appendFile "hlg-graph-test.txt" . (++"\n") . show) "hls-graph test" + thread <- withWorkerQueueSimpleRight (const $ return ()) "hls-graph test" + liftIO $ ex thread + +shakeRunDatabaseFromRight :: ShakeDatabase -> [Action a] -> IO [a] +shakeRunDatabaseFromRight db as = do + res <- shakeRunDatabase db as + case sequence res of + Left e -> error $ "shakeRunDatabaseFromRight: unexpected exception: " ++ show e + Right v -> return v spec :: Spec spec = do - describe "apply1" $ it "Test build update, Buggy dirty mechanism in hls-graph #4237" $ do + describe "apply1" $ itInThread "Test build update, Buggy dirty mechanism in hls-graph #4237" $ \q -> do let ruleStep1 :: MVar Int -> Rules () ruleStep1 m = addRule $ \CountRule _old mode -> do -- depends on ruleSubBranch, it always changed if dirty @@ -39,11 +63,11 @@ spec = do return $ RunResult ChangedNothing "" r (return ()) count <- C.newMVar 0 count1 <- C.newMVar 0 - db <- shakeNewDatabase shakeOptions $ do + db <- shakeNewDatabaseWithLogger q shakeOptions $ do ruleSubBranch count ruleStep1 count1 -- bootstrapping the database - _ <- shakeRunDatabase db $ pure $ apply1 CountRule -- count = 1 + _ <- shakeRunDatabaseFromRight db $ pure $ apply1 CountRule -- count = 1 let child = newKey SubBranchRule let parent = newKey CountRule -- instruct to RunDependenciesChanged then CountRule should be recomputed @@ -58,46 +82,46 @@ spec = do _res3 <- shakeRunDatabaseForKeys (Just [parent]) db [apply1 CountRule] -- count = 2 c1 <- readMVar count1 c1 `shouldBe` 2 - describe "apply1" $ do - it "computes a rule with no dependencies" $ do - db <- shakeNewDatabase shakeOptions ruleUnit - res <- shakeRunDatabase db $ + describe "apply1" $ do + itInThread "computes a rule with no dependencies" $ \q -> do + db <- shakeNewDatabaseWithLogger q shakeOptions ruleUnit + res <- shakeRunDatabaseFromRight db $ pure $ apply1 (Rule @()) res `shouldBe` [()] - it "computes a rule with one dependency" $ do - db <- shakeNewDatabase shakeOptions $ do + itInThread "computes a rule with one dependency" $ \q -> do + db <- shakeNewDatabaseWithLogger q shakeOptions $ do ruleUnit ruleBool - res <- shakeRunDatabase db $ pure $ apply1 Rule + res <- shakeRunDatabaseFromRight db $ pure $ apply1 Rule res `shouldBe` [True] - it "tracks direct dependencies" $ do - db@(ShakeDatabase _ _ theDb) <- shakeNewDatabase shakeOptions $ do + itInThread "tracks direct dependencies" $ \q -> do + db@(ShakeDatabase _ _ theDb) <- shakeNewDatabaseWithLogger q shakeOptions $ do ruleUnit ruleBool let theKey = Rule @Bool - res <- shakeRunDatabase db $ + res <- shakeRunDatabaseFromRight db $ pure $ apply1 theKey res `shouldBe` [True] Just (Clean res) <- lookup (newKey theKey) <$> getDatabaseValues theDb resultDeps res `shouldBe` ResultDeps [singletonKeySet $ newKey (Rule @())] - it "tracks reverse dependencies" $ do - db@(ShakeDatabase _ _ Database {..}) <- shakeNewDatabase shakeOptions $ do + itInThread "tracks reverse dependencies" $ \q -> do + db@(ShakeDatabase _ _ Database {..}) <- shakeNewDatabaseWithLogger q shakeOptions $ do ruleUnit ruleBool let theKey = Rule @Bool - res <- shakeRunDatabase db $ + res <- shakeRunDatabaseFromRight db $ pure $ apply1 theKey res `shouldBe` [True] Just KeyDetails {..} <- atomically $ STM.lookup (newKey (Rule @())) databaseValues keyReverseDeps `shouldBe` singletonKeySet (newKey theKey) - it "rethrows exceptions" $ do - db <- shakeNewDatabase shakeOptions $ addRule $ \(Rule :: Rule ()) _old _mode -> error "boom" - let res = shakeRunDatabase db $ pure $ apply1 (Rule @()) + itInThread "rethrows exceptions" $ \q -> do + db <- shakeNewDatabaseWithLogger q shakeOptions $ addRule $ \(Rule :: Rule ()) _old _mode -> error "boom" + let res = shakeRunDatabaseFromRight db $ pure $ apply1 (Rule @()) res `shouldThrow` anyErrorCall - it "computes a rule with branching dependencies does not invoke phantom dependencies #3423" $ do + itInThread "computes a rule with branching dependencies does not invoke phantom dependencies #3423" $ \q -> do cond <- C.newMVar True count <- C.newMVar 0 - (ShakeDatabase _ _ theDb) <- shakeNewDatabase shakeOptions $ do + (ShakeDatabase _ _ theDb) <- shakeNewDatabaseWithLogger q shakeOptions $ do ruleUnit ruleCond cond ruleSubBranch count @@ -105,26 +129,26 @@ spec = do -- build the one with the condition True -- This should call the SubBranchRule once -- cond rule would return different results each time - res0 <- build theDb emptyStack [BranchedRule] + res0 <- buildWithRoot theDb emptyStack [BranchedRule] snd res0 `shouldBe` [1 :: Int] incDatabase theDb Nothing -- build the one with the condition False -- This should not call the SubBranchRule - res1 <- build theDb emptyStack [BranchedRule] + res1 <- buildWithRoot theDb emptyStack [BranchedRule] snd res1 `shouldBe` [2 :: Int] -- SubBranchRule should be recomputed once before this (when the condition was True) - countRes <- build theDb emptyStack [SubBranchRule] + countRes <- buildWithRoot theDb emptyStack [SubBranchRule] snd countRes `shouldBe` [1 :: Int] - describe "applyWithoutDependency" $ it "does not track dependencies" $ do - db@(ShakeDatabase _ _ theDb) <- shakeNewDatabase shakeOptions $ do + describe "applyWithoutDependency" $ itInThread "does not track dependencies" $ \q -> do + db@(ShakeDatabase _ _ theDb) <- shakeNewDatabaseWithLogger q shakeOptions $ do ruleUnit addRule $ \Rule _old _mode -> do [()] <- applyWithoutDependency [Rule] return $ RunResult ChangedRecomputeDiff "" True $ return () let theKey = Rule @Bool - res <- shakeRunDatabase db $ + res <- shakeRunDatabaseFromRight db $ pure $ applyWithoutDependency [theKey] res `shouldBe` [[True]] Just (Clean res) <- lookup (newKey theKey) <$> getDatabaseValues theDb diff --git a/hls-graph/test/DatabaseSpec.hs b/hls-graph/test/DatabaseSpec.hs index 9061bfa89d..0d81310dfc 100644 --- a/hls-graph/test/DatabaseSpec.hs +++ b/hls-graph/test/DatabaseSpec.hs @@ -2,7 +2,10 @@ module DatabaseSpec where -import Development.IDE.Graph (newKey, shakeOptions) +import ActionSpec (itInThread) +import Control.Exception (SomeException, throw) +import Development.IDE.Graph (ShakeOptions, newKey, + shakeOptions) import Development.IDE.Graph.Database (shakeNewDatabase, shakeRunDatabase) import Development.IDE.Graph.Internal.Action (apply1) @@ -14,23 +17,34 @@ import System.Time.Extra (timeout) import Test.Hspec +exractException :: [Either SomeException ()] -> Maybe StackException +exractException [] = Nothing +exractException (Left e : _) | Just ne@StackException{} <- fromGraphException e = return ne +exractException (_: xs) = exractException xs + +shakeNewDatabaseWithLogger :: DBQue -> ShakeOptions -> Rules () -> IO ShakeDatabase +shakeNewDatabaseWithLogger = shakeNewDatabase (const $ return ()) + spec :: Spec spec = do describe "Evaluation" $ do - it "detects cycles" $ do - db <- shakeNewDatabase shakeOptions $ do + itInThread "detects cycles" $ \q -> do + db <- shakeNewDatabaseWithLogger q shakeOptions $ do ruleBool addRule $ \Rule _old _mode -> do True <- apply1 (Rule @Bool) return $ RunResult ChangedRecomputeDiff "" () (return ()) - let res = shakeRunDatabase db $ pure $ apply1 (Rule @()) - timeout 1 res `shouldThrow` \StackException{} -> True + res <- timeout 1 $ shakeRunDatabase db $ pure $ apply1 (Rule @()) + let x = exractException =<< res + let throwStack x = case x + of Just e -> throw e + Nothing -> error "Expected a StackException, got none" + throwStack x `shouldThrow` \StackException{} -> True describe "compute" $ do - it "build step and changed step updated correctly" $ do - (ShakeDatabase _ _ theDb) <- shakeNewDatabase shakeOptions $ do + itInThread "build step and changed step updated correctly" $ \q -> do + (ShakeDatabase _ _ theDb) <- shakeNewDatabaseWithLogger q shakeOptions $ do ruleStep - let k = newKey $ Rule @() -- ChangedRecomputeSame r1@Result{resultChanged=rc1, resultBuilt=rb1} <- compute theDb emptyStack k RunDependenciesChanged Nothing diff --git a/hls-test-utils/src/Development/IDE/Test.hs b/hls-test-utils/src/Development/IDE/Test.hs index a1bd2dec0e..70390ad118 100644 --- a/hls-test-utils/src/Development/IDE/Test.hs +++ b/hls-test-utils/src/Development/IDE/Test.hs @@ -25,7 +25,6 @@ module Development.IDE.Test , flushMessages , waitForAction , getInterfaceFilesDir - , garbageCollectDirtyKeys , getFilesOfInterest , waitForTypecheck , waitForBuildQueue @@ -218,8 +217,8 @@ waitForAction key TextDocumentIdentifier{_uri} = getInterfaceFilesDir :: TextDocumentIdentifier -> Session FilePath getInterfaceFilesDir TextDocumentIdentifier{_uri} = callTestPlugin (GetInterfaceFilesDir _uri) -garbageCollectDirtyKeys :: CheckParents -> Int -> Session [String] -garbageCollectDirtyKeys parents age = callTestPlugin (GarbageCollectDirtyKeys parents age) +-- garbageCollectDirtyKeys :: CheckParents -> Int -> Session [String] +-- garbageCollectDirtyKeys parents age = callTestPlugin (GarbageCollectDirtyKeys parents age) getStoredKeys :: Session [Text] getStoredKeys = callTestPlugin GetStoredKeys diff --git a/hls-test-utils/src/Test/Hls.hs b/hls-test-utils/src/Test/Hls.hs index 0ab203fe36..3ac4413860 100644 --- a/hls-test-utils/src/Test/Hls.hs +++ b/hls-test-utils/src/Test/Hls.hs @@ -770,7 +770,10 @@ runSessionWithTestConfig TestConfig{..} session = let plugins = testPluginDescriptor recorder <> lspRecorderPlugin timeoutOverride <- fmap read <$> lookupEnv "LSP_TIMEOUT" - let sconf' = testConfigSession { lspConfig = hlsConfigToClientConfig testLspConfig, messageTimeout = fromMaybe (messageTimeout defaultConfig) timeoutOverride} + let sconf' = testConfigSession { lspConfig = hlsConfigToClientConfig testLspConfig + , messageTimeout = fromMaybe (messageTimeout defaultConfig) timeoutOverride + , logStdErr = True + } arguments = testingArgs serverRoot recorderIde plugins server <- async $ IDEMain.defaultMain (cmapWithPrio LogIDEMain recorderIde) diff --git a/log copy.txt b/log copy.txt new file mode 100644 index 0000000000..5da3744ff0 --- /dev/null +++ b/log copy.txt @@ -0,0 +1,139 @@ +Run #3 +ThreadId 6 ghcide + diagnostics +| 2025-08-1 Cancellation + edit header + GetHieAst: 9T14:55:44.590216Z | Info | Test Project located in directory: /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736 +ThreadId 7 | 2025-08-19T14:55:44.591607Z | Info | Heap statistics are not enabled (RTS option -T is needed) +ThreadId 7 | 2025-08-19T14:55:44.594438Z | Info | Starting LSP server... + If you are seeing this in a terminal, you probably should have run WITHOUT the --lsp option! + PluginIds: [ test + , block-command + , ghcide-completions + , core + , ghcide-type-lenses + , LSPRecorderCallback + , ghcide-hover-and-symbols + , ghcide-core ] +ThreadId 16 | 2025-08-19T14:55:44.594799Z | Info | Starting server +ThreadId 16 | 2025-08-19T14:55:44.595197Z | Debug | LSP: set new config: { + "cabalFormattingProvider": "cabal-gild", + "checkParents": "CheckOnSave", + "checkProject": true, + "formattingProvider": "ormolu", + "maxCompletions": 40, + "plugin": { }, + "sessionLoading": "singleComponent" +} +ThreadId 16 | 2025-08-19T14:55:44.595437Z | Info | Registering IDE configuration: IdeConfiguration {workspaceFolders = fromList [NormalizedUri (-2250868254854792059) "file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736"], clientSettings = hashed (Just (Object (fromList [("haskell",Object (fromList [("cabalFormattingProvider",String "cabal-gild"),("checkParents",String "CheckOnSave"),("checkProject",Bool True),("formattingProvider",String "ormolu"),("maxCompletions",Number 40.0),("plugin",Object (fromList [])),("sessionLoading",String "singleComponent")]))])))} +ThreadId 16 | 2025-08-19T14:55:44.603799Z | Info | Started LSP server in 0.01s +ThreadId 16 | 2025-08-19T14:55:44.603902Z | Debug | shouldRunSubset: True +ThreadId 24 | 2025-08-19T14:55:44.603975Z | Debug | Initializing exports map from hiedb +ThreadId 24 | 2025-08-19T14:55:44.604080Z | Debug | Done initializing exports map from hiedb. Size: 0 +ThreadId 20 | 2025-08-19T14:55:44.604735Z | Debug | Shake session initialized +ThreadId 16 | 2025-08-19T14:55:44.605403Z | Debug | VFS: opening file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 16 | 2025-08-19T14:55:44.605775Z | Debug | LSP: set new config: { + "cabalFormattingProvider": "cabal-gild", + "checkParents": "CheckOnSave", + "checkProject": true, + "formattingProvider": "ormolu", + "maxCompletions": 40, + "plugin": { }, + "sessionLoading": "singleComponent" +} +ThreadId 26 | 2025-08-19T14:55:44.605883Z | Debug | Finished build session +AsyncCancelled +ThreadId 21 | 2025-08-19T14:55:44.605934Z | Debug | Configuration changed: Config {checkParents = CheckOnSave, checkProject = True, formattingProvider = "ormolu", cabalFormattingProvider = "cabal-gild", maxCompletions = 40, sessionLoading = PreferSingleComponentLoading, plugins = fromList []} +ThreadId 21 | 2025-08-19T14:55:44.606008Z | Debug | Restarting build session due to config change +Action Queue: [] +Keys: [GetClientSettings; ] +Aborting previous build session took 0.00s +ThreadId 30 | 2025-08-19T14:55:44.606131Z | Debug | Finished build session +AsyncCancelled +ThreadId 21 | 2025-08-19T14:55:44.606163Z | Debug | Set files of interst to + [ ( /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs + , Modified {firstOpen = True} ) ] +ThreadId 21 | 2025-08-19T14:55:44.606229Z | Debug | Restarting build session due to /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs (modified) +Action Queue: [] +Keys: [ GetClientSettings; + , GetModificationTime; /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs + , IsFileOfInterest; /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs ] +Aborting previous build session took 0.00s +ThreadId 20 | 2025-08-19T14:55:44.606351Z | Debug | Opened text document: file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 39 | 2025-08-19T14:55:44.606579Z | Warning | Typechecking file start /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 42 | 2025-08-19T14:55:44.606750Z | Debug | Looking up session cache for /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 22 | 2025-08-19T14:55:44.606794Z | Debug | Loop: getOptions for /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 22 | 2025-08-19T14:55:44.606904Z | Info | Cradle path: Foo.hs +ThreadId 22 | 2025-08-19T14:55:44.606952Z | Warning | No [cradle](https://github.com/mpickering/hie-bios#hie-bios) found for Foo.hs. +Proceeding with [implicit cradle](https://hackage.haskell.org/package/implicit-hie). +You should ignore this message, unless you see a 'Multi Cradle: No prefixes matched' error. +ThreadId 46 | 2025-08-19T14:55:44.620269Z | Debug | Cradle: Cradle{ cradleRootDir = "/private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736", cradleOptsProg = CradleAction: Default} +ThreadId 46 | 2025-08-19T14:55:44.620334Z | Info | invoking build tool to determine build flags (this may take some time depending on the cache) +ThreadId 46 | 2025-08-19T14:55:44.683118Z | Debug | ghc --print-libdir +ThreadId 46 | 2025-08-19T14:55:44.746399Z | Debug | ghc --numeric-version +ThreadId 22 | 2025-08-19T14:55:44.746594Z | Debug | Session loading result: Right (ComponentOptions {componentOptions = [], componentRoot = "/private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736", componentDependencies = []},"/Users/ares/.ghcup/ghc/9.12.2/lib/ghc-9.12.2/lib","9.12.2") +ThreadId 22 | 2025-08-19T14:55:44.751250Z | Info | Interface files cache directory: /var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/.cache/ghcide/main-da39a3ee5e6b4b0d3255bfef95601890afd80709-da39a3ee5e6b4b0d3255bfef95601890afd80709 +ThreadId 22 | 2025-08-19T14:55:44.761208Z | Info | Making new HscEnv. In-place unit ids: [ main-da39a3ee5e6b4b0d3255bfef95601890afd80709 ] +ThreadId 22 | 2025-08-19T14:55:44.766821Z | Debug | New component cache HscEnvEq: (([],Just HscEnvEq 5),fromList []) +ThreadId 22 | 2025-08-19T14:55:44.767014Z | Info | New loaded files: [ /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs ] +ThreadId 42 | 2025-08-19T14:55:44.767161Z | Debug | Looking up session cache for /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 42 | 2025-08-19T14:55:44.767193Z | Info | Add dependency /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +[] +ThreadId 71 | 2025-08-19T14:55:44.767277Z | Info | Modification time for "v1" +ThreadId 71 | 2025-08-19T14:55:44.767314Z | Info | Modification time for "v1.1" +ThreadId 33 | 2025-08-19T14:55:44.767455Z | Debug | Finished build session +AsyncCancelled +ThreadId 21 | 2025-08-19T14:55:44.767514Z | Debug | Known files updated: + fromList [(TargetFile NormalizedFilePath "/private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs",fromList ["/private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs"])] +ThreadId 21 | 2025-08-19T14:55:44.769748Z | Debug | Restarting build session due to new component +Action Queue: [ WaitForIdeRule TypeCheck Uri {getUri = "file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs"} ] +Keys: [ IsFileOfInterest; /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs + , GhcSessionIO; + , GetKnownTargets; ] +Aborting previous build session took 0.00s +ThreadId 83 | 2025-08-19T14:55:44.769932Z | Debug | Finished: InitialLoad Took: 0.00s +ThreadId 84 | 2025-08-19T14:55:44.769935Z | Warning | Typechecking file start /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 87 | 2025-08-19T14:55:44.770101Z | Debug | Looking up session cache for /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 87 | 2025-08-19T14:55:44.770141Z | Info | Add dependency /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +[] +ThreadId 84 | 2025-08-19T14:55:44.779362Z | Warning | Typechecking file mid /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 84 | 2025-08-19T14:55:44.787260Z | Warning | Typechecking file end /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 81 | 2025-08-19T14:55:44.788775Z | Debug | Finished: WaitForIdeRule TypeCheck Uri {getUri = "file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs"} Took: 0.02s +ThreadId 16 | 2025-08-19T14:55:44.990428Z | Error | LSP: no handler for: "non-existent-method" +ThreadId 78 | 2025-08-19T14:55:44.992303Z | Debug | Finished build session +AsyncCancelled +ThreadId 21 | 2025-08-19T14:55:44.992398Z | Debug | Set files of interst to + [ ( /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs + , Modified {firstOpen = False} ) ] +ThreadId 21 | 2025-08-19T14:55:44.992559Z | Debug | Restarting build session due to /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs (modified) +Action Queue: [] +Keys: [ GetModificationTime; /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs + , IsFileOfInterest; /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs ] +Aborting previous build session took 0.00s +ThreadId 20 | 2025-08-19T14:55:44.992780Z | Debug | Modified text document: file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 132 | 2025-08-19T14:55:44.993293Z | Info | Modification time for "v1" +ThreadId 132 | 2025-08-19T14:55:44.993379Z | Info | Modification time for "v1.1" +ThreadId 128 | 2025-08-19T14:55:44.994761Z | Warning | Typechecking file start /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 126 | 2025-08-19T14:55:44.995047Z | Debug | Finished: WaitForIdeRule GetHieAst Uri {getUri = "file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs"} Took: 0.00s +ThreadId 121 | 2025-08-19T14:55:44.996016Z | Debug | Finished build session +AsyncCancelled +ThreadId 21 | 2025-08-19T14:55:44.996055Z | Debug | Restarting build session due to /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs (modified) +Action Queue: [] +Keys: [ GetModificationTime; /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs ] +Aborting previous build session took 0.00s +ThreadId 20 | 2025-08-19T14:55:44.996292Z | Debug | Modified text document: file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 153 | 2025-08-19T14:55:45.005864Z | Info | Modification time for "v1" +ThreadId 153 | 2025-08-19T14:55:45.005981Z | Info | Modification time for "v1.1" +ThreadId 149 | 2025-08-19T14:55:45.007173Z | Warning | Typechecking file start /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 149 | 2025-08-19T14:55:45.007522Z | Warning | Typechecking file mid /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 149 | 2025-08-19T14:55:45.008236Z | Warning | Typechecking file end /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs +ThreadId 148 | 2025-08-19T14:55:45.008442Z | Debug | Finished: WaitForIdeRule TypeCheck Uri {getUri = "file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-26398544736/Foo.hs"} Took: 0.01s +ThreadId 16 | 2025-08-19T14:55:45.211497Z | Error | LSP: no handler for: "non-existent-method" +ThreadId 16 | 2025-08-19T14:55:45.717804Z | Info | LSP: received shutdown +ThreadId 16 | 2025-08-19T14:55:45.717897Z | Debug | Received shutdown message +ThreadId 143 | 2025-08-19T14:55:45.717964Z | Debug | Finished build session +AsyncCancelled +ThreadId 6 | 2025-08-19T14:55:45.718622Z | Debug | Cleaned up temporary directory + GetHieAst: OK (1.13s) + +All 1 tests passed (1.13s) diff --git a/log.txt b/log.txt new file mode 100644 index 0000000000..86afac3e96 --- /dev/null +++ b/log.txt @@ -0,0 +1,111 @@ +Run #4 +Thghcide + diagnostics + Cancellation + edit header +readId 6 | GetHieAst: 2025-08-19T14:55:45.773048Z | Info | Test Project located in directory: /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736 +ThreadId 7 | 2025-08-19T14:55:45.774261Z | Info | Heap statistics are not enabled (RTS option -T is needed) +ThreadId 7 | 2025-08-19T14:55:45.776775Z | Info | Starting LSP server... + If you are seeing this in a terminal, you probably should have run WITHOUT the --lsp option! + PluginIds: [ test + , block-command + , ghcide-completions + , core + , ghcide-type-lenses + , LSPRecorderCallback + , ghcide-hover-and-symbols + , ghcide-core ] +ThreadId 16 | 2025-08-19T14:55:45.777036Z | Info | Starting server +ThreadId 16 | 2025-08-19T14:55:45.777814Z | Debug | LSP: set new config: { + "cabalFormattingProvider": "cabal-gild", + "checkParents": "CheckOnSave", + "checkProject": true, + "formattingProvider": "ormolu", + "maxCompletions": 40, + "plugin": { }, + "sessionLoading": "singleComponent" +} +ThreadId 16 | 2025-08-19T14:55:45.778159Z | Info | Registering IDE configuration: IdeConfiguration {workspaceFolders = fromList [NormalizedUri (-4077115142264691803) "file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736"], clientSettings = hashed (Just (Object (fromList [("haskell",Object (fromList [("cabalFormattingProvider",String "cabal-gild"),("checkParents",String "CheckOnSave"),("checkProject",Bool True),("formattingProvider",String "ormolu"),("maxCompletions",Number 40.0),("plugin",Object (fromList [])),("sessionLoading",String "singleComponent")]))])))} +ThreadId 16 | 2025-08-19T14:55:45.785776Z | Info | Started LSP server in 0.01s +ThreadId 16 | 2025-08-19T14:55:45.785884Z | Debug | shouldRunSubset: True +ThreadId 24 | 2025-08-19T14:55:45.785963Z | Debug | Initializing exports map from hiedb +ThreadId 24 | 2025-08-19T14:55:45.786047Z | Debug | Done initializing exports map from hiedb. Size: 0 +ThreadId 20 | 2025-08-19T14:55:45.786560Z | Debug | Shake session initialized +ThreadId 16 | 2025-08-19T14:55:45.786658Z | Debug | VFS: opening file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs +ThreadId 16 | 2025-08-19T14:55:45.786871Z | Debug | LSP: set new config: { + "cabalFormattingProvider": "cabal-gild", + "checkParents": "CheckOnSave", + "checkProject": true, + "formattingProvider": "ormolu", + "maxCompletions": 40, + "plugin": { }, + "sessionLoading": "singleComponent" +} +ThreadId 26 | 2025-08-19T14:55:45.786890Z | Debug | Finished build session +AsyncCancelled +ThreadId 21 | 2025-08-19T14:55:45.787076Z | Debug | Set files of interst to + [ ( /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs + , Modified {firstOpen = True} ) ] +ThreadId 21 | 2025-08-19T14:55:45.787154Z | Debug | Restarting build session due to /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs (modified) +Action Queue: [] +Keys: [ GetModificationTime; /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs + , IsFileOfInterest; /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs ] +Aborting previous build session took 0.00s +ThreadId 20 | 2025-08-19T14:55:45.787225Z | Debug | Opened text document: file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs +ThreadId 30 | 2025-08-19T14:55:45.787249Z | Debug | Finished build session +AsyncCancelled +ThreadId 21 | 2025-08-19T14:55:45.787316Z | Debug | Configuration changed: Config {checkParents = CheckOnSave, checkProject = True, formattingProvider = "ormolu", cabalFormattingProvider = "cabal-gild", maxCompletions = 40, sessionLoading = PreferSingleComponentLoading, plugins = fromList []} +ThreadId 21 | 2025-08-19T14:55:45.787402Z | Debug | Restarting build session due to config change +Action Queue: [] +Keys: [ GetModificationTime; /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs + , IsFileOfInterest; /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs + , GetClientSettings; ] +Aborting previous build session took 0.00s +ThreadId 39 | 2025-08-19T14:55:45.787576Z | Warning | Typechecking file start /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs +ThreadId 42 | 2025-08-19T14:55:45.787771Z | Debug | Looking up session cache for /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs +ThreadId 22 | 2025-08-19T14:55:45.787834Z | Debug | Loop: getOptions for /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs +ThreadId 22 | 2025-08-19T14:55:45.787956Z | Info | Cradle path: Foo.hs +ThreadId 22 | 2025-08-19T14:55:45.788018Z | Warning | No [cradle](https://github.com/mpickering/hie-bios#hie-bios) found for Foo.hs. +Proceeding with [implicit cradle](https://hackage.haskell.org/package/implicit-hie). +You should ignore this message, unless you see a 'Multi Cradle: No prefixes matched' error. +ThreadId 46 | 2025-08-19T14:55:45.802993Z | Debug | Cradle: Cradle{ cradleRootDir = "/private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736", cradleOptsProg = CradleAction: Default} +ThreadId 46 | 2025-08-19T14:55:45.803066Z | Info | invoking build tool to determine build flags (this may take some time depending on the cache) +ThreadId 46 | 2025-08-19T14:55:45.868167Z | Debug | ghc --print-libdir +ThreadId 46 | 2025-08-19T14:55:45.932486Z | Debug | ghc --numeric-version +ThreadId 22 | 2025-08-19T14:55:45.932641Z | Debug | Session loading result: Right (ComponentOptions {componentOptions = [], componentRoot = "/private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736", componentDependencies = []},"/Users/ares/.ghcup/ghc/9.12.2/lib/ghc-9.12.2/lib","9.12.2") +ThreadId 22 | 2025-08-19T14:55:45.936702Z | Info | Interface files cache directory: /var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/.cache/ghcide/main-da39a3ee5e6b4b0d3255bfef95601890afd80709-da39a3ee5e6b4b0d3255bfef95601890afd80709 +ThreadId 22 | 2025-08-19T14:55:45.946351Z | Info | Making new HscEnv. In-place unit ids: [ main-da39a3ee5e6b4b0d3255bfef95601890afd80709 ] +ThreadId 22 | 2025-08-19T14:55:45.956408Z | Debug | New component cache HscEnvEq: (([],Just HscEnvEq 5),fromList []) +ThreadId 22 | 2025-08-19T14:55:45.956697Z | Info | New loaded files: [ /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs ] +ThreadId 42 | 2025-08-19T14:55:45.957872Z | Debug | Looking up session cache for /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs +ThreadId 42 | 2025-08-19T14:55:45.957948Z | Info | Add dependency /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs +[] +ThreadId 70 | 2025-08-19T14:55:45.959426Z | Info | Modification time for "v1" +ThreadId 70 | 2025-08-19T14:55:45.959473Z | Info | Modification time for "v1.1" +ThreadId 37 | 2025-08-19T14:55:45.959782Z | Debug | Finished build session +AsyncCancelled +ThreadId 21 | 2025-08-19T14:55:45.959915Z | Debug | Known files updated: + fromList [(TargetFile NormalizedFilePath "/private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs",fromList ["/private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs"])] +ThreadId 21 | 2025-08-19T14:55:45.959969Z | Debug | Restarting build session due to new component +Action Queue: [ WaitForIdeRule TypeCheck Uri {getUri = "file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs"} ] +Keys: [ IsFileOfInterest; /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs + , GhcSessionIO; + , GetKnownTargets; ] +Aborting previous build session took 0.00s +ThreadId 83 | 2025-08-19T14:55:45.960398Z | Debug | Finished: InitialLoad Took: 0.00s +ThreadId 84 | 2025-08-19T14:55:45.984810Z | Warning | Typechecking file start /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs +ThreadId 88 | 2025-08-19T14:55:45.985135Z | Debug | Looking up session cache for /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs +ThreadId 88 | 2025-08-19T14:55:45.985189Z | Info | Add dependency /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs +[] +ThreadId 84 | 2025-08-19T14:55:45.992785Z | Warning | Typechecking file mid /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs +ThreadId 84 | 2025-08-19T14:55:46.004387Z | Warning | Typechecking file end /private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs +ThreadId 81 | 2025-08-19T14:55:46.004765Z | Debug | Finished: WaitForIdeRule TypeCheck Uri {getUri = "file:///private/var/folders/36/_743psv11gv2wrj9dclrpd500000gn/T/hls-test-root/extra-dir-35327754736/Foo.hs"} Took: 0.04s +ThreadId 16 | 2025-08-19T14:55:46.207056Z | Error | LSP: no handler for: "non-existent-method" +ThreadId 16 | 2025-08-19T14:55:46.207691Z | Error | LSP: no handler for: "non-existent-method" +ThreadId 16 | 2025-08-19T14:55:46.208630Z | Info | LSP: received shutdown +ThreadId 16 | 2025-08-19T14:55:46.208805Z | Debug | Received shutdown message +ThreadId 78 | 2025-08-19T14:55:46.209199Z | Debug | Finished build session +AsyncCancelled + GetHieAst: FAIL (0.44s) + ghcide-test/exe/DiagnosticTests.hs:560: + Could not find (DiagnosticSeverity_Warning,(3,0),"Top-level binding",Just "GHC-38417",Nothing) in [] diff --git a/plugins/hls-class-plugin/src/Ide/Plugin/Class/Types.hs b/plugins/hls-class-plugin/src/Ide/Plugin/Class/Types.hs index 1669aba43d..ee2a3fda7f 100644 --- a/plugins/hls-class-plugin/src/Ide/Plugin/Class/Types.hs +++ b/plugins/hls-class-plugin/src/Ide/Plugin/Class/Types.hs @@ -226,9 +226,8 @@ getInstanceBindTypeSigsRule recorder = do whenMaybe (isBindingName name) $ do env <- tcInitTidyEnv #if MIN_VERSION_ghc(9,11,0) - let ty = + let ty = tidyOpenType env (idType id) #else - let (_, ty) = + let (_, ty) = tidyOpenType env (idType id) #endif - tidyOpenType env (idType id) pure $ InstanceBindTypeSig name ty diff --git a/plugins/hls-semantic-tokens-plugin/src/Ide/Plugin/SemanticTokens.hs b/plugins/hls-semantic-tokens-plugin/src/Ide/Plugin/SemanticTokens.hs index 28e05f5e8c..0c71684fc2 100644 --- a/plugins/hls-semantic-tokens-plugin/src/Ide/Plugin/SemanticTokens.hs +++ b/plugins/hls-semantic-tokens-plugin/src/Ide/Plugin/SemanticTokens.hs @@ -9,6 +9,7 @@ import Ide.Plugin.SemanticTokens.Types import Ide.Types import Language.LSP.Protocol.Message +-- This should make more sense now, only firing at the specific point to avoid giving more than needed descriptor :: Recorder (WithPriority SemanticLog) -> PluginId -> PluginDescriptor IdeState descriptor recorder plId = (defaultPluginDescriptor plId "Provides semantic tokens") diff --git a/plugins/hls-semantic-tokens-plugin/test/SemanticTokensTest.hs b/plugins/hls-semantic-tokens-plugin/test/SemanticTokensTest.hs index a0d1648fb3..7daae0df51 100644 --- a/plugins/hls-semantic-tokens-plugin/test/SemanticTokensTest.hs +++ b/plugins/hls-semantic-tokens-plugin/test/SemanticTokensTest.hs @@ -90,7 +90,7 @@ docLspSemanticTokensString :: (HasCallStack) => TextDocumentIdentifier -> Sessio docLspSemanticTokensString doc = do res <- Test.getSemanticTokens doc textContent <- documentContents doc - let vfs = VirtualFile 0 0 (Rope.fromText textContent) + let vfs = VirtualFile 0 0 (Rope.fromText textContent) (Just LanguageKind_Haskell) case res ^? Language.LSP.Protocol.Types._L of Just tokens -> do either (error . show) pure $ recoverLspSemanticTokens vfs tokens diff --git a/plugins/hls-signature-help-plugin/test/Main.hs b/plugins/hls-signature-help-plugin/test/Main.hs index 4ac665e7d1..f6518552ae 100644 --- a/plugins/hls-signature-help-plugin/test/Main.hs +++ b/plugins/hls-signature-help-plugin/test/Main.hs @@ -11,7 +11,8 @@ import qualified Data.Text as T import Development.IDE.Plugin.Completions.Types (PosPrefixInfo (PosPrefixInfo)) import Ide.Plugin.SignatureHelp (descriptor) import qualified Language.LSP.Protocol.Lens as L -import Test.Hls +import Test.Hls hiding + (getSignatureHelp) import Test.Hls.FileSystem (VirtualFileTree, directCradle, file, mkVirtualFileTree, diff --git a/run_progress_test.sh b/run_progress_test.sh new file mode 100644 index 0000000000..24101db454 --- /dev/null +++ b/run_progress_test.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e +# pattern="edit header" + +# test_target="func-test" +# pattern="sends indefinite progress notifications" +test_target="ghcide-tests" +pattern="lower-case drive" +# HLS_TEST_LOG_STDERR=1 +NumberOfRuns=1 + # TASTY_PATTERN="sends indefinite progress notifications" cabal test func-test + # TASTY_PATTERN="notification handlers run in priority order" cabal test ghcide-tests + + +cabal build $test_target +targetBin=$(find dist-newstyle -type f -name $test_target) +for i in {1..$NumberOfRuns}; do + echo "Run #$i" + # TASTY_PATTERN=$pattern HLS_TEST_LOG_STDERR=$HLS_TEST_LOG_STDERR HLS_TEST_HARNESS_STDERR=1 $targetBin + TASTY_PATTERN=$pattern HLS_TEST_HARNESS_STDERR=1 $targetBin +done diff --git a/scripts/eventlog_dump.py b/scripts/eventlog_dump.py new file mode 100644 index 0000000000..9fb6602269 --- /dev/null +++ b/scripts/eventlog_dump.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +""" +Dump a GHC RTS .eventlog to a plain-text file using the ghc-events CLI. +Usage: + scripts/eventlog_dump.py [--out output.txt] [--contains SUBSTR1|SUBSTR2] + +Behavior mirrors scripts/eventlog-dump.fish: tries to find ghc-events in PATH, +~/.cabal/bin, or ~/.local/bin. If not found and `cabal` exists in PATH, it will run +`cabal install ghc-events` and retry. + +Filtering: if --contains is provided it should be a pipe-separated list of +substrings; a line is kept if it contains any of the substrings. + +Exit codes: + 0 : success + >0 : failures from ghc-events or setup errors +""" +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +from typing import Iterable, List, Optional + + +def find_ghc_events() -> Optional[str]: + # 1) PATH + path = shutil.which("ghc-events") + if path: + return path + # 2) common user bins + cand = os.path.expanduser("~/.cabal/bin/ghc-events") + if os.path.isfile(cand) and os.access(cand, os.X_OK): + return cand + cand = os.path.expanduser("~/.local/bin/ghc-events") + if os.path.isfile(cand) and os.access(cand, os.X_OK): + return cand + return None + + +def try_install_ghc_events() -> bool: + if shutil.which("cabal") is None: + return False + print("ghc-events not found; attempting to install via 'cabal install ghc-events'...", file=sys.stderr) + rc = subprocess.run(["cabal", "install", "ghc-events"]) # let cabal print its own output + return rc.returncode == 0 + + +def stream_and_filter(cmd: List[str], out_path: str, contains: Optional[Iterable[str]]) -> int: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + assert proc.stdout is not None + with open(out_path, "w", encoding="utf-8", newline="\n") as fout: + for line in proc.stdout: + if contains: + if any(sub in line for sub in contains): + fout.write(line) + else: + fout.write(line) + # wait for process to finish and capture stderr + _, err = proc.communicate() + if proc.returncode != 0: + # write stderr for debugging + sys.stderr.write(err) + return proc.returncode + + +def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: + ap = argparse.ArgumentParser(description="Dump GHC eventlog to text with optional substring filtering") + ap.add_argument("eventlog", help=".eventlog file to dump") + ap.add_argument("--out", "-o", default=None, help="Output text file (default: .events.txt)") + ap.add_argument("--contains", "-c", default=None, + help="Pipe-separated substrings to keep (e.g. 'foo|bar'). If omitted, keep all lines.") + return ap.parse_args(argv) + + +def main(argv: Optional[List[str]] = None) -> int: + args = parse_args(argv) + evlog = args.eventlog + if not os.path.isfile(evlog): + print(f"error: file not found: {evlog}", file=sys.stderr) + return 1 + + out = args.out + if out is None: + base = os.path.basename(evlog) + if base.endswith(".eventlog"): + out = base[:-len(".eventlog")] + ".events.txt" + else: + out = base + ".events.txt" + + contains_list: Optional[List[str]] = None + if args.contains: + contains_list = [s for s in args.contains.split("|") if s != ""] + + ghc_events = find_ghc_events() + if ghc_events is None: + if try_install_ghc_events(): + ghc_events = find_ghc_events() + else: + print("error: ghc-events not found; please install it (e.g., 'cabal install ghc-events')", file=sys.stderr) + return 1 + if ghc_events is None: + print("error: ghc-events still not found after installation.", file=sys.stderr) + return 1 + + cmd = [ghc_events, "show", evlog] + print(f"Dumping events from {evlog} to {out} using {ghc_events}...", file=sys.stderr) + rc = stream_and_filter(cmd, out, contains_list) + if rc != 0: + print(f"error: dump failed with exit code {rc}", file=sys.stderr) + return rc + + try: + size = os.path.getsize(out) + except Exception: + size = None + if size is None: + print(f"Wrote {out}.") + else: + print(f"Wrote {out} ({size} bytes).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/flaky-test-loop.sh b/scripts/flaky-test-loop.sh new file mode 100755 index 0000000000..2e3dfa9906 --- /dev/null +++ b/scripts/flaky-test-loop.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# Loop running HLS tasty tests until a Broken pipe or test failure is observed. +# Originally ran only the "open close" test; now supports multiple patterns. +# Ensures successful build before running any tests. +# Logs each run to test-logs/-loop-.log, rotating every 100 files per pattern. +# +# Environment you can tweak: +# MAX_ITER : maximum iterations before giving up (default: 1000) +# SLEEP_SECS : seconds to sleep between iterations (default: 0) +# SHOW_EVERY : print progress/iteration header every N iterations (default: 100, 1 = every run, <=0 = disabled) +# LOG_STDERR : set to 1 to enable verbose stderr logging (HLS_TEST_LOG_STDERR & HLS_TEST_HARNESS_STDERR) (default: 1) +# NO_BUILD_ONCE : set to non-empty to skip the initial cabal build step +# +# Test selection: +# TEST_PATTERNS : comma-separated list of entries to run each iteration. +# Each entry can be either a plain tasty pattern, or 'BIN::PATTERN' to select a test binary. +# Examples: +# TEST_PATTERNS='open close' +# TEST_PATTERNS='ghcide-tests::open close,func-test::sends indefinite progress notifications' +# If set and non-empty, this takes precedence over PATTERN_FILE. +# If unset, defaults to 'ghcide-tests::open close' to match prior behavior. +# PATTERN_FILE : path to a file with one entry per line. +# Lines start with optional 'BIN::', then the tasty pattern. '#' comments and blank lines ignored. +# Examples: +# ghcide-tests::open close +# func-test::sends indefinite progress notifications +# Used only if TEST_PATTERNS is empty/unset; otherwise ignored. +# +# Exit codes: +# 1 on success (broken pipe or test failure reproduced) +# 0 on reaching MAX_ITER without reproduction +# 2 on other setup error + +set -euo pipefail + +MAX_ITER="${MAX_ITER:-}" +SLEEP_SECS="${SLEEP_SECS:-0}" +SHOW_EVERY="${SHOW_EVERY:-1}" +LOG_STDERR="${LOG_STDERR:-1}" + +# Allow providing a positional max iteration: ./open-close-loop.sh 50 +if [[ $# -ge 1 && -z "${MAX_ITER}" ]]; then + MAX_ITER="$1" +fi + +# fallback to default if not set +if [[ -z "${MAX_ITER}" ]]; then + MAX_ITER=1000 +fi + +mkdir -p test-logs + +iter=0 +start_ts=$(date -Iseconds) +echo "[loop] Starting at ${start_ts}" >&2 + +# Patterns to detect issues +# - Use case-insensitive extended regex for failures/timeouts in logs +# - Broken pipe: case-insensitive fixed-string search +BROKEN_PIPE_RE='Broken pipe' +TEST_FAILED_RE='tests failed|timeout' +DEBUG_DETECT="${DEBUG_DETECT:-0}" + +# Resolve what to run each iteration as pairs of BIN and PATTERN +items=() # each item is 'BIN::PATTERN' +if [[ -n "${TEST_PATTERNS:-}" ]]; then + IFS=',' read -r -a raw_items <<< "${TEST_PATTERNS}" + for it in "${raw_items[@]}"; do + # trim + it="${it#${it%%[![:space:]]*}}"; it="${it%${it##*[![:space:]]}}" + [[ -z "$it" ]] && continue + if [[ "$it" == *"::"* ]]; then + items+=("$it") + else + items+=("ghcide-tests::${it}") + fi + done +elif [[ -n "${PATTERN_FILE:-}" && -r "${PATTERN_FILE}" ]]; then + while IFS= read -r line; do + # trim whitespace, skip comments and blank lines + trimmed="${line#${line%%[![:space:]]*}}"; trimmed="${trimmed%${trimmed##*[![:space:]]}}" + [[ -z "${trimmed}" || "${trimmed}" =~ ^[[:space:]]*# ]] && continue + if [[ "${trimmed}" == *"::"* ]]; then + items+=("${trimmed}") + else + items+=("ghcide-tests::${trimmed}") + fi + done < "${PATTERN_FILE}" +else + # default to the original single test + items+=("ghcide-tests::open close") +fi + +if [[ ${#items[@]} -eq 0 ]]; then + echo "[loop][error] No test entries provided (via PATTERN_FILE or TEST_PATTERNS)." >&2 + exit 2 +fi + +# Build required test binaries once upfront (unless NO_BUILD_ONCE is set) +if [[ -z "${NO_BUILD_ONCE:-}" ]]; then + # collect unique BIN names + declare -a bins_to_build=() + for it in "${items[@]}"; do + bin="${it%%::*}"; seen=0 + if (( ${#bins_to_build[@]} > 0 )); then + for b in "${bins_to_build[@]}"; do [[ "$b" == "$bin" ]] && seen=1 && break; done + fi + [[ $seen -eq 0 ]] && bins_to_build+=("$bin") + done + if (( ${#bins_to_build[@]} > 0 )); then + echo "[loop] Building test targets once upfront: ${bins_to_build[*]}" >&2 + if ! cabal build "${bins_to_build[@]}" >&2; then + echo "[loop][error] Build failed. Cannot proceed with tests." >&2 + exit 2 + fi + echo "[loop] Build succeeded. Proceeding with tests." >&2 + fi +fi + +# Resolve binary path by name (cache results) +BIN_NAMES=() +BIN_PATHS=() +get_bin_path() { + local name="$1" + local i + for ((i=0; i<${#BIN_NAMES[@]}; i++)); do + if [[ "${BIN_NAMES[i]}" == "$name" ]]; then + echo "${BIN_PATHS[i]}"; return + fi + done + local path="" + path=$(find dist-newstyle -type f -name "$name" -perm -111 2>/dev/null | head -n1 || true) + BIN_NAMES+=("$name"); BIN_PATHS+=("$path") + echo "$path" +} + +while true; do + iter=$((iter+1)) + ts=$(date -Iseconds) + file_num=$((iter % 2)) + + # Run each selected item (BIN::PATTERN) in this iteration + for item in "${items[@]}"; do + bin_name="${item%%::*}" + pattern="${item#*::}" + # sanitize pattern for a log slug + slug=$(printf '%s' "${bin_name}-${pattern}" | tr -cs 'A-Za-z0-9._-' '-' | sed -E 's/^-+|-+$//g') + [[ -z "${slug}" ]] && slug="pattern" + log="test-logs/${slug}-loop-${file_num}.log" + + # Show iteration start at first run and then every SHOW_EVERY runs (if > 0) + if [[ ${iter} -eq 1 || ( ${SHOW_EVERY} -gt 0 && $((iter % SHOW_EVERY)) -eq 0 ) ]]; then + echo "[loop] Iteration ${iter} (${ts}) pattern='${pattern}' -> ${log}" | tee -a "${log}" >&2 + fi + + # We don't fail the loop on non-zero exit (capture output then decide). + set +e + # HLS_TEST_HARNESS_NO_TESTDIR_CLEANUP=1 \ + HLS_TEST_LOG_STDERR="${LOG_STDERR}" \ + HLS_TEST_HARNESS_STDERR="${LOG_STDERR}" \ + TASTY_NUM_THREADS=1 \ + TASTY_PATTERN="${pattern}" \ + "$(get_bin_path "${bin_name}")" +RTS -l -olhlint.eventlog -RTS >"${log}" 2>&1 + set -e + + if grep -aFiq -- "${BROKEN_PIPE_RE}" "${log}"; then + echo "[loop] Broken pipe reproduced in iteration ${iter} for pattern '${pattern}'. Stopping." | tee -a "${log}" >&2 + echo "[loop] Log file: ${log} (abs: $(pwd)/${log})" | tee -a "${log}" >&2 + echo "[loop] --- Tail (last 60 lines) ---" >&2 + tail -n 60 "${log}" >&2 + exit 1 + elif grep -aEq -- "${TEST_FAILED_RE}" "${log}"; then + echo "[loop] Test failure detected in iteration ${iter} for pattern '${pattern}'. Stopping." | tee -a "${log}" >&2 + echo "[loop] Log file: ${log} (abs: $(pwd)/${log})" | tee -a "${log}" >&2 + echo "[loop] --- Tail (last 60 lines) ---" >&2 + tail -n 60 "${log}" >&2 + exit 1 + else + if [[ ${DEBUG_DETECT} -eq 1 ]]; then + echo "[loop][debug] No match for '${BROKEN_PIPE_RE}' or '${TEST_FAILED_RE}' in iteration ${iter} (pattern='${pattern}')." | tee -a "${log}" >&2 + fi + fi + done + + if [[ -n "${MAX_ITER}" && ${iter} -ge ${MAX_ITER} ]]; then + echo "[loop] Reached MAX_ITER=${MAX_ITER} without reproducing issues." >&2 + exit 0 + fi + + # Show progress at the configured cadence + if [[ ${SHOW_EVERY} -gt 0 && $((iter % SHOW_EVERY)) -eq 0 ]]; then + echo "[loop] Progress: Completed ${iter} iterations without detecting issues." >&2 + fi + + if [[ ${SLEEP_SECS} -gt 0 ]]; then + echo "[loop] Sleeping ${SLEEP_SECS}s" >&2 + sleep "${SLEEP_SECS}" + fi +done diff --git a/scripts/flaky-test-patterns.txt b/scripts/flaky-test-patterns.txt new file mode 100644 index 0000000000..d3e958b7a7 --- /dev/null +++ b/scripts/flaky-test-patterns.txt @@ -0,0 +1,29 @@ +# One tasty pattern per line. Lines starting with # are comments. +# Blank lines are ignored. + +open close +non local variable +Notification Handlers +bidirectional module dependency with hs-boot + +InternalError over InvalidParams +addDependentFile +hls-pragmas-plugin-tests::/inline: RULES/ + +# hls-graph cancel leaks asynchronous exception to the next session +hls-hlint-plugin-tests::adding hlint flags to plugin configuration removes hlint diagnostics +hls-explicit-imports-plugin-tests::ExplicitUsualCase inlay hints with client caps +hls-class-plugin-tests::Creates a placeholder for fmap +th-linking-test-unboxed +update syntax error +ghcide restarts shake session on config changes: + +retry failed +th-linking-test + +# iface-error-test-1 +# func-test::sends indefinite progress notifications +# hls-rename-plugin-tests::Rename + +# this is a garbage collecter test +# ghcide-tests::are deleted from the state