Skip to content

Commit

Permalink
Refactor FileWatcher to work on large project on macOS
Browse files Browse the repository at this point in the history
  • Loading branch information
@pvid
pvid committed Aug 18, 2021
1 parent 0576d62 commit 2422494
Show file tree
Hide file tree
Showing 3 changed files with 254 additions and 106 deletions.
20 changes: 19 additions & 1 deletion metals/src/main/scala/scala/meta/internal/metals/MetalsLanguageServer.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,9 @@ class MetalsLanguageServer(
)
private val fileWatcher = register(
new FileWatcher(
() => workspace,
buildTargets,
fileWatchFilter,
params => didChangeWatchedFiles(params)
)
)
Expand Down Expand Up @@ -1234,7 +1236,23 @@ class MetalsLanguageServer(
onChange(paths).asJava
}

// This method is run synchronously in the FileWatcher, so it should not do anything expensive on the main thread
/**
* This filter is an optimization and it is closely related to which files are processed
* in [[didChangeWatchedFiles]]
*/
private def fileWatchFilter(path: Path): Boolean = {
val abs = AbsolutePath(path)
abs.isScalaOrJava || abs.isSemanticdb || abs.isBuild
}

/**
* Callback that is executed on a file change event by the file watcher.
*
* Note that if you are adding processing of another kind of a file,
* be sure to include it in the [[fileWatchFilter]]
*
* This method is run synchronously in the FileWatcher, so it should not do anything expensive on the main thread
*/
private def didChangeWatchedFiles(
event: FileWatcherEvent
): CompletableFuture[Unit] = {
Expand Down
267 changes: 162 additions & 105 deletions metals/src/main/scala/scala/meta/internal/metals/watcher/FileWatcher.scala
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
package scala.meta.internal.metals.watcher

import java.io.BufferedInputStream
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path

import scala.collection.mutable
import scala.util.hashing.MurmurHash3

import scala.meta.internal.metals.BuildTargets
import scala.meta.internal.metals.Cancelable
Expand All @@ -16,90 +19,63 @@ import com.swoval.files.FileTreeDataViews.Converter
import com.swoval.files.FileTreeDataViews.Entry
import com.swoval.files.FileTreeRepositories
import com.swoval.files.FileTreeRepository
import java.io.BufferedInputStream
import java.nio.file.Files
import scala.util.hashing.MurmurHash3

/**
* Handles file watching of interesting files in this build.
* Watch selected files and execute a callback on file events.
*
* This class recursively watches selected directories and selected files.
* File events can be further filtered by the `watchFiler` parameter, which can speed by watching for changes
* by limiting the number of files that need to be hashed.
*
* Tries to minimize file events by dynamically watching only relevant directories for
* the structure of the build. We don't use the LSP dynamic file watcher capability because
* We don't use the LSP dynamic file watcher capability because
*
* 1. the glob syntax is not defined in the LSP spec making it difficult to deliver a
* consistent file watching experience with all editor clients on all operating systems.
* 2. we may have a lot of file watching events and it's presumably less overhead to
* get the notifications directly from the OS instead of through the editor via LSP.
*
* Given we rely on file watching for critical functionality like Goto Definition and it's
* really difficult to reproduce/investigate file watching issues, I think it's best to
* have a single file watching solution that we have control over.
*
* This class does not watch for changes in `*.sbt` files in the workspace directory and
* in the `project/`. Those notifications are nice-to-have, but not critical.
*/
final class FileWatcher(
workspaceDeferred: () => AbsolutePath,
buildTargets: BuildTargets,
didChangeWatchedFiles: FileWatcherEvent => Unit
watchFilter: Path => Boolean,
onFileWatchEvent: FileWatcherEvent => Unit
) extends Cancelable {
import FileWatcher._

private var repository: Option[FileTreeRepository[Hash]] = None

private def newRepository: FileTreeRepository[Hash] = {
val converter: Converter[Hash] = typedPath => hashFile(typedPath.getPath())
val repo = FileTreeRepositories.get(converter, /*follow symlinks*/ true)

repo.addCacheObserver(new CacheObserver[Hash] {
override def onCreate(entry: Entry[Hash]): Unit = {
didChangeWatchedFiles(
FileWatcherEvent.create(entry.getTypedPath.getPath)
)
}
override def onDelete(entry: Entry[Hash]): Unit = {
didChangeWatchedFiles(
FileWatcherEvent.delete(entry.getTypedPath.getPath)
)
}
override def onUpdate(
previous: Entry[Hash],
current: Entry[Hash]
): Unit = {
if (previous.getValue != current.getValue) {
didChangeWatchedFiles(
FileWatcherEvent.modify(current.getTypedPath.getPath)
)
}
}
override def onError(ex: IOException) = {}
})
repo
}
private var disposeAction: Option[() => Unit] = None

override def cancel(): Unit = {
repository.map(_.close())
repository = None
disposeAction.map(_.apply())
disposeAction = None
}

def restart(): Unit = {
disposeAction.map(_.apply())

val newDispose = startWatch(
workspaceDeferred().toNIO,
collectFilesToWatch(buildTargets),
onFileWatchEvent,
watchFilter
)
disposeAction = Some(newDispose)
}
}

object FileWatcher {
type Hash = Int

private case class FilesToWatch(
sourceFiles: Set[Path],
sourceDirectories: Set[Path],
semanticdDirectories: Set[Path]
)

private def collectFilesToWatch(buildTargets: BuildTargets): FilesToWatch = {
val sourceDirectoriesToWatch = mutable.Set.empty[Path]
val sourceFilesToWatch = mutable.Set.empty[Path]
val createdSourceDirectories = new java.util.ArrayList[AbsolutePath]()
def watch(path: AbsolutePath, isSource: Boolean): Unit = {
if (!path.isDirectory && !path.isFile) {
val pathToCreate = if (path.isScalaOrJava) {
AbsolutePath(path.toNIO.getParent())
} else {
path
}
val createdPaths = pathToCreate.createAndGetDirectories()
// this is a workaround for MacOS, it will continue watching
// directories even if they are removed, however it doesn't
// work on some other systems like Linux
if (isSource) {
createdPaths.foreach(createdSourceDirectories.add)
}
}

def collect(path: AbsolutePath): Unit = {
if (buildTargets.isInsideSourceRoot(path)) {
() // Do nothing, already covered by a source root
} else if (path.isScalaOrJava) {
Expand All @@ -109,57 +85,138 @@ final class FileWatcher(
}
}
// Watch the source directories for "goto definition" index.
buildTargets.sourceRoots.foreach(watch(_, isSource = true))
buildTargets.sourceItems.foreach(watch(_, isSource = true))
buildTargets.scalacOptions.foreach { item =>
buildTargets.sourceRoots.foreach(collect)
buildTargets.sourceItems.foreach(collect)
val semanticdbs = buildTargets.scalacOptions.flatMap { item =>
for {
scalaInfo <- buildTargets.scalaInfo(item.getTarget)
} {
val targetroot = item.targetroot(scalaInfo.getScalaVersion)
if (!targetroot.isJar) {
// Watch META-INF/semanticdb directories for "find references" index.
watch(
targetroot.resolve(Directories.semanticdb),
isSource = false
)
}
}

targetroot = item.targetroot(scalaInfo.getScalaVersion)
path = targetroot.resolve(Directories.semanticdb) if !targetroot.isJar
} yield path.toNIO
}

repository.map(_.close())
val repo = newRepository
repository = Some(repo)

// The second parameter of repo.register is the recursive depth of the watch.
// A value of -1 means only watch this exact path. A value of 0 means only
// watch the immediate children of the path. A value of 1 means watch the
// children of the path and each child's children and so on.
sourceDirectoriesToWatch.foreach(repo.register(_, Int.MaxValue))
sourceFilesToWatch.foreach(repo.register(_, -1))

// reverse sorting here is necessary to delete parent paths at the end
createdSourceDirectories.asScala.sortBy(_.toNIO).reverse.foreach { dir =>
if (dir.isEmptyDirectory) {
dir.delete()
FilesToWatch(
sourceFilesToWatch.toSet,
sourceDirectoriesToWatch.toSet,
semanticdbs.toSet
)
}

/**
* Start file watching
*
* Contains platform specific file watch initialization logic
*
* @param workspace current project workspace directory
* @param filesToWatch source files and directories to watch
* @param callback to execute on FileWatchEvent
* @param watchFilter predicate that filters which files
* generate a FileWatchEvent on create/delete/change
* @return a dispose action resources used by file watching
*/
private def startWatch(
workspace: Path,
filesToWatch: FilesToWatch,
callback: FileWatcherEvent => Unit,
watchFilter: Path => Boolean
): () => Unit = {
if (scala.util.Properties.isMac) {
// Due to a hard limit on the number of FSEvents streams that can be opened on macOS,
// only the root workspace directory is registered for a recursive watch.
// However, the events are then filtered to receive only relevant events
// and also to hash only revelevant files when watching for changes

val trie = PathTrie(
filesToWatch.sourceFiles ++ filesToWatch.sourceDirectories ++ filesToWatch.semanticdDirectories
)
val isWatched = trie.containsPrefixOf _

val repo = initFileTreeRepository(
path => watchFilter(path) && isWatched(path),
callback
)
repo.register(workspace, Int.MaxValue)
() => repo.close()
} else {
// Other OSes register all the files and directories individually
val repo = initFileTreeRepository(watchFilter, callback)

// TODO(@pvid) swoval's FileTreeRepository should be able to create watch
// for files/directories that do not exist yet. However, there is an issue
// with watching **semanticdb** files when not creating source directories.
// I was not able to diagnose the issue.
// If you'd like to dive deeper into, try to remove the file creation and deletion
// and run some tests with `-Dswoval.log.level=debug` to see which files are registered
// and which file events are received.
val directoriesToCreate =
filesToWatch.sourceDirectories ++ filesToWatch.sourceFiles.map(
_.getParent()
)

val createdDirectories = directoriesToCreate.flatMap(path =>
AbsolutePath(path).createAndGetDirectories()
)

filesToWatch.sourceDirectories.foreach(repo.register(_, Int.MaxValue))
filesToWatch.semanticdDirectories.foreach(repo.register(_, Int.MaxValue))
filesToWatch.sourceFiles.foreach(repo.register(_, -1))

createdDirectories.toSeq.sortBy(_.toNIO).reverse.foreach { dir =>
if (dir.isEmptyDirectory) dir.delete()
}

() => repo.close()
}
}
}

object FileWatcher {
type Hash = Int

def hashFile(path: Path): Hash = {
val inputStream = new BufferedInputStream(Files.newInputStream(path))
try {
MurmurHash3.orderedHash(
Stream.continually(inputStream.read()).takeWhile(_ != -1)
private def initFileTreeRepository(
watchFilter: Path => Boolean,
callback: FileWatcherEvent => Unit
): FileTreeRepository[Hash] = {
val converter: Converter[Hash] = typedPath =>
hashFile(
typedPath.getPath(),
watchFilter
)
} catch {
case _: IOException => 0
} finally {
inputStream.close()
val repo = FileTreeRepositories.get(converter, /*follow symlinks*/ true)

repo.addCacheObserver(new CacheObserver[Hash] {
override def onCreate(entry: Entry[Hash]): Unit = {
val path = entry.getTypedPath().getPath()
if (watchFilter(path)) callback(FileWatcherEvent.create(path))
}
override def onDelete(entry: Entry[Hash]): Unit = {
val path = entry.getTypedPath().getPath()
if (watchFilter(path)) callback(FileWatcherEvent.delete(path))
}
override def onUpdate(
previous: Entry[Hash],
current: Entry[Hash]
): Unit = {
val path = current.getTypedPath().getPath()
if (previous.getValue != current.getValue && watchFilter(path)) {
callback(FileWatcherEvent.modify(path))
}
}
override def onError(ex: IOException) = {}
})
repo
}

private def hashFile(path: Path, hashFilter: Path => Boolean): Hash = {
if (hashFilter(path)) {
val inputStream = new BufferedInputStream(Files.newInputStream(path))
try {
MurmurHash3.orderedHash(
Stream.continually(inputStream.read()).takeWhile(_ != -1)
)
} catch {
case _: IOException => 0
} finally {
inputStream.close()
}
} else {
0
}
}
}
Loading

0 comments on commit 2422494

Please sign in to comment.