Support directory listing (#284)

fzhinkin · web-flow · commit 8d8735015ff9 · 2024-05-07T11:49:51.000+02:00
Closes #222
diff --git a/build-logic/src/main/kotlin/kotlinx/io/conventions/kotlinx-io-multiplatform.gradle.kts b/build-logic/src/main/kotlin/kotlinx/io/conventions/kotlinx-io-multiplatform.gradle.kts
@@ -73,13 +73,19 @@ kotlin {
     applyDefaultHierarchyTemplate {
         common {
             group("native") {
-                group("nonApple") {
+                group("nativeNonApple") {
                     group("mingw")
                     group("unix") {
                         group("linux")
                         group("androidNative")
                     }
                 }
+
+                group("nativeNonAndroid") {
+                    group("apple")
+                    group("mingw")
+                    group("linux")
+                }
             }
             group("nodeFilesystemShared") {
                 withJs()
diff --git a/core/Module.md b/core/Module.md
@@ -82,3 +82,8 @@ Core IO primitives.
 # Package kotlinx.io.files
 
 Basic API for working with files.
+
+#### Known issues
+
+- [#312](https://github.com/Kotlin/kotlinx-io/issues/312) For `wasmWasi` target, directory listing ([kotlinx.io.files.FileSystem.list]) does not work with NodeJS runtime on Windows,
+as `fd_readdir` function is [not implemented there](https://github.com/nodejs/node/blob/6f4d6011ea1b448cf21f5d363c44e4a4c56ca34c/deps/uvwasi/src/uvwasi.c#L19).
diff --git a/core/androidNative/src/files/FileSystemAndroid.kt b/core/androidNative/src/files/FileSystemAndroid.kt
@@ -5,10 +5,12 @@
 
 package kotlinx.io.files
 
+import kotlinx.cinterop.CPointer
 import kotlinx.cinterop.ExperimentalForeignApi
+import kotlinx.cinterop.get
 import kotlinx.cinterop.toKString
-import platform.posix.__posix_basename
-import platform.posix.dirname
+import kotlinx.io.IOException
+import platform.posix.*
 
 @OptIn(ExperimentalForeignApi::class)
 internal actual fun dirnameImpl(path: String): String {
@@ -24,3 +26,22 @@ internal actual fun basenameImpl(path: String): String {
 }
 
 internal actual fun isAbsoluteImpl(path: String): Boolean = path.startsWith('/')
+
+@OptIn(ExperimentalForeignApi::class, ExperimentalStdlibApi::class)
+internal actual class OpaqueDirEntry constructor(private val dir: CPointer<cnames.structs.DIR>) : AutoCloseable {
+    actual fun readdir(): String? {
+        val entry = platform.posix.readdir(dir) ?: return null
+        return entry[0].d_name.toKString()
+    }
+
+    override fun close() {
+        closedir(dir)
+    }
+}
+
+@OptIn(ExperimentalForeignApi::class)
+internal actual fun opendir(path: String): OpaqueDirEntry {
+    val dirent = platform.posix.opendir(path)
+    if (dirent != null) return OpaqueDirEntry(dirent)
+    throw IOException("Can't open directory $path: ${strerror(errno)?.toKString() ?: "reason unknown"}")
+}
diff --git a/core/api/kotlinx-io-core.api b/core/api/kotlinx-io-core.api
@@ -215,6 +215,7 @@ public abstract interface class kotlinx/io/files/FileSystem {
 	public abstract fun delete (Lkotlinx/io/files/Path;Z)V
 	public static synthetic fun delete$default (Lkotlinx/io/files/FileSystem;Lkotlinx/io/files/Path;ZILjava/lang/Object;)V
 	public abstract fun exists (Lkotlinx/io/files/Path;)Z
+	public abstract fun list (Lkotlinx/io/files/Path;)Ljava/util/Collection;
 	public abstract fun metadataOrNull (Lkotlinx/io/files/Path;)Lkotlinx/io/files/FileMetadata;
 	public abstract fun resolve (Lkotlinx/io/files/Path;)Lkotlinx/io/files/Path;
 	public abstract fun sink (Lkotlinx/io/files/Path;Z)Lkotlinx/io/RawSink;
diff --git a/core/api/kotlinx-io-core.klib.api b/core/api/kotlinx-io-core.klib.api
@@ -164,6 +164,7 @@ sealed interface kotlinx.io.files/FileSystem { // kotlinx.io.files/FileSystem|nu
     abstract fun createDirectories(kotlinx.io.files/Path, kotlin/Boolean =...) // kotlinx.io.files/FileSystem.createDirectories|createDirectories(kotlinx.io.files.Path;kotlin.Boolean){}[0]
     abstract fun delete(kotlinx.io.files/Path, kotlin/Boolean =...) // kotlinx.io.files/FileSystem.delete|delete(kotlinx.io.files.Path;kotlin.Boolean){}[0]
     abstract fun exists(kotlinx.io.files/Path): kotlin/Boolean // kotlinx.io.files/FileSystem.exists|exists(kotlinx.io.files.Path){}[0]
+    abstract fun list(kotlinx.io.files/Path): kotlin.collections/Collection<kotlinx.io.files/Path> // kotlinx.io.files/FileSystem.list|list(kotlinx.io.files.Path){}[0]
     abstract fun metadataOrNull(kotlinx.io.files/Path): kotlinx.io.files/FileMetadata? // kotlinx.io.files/FileSystem.metadataOrNull|metadataOrNull(kotlinx.io.files.Path){}[0]
     abstract fun resolve(kotlinx.io.files/Path): kotlinx.io.files/Path // kotlinx.io.files/FileSystem.resolve|resolve(kotlinx.io.files.Path){}[0]
     abstract fun sink(kotlinx.io.files/Path, kotlin/Boolean =...): kotlinx.io/RawSink // kotlinx.io.files/FileSystem.sink|sink(kotlinx.io.files.Path;kotlin.Boolean){}[0]
diff --git a/core/apple/src/files/FileSystemApple.kt b/core/apple/src/files/FileSystemApple.kt
@@ -6,10 +6,7 @@
 
 package kotlinx.io.files
 
-import kotlinx.cinterop.ExperimentalForeignApi
-import kotlinx.cinterop.cstr
-import kotlinx.cinterop.memScoped
-import kotlinx.cinterop.toKString
+import kotlinx.cinterop.*
 import kotlinx.io.IOException
 import platform.Foundation.*
 import platform.posix.*
diff --git a/core/build.gradle.kts b/core/build.gradle.kts
@@ -3,6 +3,7 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENCE file.
  */
 
+import org.gradle.internal.os.OperatingSystem
 import org.jetbrains.dokka.gradle.DokkaTaskPartial
 
 plugins {
@@ -30,6 +31,17 @@ kotlin {
             }
         }
     }
+    wasmWasi {
+        nodejs {
+            testTask {
+                // fd_readdir is unsupported on Windows:
+                // https://github.com/nodejs/node/blob/6f4d6011ea1b448cf21f5d363c44e4a4c56ca34c/deps/uvwasi/src/uvwasi.c#L19
+                if (OperatingSystem.current().isWindows) {
+                    filter.setExcludePatterns("*SmokeFileTest.listDirectory")
+                }
+            }
+        }
+    }
 
     sourceSets {
         commonMain.dependencies {
diff --git a/core/common/src/files/FileSystem.kt b/core/common/src/files/FileSystem.kt
@@ -145,6 +145,25 @@ public sealed interface FileSystem {
      * @throws FileNotFoundException if there is no file or directory corresponding to the specified path.
      */
     public fun resolve(path: Path): Path
+
+    /**
+     * Returns paths corresponding to [directory]'s immediate children.
+     *
+     * There are no guarantees on children paths order within a returned collection.
+     *
+     * If path [directory] was an absolute path, a returned collection will also contain absolute paths.
+     * If it was a relative path, a returned collection will contain relative paths.
+     *
+     * *For `wasmWasi` target, function does not work with NodeJS runtime on Windows,
+     * as `fd_readdir` function is [not implemented there](https://github.com/nodejs/node/blob/6f4d6011ea1b448cf21f5d363c44e4a4c56ca34c/deps/uvwasi/src/uvwasi.c#L19).*
+     *
+     * @param directory a directory to list.
+     * @return a collection of [directory]'s immediate children.
+     * @throws FileNotFoundException if [directory] does not exist.
+     * @throws IOException if [directory] points to something other than directory.
+     * @throws IOException if there was an underlying error preventing listing [directory] children.
+     */
+    public fun list(directory: Path): Collection<Path>
 }
 
 internal abstract class SystemFileSystemImpl : FileSystem
diff --git a/core/common/test/files/SmokeFileTest.kt b/core/common/test/files/SmokeFileTest.kt
@@ -443,6 +443,36 @@ class SmokeFileTest {
         source.close()  // there should be no error
     }
 
+    @Test
+    fun listDirectory() {
+        assertFailsWith<FileNotFoundException> { SystemFileSystem.list(createTempPath()) }
+
+        val tmpFile = createTempPath().also {
+            SystemFileSystem.sink(it).close()
+        }
+        assertFailsWith<IOException> { SystemFileSystem.list(tmpFile) }
+
+        val dir = createTempPath().also {
+            SystemFileSystem.createDirectories(it)
+        }
+        assertEquals(emptyList(), SystemFileSystem.list(dir))
+
+        val subdir = Path(dir, "subdir").also {
+            SystemFileSystem.createDirectories(it)
+            SystemFileSystem.sink(Path(it, "file")).close()
+        }
+        assertEquals(listOf(subdir), SystemFileSystem.list(dir))
+
+        val file = Path(dir, "file").also {
+            SystemFileSystem.sink(it).close()
+        }
+        assertEquals(setOf(file, subdir), SystemFileSystem.list(dir).toSet())
+
+        SystemFileSystem.delete(file)
+        SystemFileSystem.delete(Path(subdir, "file"))
+        SystemFileSystem.delete(subdir)
+    }
+
     private fun constructAbsolutePath(vararg parts: String): String {
         return SystemPathSeparator.toString() + parts.joinToString(SystemPathSeparator.toString())
     }
diff --git a/core/jvm/src/files/FileSystemJvm.kt b/core/jvm/src/files/FileSystemJvm.kt
@@ -96,6 +96,17 @@ public actual val SystemFileSystem: FileSystem = object : SystemFileSystemImpl()
         if (!path.file.exists()) throw FileNotFoundException(path.file.absolutePath)
         return Path(path.file.canonicalFile)
     }
+
+    override fun list(directory: Path): Collection<Path> {
+        val file = directory.file
+        if (!file.exists()) throw FileNotFoundException(file.absolutePath)
+        if (!file.isDirectory) throw IOException("Not a directory: ${file.absolutePath}")
+        return buildList {
+            file.list()?.forEach { childName ->
+                add(Path(directory, childName))
+            }
+        }
+    }
 }
 
 @JvmField
diff --git a/core/native/src/files/FileSystemNative.kt b/core/native/src/files/FileSystemNative.kt
@@ -12,7 +12,7 @@ import kotlinx.io.RawSource
 import platform.posix.*
 import kotlin.experimental.ExperimentalNativeApi
 
-@OptIn(ExperimentalForeignApi::class)
+@OptIn(ExperimentalForeignApi::class, ExperimentalStdlibApi::class)
 public actual val SystemFileSystem: FileSystem = object : SystemFileSystemImpl() {
     override fun exists(path: Path): Boolean {
         return access(path.path, F_OK) == 0
@@ -86,6 +86,22 @@ public actual val SystemFileSystem: FileSystem = object : SystemFileSystemImpl()
             ?: throw IOException("Failed to open $path with ${strerror(errno)?.toKString()}")
         return FileSink(openFile)
     }
+
+    override fun list(directory: Path): Collection<Path> {
+        val metadata = metadataOrNull(directory) ?: throw FileNotFoundException(directory.path)
+        if (!metadata.isDirectory) throw IOException("Not a directory: ${directory.path}")
+        return buildList {
+            opendir(directory.path).use {
+                var child = it.readdir()
+                while (child != null) {
+                    if (child != "." && child != "..") {
+                        add(Path(directory, child))
+                    }
+                    child = it.readdir()
+                }
+            }
+        }
+    }
 }
 
 internal expect fun metadataOrNullImpl(path: Path): FileMetadata?
@@ -105,3 +121,10 @@ internal const val PermissionAllowAll: UShort = 511u
 
 @OptIn(ExperimentalNativeApi::class)
 internal actual val isWindows: Boolean = Platform.osFamily == OsFamily.WINDOWS
+
+@OptIn(ExperimentalStdlibApi::class)
+internal expect class OpaqueDirEntry : AutoCloseable {
+    fun readdir(): String?
+}
+
+internal expect fun opendir(path: String): OpaqueDirEntry
diff --git a/core/nativeNonAndroid/src/files/FileSystemNativeNonAndroid.kt b/core/nativeNonAndroid/src/files/FileSystemNativeNonAndroid.kt
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
+ */
+
+package kotlinx.io.files
+
+import kotlinx.cinterop.CPointer
+import kotlinx.cinterop.ExperimentalForeignApi
+import kotlinx.cinterop.get
+import kotlinx.cinterop.toKString
+import kotlinx.io.IOException
+import platform.posix.DIR
+import platform.posix.closedir
+import platform.posix.errno
+import platform.posix.strerror
+
+@OptIn(ExperimentalForeignApi::class, ExperimentalStdlibApi::class)
+internal actual class OpaqueDirEntry constructor(private val dir: CPointer<DIR>) : AutoCloseable {
+    actual fun readdir(): String? {
+        val entry = platform.posix.readdir(dir) ?: return null
+        return entry[0].d_name.toKString()
+    }
+
+    override fun close() {
+        closedir(dir)
+    }
+}
+
+@OptIn(ExperimentalForeignApi::class)
+internal actual fun opendir(path: String): OpaqueDirEntry {
+    val dirent = platform.posix.opendir(path)
+    if (dirent != null) return OpaqueDirEntry(dirent)
+    throw IOException("Can't open directory $path: ${strerror(errno)?.toKString() ?: "reason unknown"}")
+}
diff --git a/core/nativeNonApple/src/files/FileSystemNativeNonApple.kt b/core/nativeNonApple/src/files/FileSystemNativeNonApple.kt
diff --git a/core/nodeFilesystemShared/src/files/FileSystemNodeJs.kt b/core/nodeFilesystemShared/src/files/FileSystemNodeJs.kt
@@ -100,6 +100,23 @@ public actual val SystemFileSystem: FileSystem = object : SystemFileSystemImpl()
         if (!exists(path)) throw FileNotFoundException(path.path)
         return Path(fs.realpathSync.native(path.path))
     }
+
+    override fun list(directory: Path): Collection<Path> {
+        val metadata = metadataOrNull(directory) ?: throw FileNotFoundException(directory.path)
+        if (!metadata.isDirectory) throw IOException("Not a directory: ${directory.path}")
+        val dir = fs.opendirSync(directory.path) ?: throw IOException("Unable to read directory: ${directory.path}")
+        try {
+            return buildList {
+                var child = dir.readSync()
+                while (child != null) {
+                    add(Path(directory, child.name))
+                    child = dir.readSync()
+                }
+            }
+        } finally {
+            dir.closeSync()
+        }
+    }
 }
 
 public actual val SystemTemporaryDirectory: Path
diff --git a/core/nodeFilesystemShared/src/node/fs.kt b/core/nodeFilesystemShared/src/node/fs.kt
@@ -56,6 +56,11 @@ internal external interface Fs {
      */
     fun writeFileSync(fd: Int, buffer: Buffer)
 
+    /**
+     * See https://nodejs.org/api/fs.html#fsopendirsyncpath-options
+     */
+    fun opendirSync(path: String): Dir?
+
     val realpathSync: realpathSync
 
     val constants: constants
@@ -86,4 +91,20 @@ internal external interface realpathSync {
     fun native(path: String): String
 }
 
+/**
+ * See https://nodejs.org/api/fs.html#class-fsdir
+ */
+internal external interface Dir {
+    fun closeSync()
+
+    fun readSync(): Dirent?
+}
+
+/**
+ * See https://nodejs.org/api/fs.html#class-fsdirent
+ */
+internal external interface Dirent {
+    val name: String
+}
+
 internal expect val fs: Fs
diff --git a/core/wasmWasi/src/-WasmUtils.kt b/core/wasmWasi/src/-WasmUtils.kt
@@ -109,3 +109,14 @@ internal fun Pointer.allocateString(value: String): Int {
  */
 @UnsafeWasmMemoryApi
 internal fun MemoryAllocator.allocateInt(): Pointer = allocate(Int.SIZE_BYTES)
+
+/**
+ * Decodes zero-terminated string from a sequence of bytes that should not exceed [maxLength] bytes in length.
+ */
+@UnsafeWasmMemoryApi
+internal fun Pointer.loadString(maxLength: Int): String {
+    val bytes = loadBytes(maxLength)
+    val firstZeroByte = bytes.indexOf(0)
+    val length = if (firstZeroByte == -1) maxLength else firstZeroByte
+    return bytes.decodeToString(0, length)
+}
diff --git a/core/wasmWasi/src/files/FileSystemWasm.kt b/core/wasmWasi/src/files/FileSystemWasm.kt
diff --git a/core/wasmWasi/src/wasi/functions.kt b/core/wasmWasi/src/wasi/functions.kt
diff --git a/core/wasmWasi/test/WasiFsTest.kt b/core/wasmWasi/test/WasiFsTest.kt
