diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 2405da819..4e8da5c11 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -211,6 +211,7 @@ jobs: docker pull --quiet $IMAGE ID=$(docker create $IMAGE true) docker cp ${ID}:/vmlinuz neonvm-kernel/vmlinuz-$ARCH + docker cp ${ID}:/tools.img neonvm-kernel/tools.img docker rm -f ${ID} - name: Get QEMU BIOS for platform diff --git a/Makefile b/Makefile index b5f871300..b8a557ac5 100644 --- a/Makefile +++ b/Makefile @@ -368,8 +368,9 @@ download-qemu-bios: kernel: ## Build linux kernel. set -eux; \ rm -f neonvm-kernel/vmlinuz; \ + rm -rf neonvm-kernel/tools.img; \ kernel_version="$$(neonvm-kernel/echo-version.sh)"; \ - version_suffix="-local-$$(date -u '+%FT%TZ')-$$(git describe --dirty)"; \ + version_suffix="-$$(git describe --tags --dirty= --always)"; \ docker buildx build \ --tag neonvm-kernel:dev \ --build-arg UBUNTU_IMG_TAG=$(UBUNTU_IMG_TAG) \ @@ -386,6 +387,7 @@ kernel: ## Build linux kernel. neonvm-kernel; \ id=$$(docker create neonvm-kernel:dev); \ docker cp $$id:/vmlinuz neonvm-kernel/vmlinuz-$(TARGET_ARCH); \ + docker cp $$id:/tools.img neonvm-kernel/tools.img; \ docker rm -f $$id .PHONY: kernel-source @@ -596,7 +598,7 @@ k3d-setup: k3d kubectl logs-dir-setup ## Create local cluster by k3d tool and pr --config k3d/config.yaml \ --volume "$(PWD)/tests/logs:/logs@all" \ $(if $(USE_REGISTRIES_FILE),--registry-config=k3d/registries.yaml) - + $(KUBECTL) --context k3d-$(CLUSTER_NAME) apply -f k3d/cilium.yaml $(KUBECTL) --context k3d-$(CLUSTER_NAME) -n kube-system rollout status daemonset cilium $(KUBECTL) --context k3d-$(CLUSTER_NAME) -n kube-system rollout status deployment cilium-operator diff --git a/neonvm-kernel/.gitignore b/neonvm-kernel/.gitignore index bfbb16333..9183704c8 100644 --- a/neonvm-kernel/.gitignore +++ b/neonvm-kernel/.gitignore @@ -1,4 +1,5 @@ vmlinuz* +tools.img # Exclude files for local kernel development. # Otherwise certain git operations become unexpectedly very expensive. diff --git a/neonvm-kernel/Dockerfile b/neonvm-kernel/Dockerfile index ad96c6572..32a1ad427 100644 --- a/neonvm-kernel/Dockerfile +++ b/neonvm-kernel/Dockerfile @@ -11,9 +11,9 @@ WORKDIR /build RUN set -e \ && echo "Build linux kernel ${KERNEL_VERSION}" \ && test -n "${KERNEL_VERSION}" \ - && echo "force this as a requirement for build-deps" > /build/arg-check-succeeded + && echo "force this as a requirement for build-kernel-deps" > /build/arg-check-succeeded -FROM ubuntu:$UBUNTU_IMG_TAG$UBUNTU_IMG_SHA AS build-deps +FROM ubuntu:$UBUNTU_IMG_TAG$UBUNTU_IMG_SHA AS build-kernel-deps WORKDIR /build RUN apt-get update && apt-get -y install \ @@ -25,10 +25,13 @@ RUN apt-get update && apt-get -y install \ flex \ bison \ libelf-dev \ + dwarves \ + pahole \ bc \ libssl-dev \ python3 \ cpio \ + rsync \ zstd \ libncurses-dev @@ -59,37 +62,110 @@ RUN set -e \ && cd linux-${KERNEL_VERSION} \ && for f in $(ls -1 ../patches | sort); do echo "Applying $f..."; patch -p1 < ../patches/$f; done +FROM build-kernel-deps AS build-tools-deps +RUN apt-get update && apt-get install -y \ + clang \ + llvm-dev \ + libdw-dev \ + libslang2-dev \ + libbfd-dev \ + libiberty-dev \ + libcap-dev \ + libunwind-dev \ + zlib1g-dev \ + libzstd-dev \ + liblzma-dev \ + libaio-dev \ + libtraceevent-dev \ + libpfm4-dev \ + systemtap-sdt-dev \ + musl \ + musl-dev \ + musl-tools \ + pkg-config ### Cross-compilation related steps # Build the kernel on amd64 -FROM build-deps AS build_amd64 +FROM build-kernel-deps AS build_kernel_amd64 ARG KERNEL_VERSION ADD linux-config-amd64-${KERNEL_VERSION} linux-${KERNEL_VERSION}/.config ARG VERSION_SUFFIX RUN cd linux-${KERNEL_VERSION} \ - && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- EXTRAVERSION="$VERSION_SUFFIX" -j `nproc` + && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- EXTRAVERSION="$VERSION_SUFFIX" -j `nproc` \ + && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- prepare -j `nproc` \ + && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- headers -j `nproc` \ + && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- modules_prepare -j `nproc` \ + && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- modules -j `nproc` \ + && ls -l && ls -l kernel \ + && zstd -19 kernel/kheaders.ko -o kheaders.ko.zst \ + && mkdir -p /build/linux-headers/build \ + && mkdir -p /build/linux-headers/kernel/kernel \ + && mkdir -p /build/linux-headers/build/arch/x86 \ + && make headers_install INSTALL_HDR_PATH=/build/linux-headers/build \ + && rsync -a arch/x86/include /build/linux-headers/build/arch/x86/ \ + && cp kheaders.ko.zst /build/linux-headers/kernel/kernel/kheaders.ko.zst \ + && cp -a include arch/x86/include scripts Module.symvers .config Makefile /build/linux-headers/build + +FROM build-tools-deps AS build_tools_amd64 +ARG KERNEL_VERSION +ADD linux-config-amd64-${KERNEL_VERSION} linux-${KERNEL_VERSION}/.config +RUN cd linux-${KERNEL_VERSION}/tools/perf \ + && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- LDFLAGS="-static -L/usr/lib/x86_64-linux-gnu -Wl,--whole-archive -lzstd -Wl,--no-whole-archive" NO_LIBPERL=1 NO_GTK2=1 NO_SDT=1 NO_JEVENTS=1 # Copy the kernel image to a separate step # Use alpine so that `cp` is available when loading custom kernels for the runner pod. # See the neonvm controller's pod creation logic for more detail. FROM --platform=linux/amd64 alpine:$ALPINE_IMG_TAG$ALPINE_IMG_SHA_AMD64 AS kernel_amd64 ARG KERNEL_VERSION -COPY --from=build_amd64 /build/linux-${KERNEL_VERSION}/arch/x86/boot/bzImage /vmlinuz +ARG VERSION_SUFFIX +COPY --from=build_kernel_amd64 /build/linux-${KERNEL_VERSION}/arch/x86/boot/bzImage /vmlinuz +COPY --from=build_kernel_amd64 /build/linux-headers /tools/lib/modules/${KERNEL_VERSION}${VERSION_SUFFIX} +COPY --from=build_tools_amd64 /build/linux-${KERNEL_VERSION}/tools/perf/perf /tools/bin/perf +RUN apk update && apk add e2fsprogs && DIR_SIZE=$(du -sm /tools | cut -f1) \ + && BUFFER_SIZE=$((DIR_SIZE + DIR_SIZE / 5)) \ + && dd if=/dev/zero of=/tools.img bs=1M count=$BUFFER_SIZE \ + && mkfs.ext4 -L vm-tools -d /tools /tools.img # Build the kernel on arm64 -FROM build-deps AS build_arm64 +FROM build-kernel-deps AS build_kernel_arm64 ARG KERNEL_VERSION ADD linux-config-aarch64-${KERNEL_VERSION} linux-${KERNEL_VERSION}/.config RUN cd linux-${KERNEL_VERSION} \ - && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- EXTRAVERSION="$VERSION_SUFFIX" -j `nproc` + && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- EXTRAVERSION="$VERSION_SUFFIX" -j `nproc` \ + && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- prepare -j `nproc` \ + && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- headers -j `nproc` \ + && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- modules_prepare -j `nproc` \ + && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- modules -j `nproc` \ + && ls -l && ls -l kernel \ + && zstd -19 kernel/kheaders.ko -o kheaders.ko.zst \ + && mkdir -p /build/linux-headers/build \ + && mkdir -p /build/linux-headers/kernel/kernel \ + && mkdir -p /build/linux-headers/build/arch/arm64 \ + && make headers_install INSTALL_HDR_PATH=/build/linux-headers/build \ + && rsync -a arch/arm64/include /build/linux-headers/build/arch/arm64/ \ + && cp kheaders.ko.zst /build/linux-headers/kernel/kernel/kheaders.ko.zst \ + && cp -a include arch/arm64/include scripts Module.symvers .config Makefile /build/linux-headers/build + +FROM build-tools-deps AS build_tools_arm64 +ARG KERNEL_VERSION +RUN cd linux-${KERNEL_VERSION}/tools/perf \ + && make LDFLAGS="-static -L/usr/lib/aarch64-linux-gnu -Wl,--whole-archive -lzstd -Wl,--no-whole-archive" NO_LIBPERL=1 NO_GTK2=1 NO_SDT=1 NO_JEVENTS=1 # Copy the kernel image to a separate step # Use alpine so that `cp` is available when loading custom kernels for the runner pod. # See the neonvm controller's pod creation logic for more detail. FROM --platform=linux/arm64 alpine:$ALPINE_IMG_TAG$ALPINE_IMG_SHA_ARM64 AS kernel_arm64 ARG KERNEL_VERSION -COPY --from=build_arm64 /build/linux-${KERNEL_VERSION}/arch/arm64/boot/Image /vmlinuz +ARG VERSION_SUFFIX +COPY --from=build_kernel_arm64 /build/linux-${KERNEL_VERSION}/arch/arm64/boot/Image /vmlinuz +COPY --from=build_kernel_arm64 /build/linux-headers /tools/lib/modules/${KERNEL_VERSION}${VERSION_SUFFIX} +COPY --from=build_tools_arm64 /build/linux-${KERNEL_VERSION}/tools/perf/perf /tools/bin/perf +RUN apk update && apk add e2fsprogs \ + && DIR_SIZE=$(du -sm /tools | cut -f1) \ + && BUFFER_SIZE=$((DIR_SIZE + DIR_SIZE / 5)) \ + && dd if=/dev/zero of=/tools.img bs=1M count=$BUFFER_SIZE \ + && mkfs.ext4 -L vm-tools -d /tools /tools.img # Dummy default target without target architecture FROM ubuntu:$UBUNTU_IMG_TAG$UBUNTU_IMG_SHA diff --git a/neonvm-kernel/README.md b/neonvm-kernel/README.md index e0c1b6cd8..dd1d0a18f 100644 --- a/neonvm-kernel/README.md +++ b/neonvm-kernel/README.md @@ -26,7 +26,7 @@ make olddefconfig ARCH=x86_64 # or ARCH=arm64 # OR -# Interactively review all new config options +# Interactively review all new config options make oldconfig ARCH=x86_64 # or ARCH=arm64 ``` @@ -90,3 +90,24 @@ docker build --build-arg KERNEL_VERSION=6.6.64 --platform linux/x86_64 --target docker run --rm -v $PWD:/host --name kernel-build -it kernel-build-deps bash # inside that bash shell, do the menuconfig, then copy-out the config to /host ``` + +## Tools + +The docker image also contains the **tools** - useful stuff which can +**ONLY** be provided at the same time as the kernel. That means, either +those are such tools that have a direct dependency on the kernel +version, or the tools built from the same repository as the kernel +(like `perf`), or anything else dependent on the kernel. + +The `tools` is a **root** directory for this environment which provides +them. Its structure is the same as if the internals were installed onto +the host the usual way, so `/bin` is the path for the binaries, +`/include` for the include files, `/lib` for the libraries +and so on. + +To make it easier to deploy it inside the VM, the tools are packaged +into an ext4 filesystem and then put into a disk image file with one +partition with this filesystem. Later it can be simply attached to the +environment (`qemu`, for example), or mounted to the local filesystem. + +The image file is called `tools.img` and is labelled `vm-tools`. diff --git a/neonvm-kernel/linux-config-aarch64-6.12.26 b/neonvm-kernel/linux-config-aarch64-6.12.26 index 119abd071..545e77622 100644 --- a/neonvm-kernel/linux-config-aarch64-6.12.26 +++ b/neonvm-kernel/linux-config-aarch64-6.12.26 @@ -143,8 +143,9 @@ CONFIG_RCU_STALL_COMMON=y CONFIG_RCU_NEED_SEGCBLIST=y # end of RCU Subsystem -# CONFIG_IKCONFIG is not set -# CONFIG_IKHEADERS is not set +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_IKHEADERS=m CONFIG_LOG_BUF_SHIFT=17 CONFIG_LOG_CPU_MAX_BUF_SHIFT=12 CONFIG_GENERIC_SCHED_CLOCK=y @@ -637,7 +638,8 @@ CONFIG_CPU_MITIGATIONS=y # CONFIG_HOTPLUG_CORE_SYNC=y CONFIG_HOTPLUG_CORE_SYNC_DEAD=y -# CONFIG_KPROBES is not set +CONFIG_KPROBES=y +CONFIG_KPROBE_EVENTS=y CONFIG_JUMP_LABEL=y # CONFIG_STATIC_KEYS_SELFTEST is not set CONFIG_UPROBES=y @@ -778,7 +780,9 @@ CONFIG_MODULE_SIG_SHA512=y # CONFIG_MODULE_SIG_SHA3_384 is not set # CONFIG_MODULE_SIG_SHA3_512 is not set CONFIG_MODULE_SIG_HASH="sha512" -# CONFIG_MODULE_COMPRESS is not set +CONFIG_MODULE_COMPRESS=y +CONFIG_MODULE_COMPRESS_GZIP=n +CONFIG_MODULE_COMPRESS_ZSTD=y # CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS is not set CONFIG_MODPROBE_PATH="/sbin/modprobe" # CONFIG_TRIM_UNUSED_KSYMS is not set @@ -3410,6 +3414,7 @@ CONFIG_DEBUG_MISC=y # Compile-time checks and compiler options # CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_BTF=y CONFIG_AS_HAS_NON_CONST_ULEB128=y # CONFIG_DEBUG_INFO_NONE is not set CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y @@ -3438,7 +3443,7 @@ CONFIG_FRAME_POINTER=y # Generic Kernel Debugging Instruments # # CONFIG_MAGIC_SYSRQ is not set -# CONFIG_DEBUG_FS is not set +CONFIG_DEBUG_FS=y CONFIG_HAVE_ARCH_KGDB=y # CONFIG_KGDB is not set CONFIG_ARCH_HAS_UBSAN=y @@ -3591,7 +3596,7 @@ CONFIG_TRACING=y CONFIG_TRACING_SUPPORT=y CONFIG_FTRACE=y CONFIG_BOOTTIME_TRACING=y -# CONFIG_FUNCTION_TRACER is not set +CONFIG_FUNCTION_TRACER=y # CONFIG_STACK_TRACER is not set # CONFIG_IRQSOFF_TRACER is not set # CONFIG_PREEMPT_TRACER is not set @@ -3600,7 +3605,7 @@ CONFIG_BOOTTIME_TRACING=y # CONFIG_OSNOISE_TRACER is not set # CONFIG_TIMERLAT_TRACER is not set # CONFIG_ENABLE_DEFAULT_TRACERS is not set -# CONFIG_FTRACE_SYSCALLS is not set +CONFIG_FTRACE_SYSCALLS=y # CONFIG_TRACER_SNAPSHOT is not set CONFIG_BRANCH_PROFILE_NONE=y # CONFIG_PROFILE_ANNOTATED_BRANCHES is not set diff --git a/neonvm-kernel/linux-config-amd64-6.12.26 b/neonvm-kernel/linux-config-amd64-6.12.26 index 703f5d894..e9fca3527 100644 --- a/neonvm-kernel/linux-config-amd64-6.12.26 +++ b/neonvm-kernel/linux-config-amd64-6.12.26 @@ -163,8 +163,9 @@ CONFIG_RCU_STALL_COMMON=y CONFIG_RCU_NEED_SEGCBLIST=y # end of RCU Subsystem -# CONFIG_IKCONFIG is not set -# CONFIG_IKHEADERS is not set +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_IKHEADERS=m CONFIG_LOG_BUF_SHIFT=17 CONFIG_LOG_CPU_MAX_BUF_SHIFT=12 CONFIG_HAVE_UNSTABLE_SCHED_CLOCK=y @@ -440,7 +441,7 @@ CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT=1 CONFIG_X86_PAT=y CONFIG_X86_UMIP=y CONFIG_CC_HAS_IBT=y -# CONFIG_X86_KERNEL_IBT is not set +CONFIG_X86_KERNEL_IBT=y # CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS is not set CONFIG_ARCH_PKEY_BITS=4 CONFIG_X86_INTEL_TSX_MODE_OFF=y @@ -689,7 +690,8 @@ CONFIG_HOTPLUG_CORE_SYNC_FULL=y CONFIG_HOTPLUG_SPLIT_STARTUP=y CONFIG_HOTPLUG_PARALLEL=y CONFIG_GENERIC_ENTRY=y -# CONFIG_KPROBES is not set +CONFIG_KPROBES=y +CONFIG_KPROBE_EVENTS=y # CONFIG_JUMP_LABEL is not set # CONFIG_STATIC_CALL_SELFTEST is not set CONFIG_UPROBES=y @@ -855,7 +857,9 @@ CONFIG_MODULE_SIG_SHA512=y # CONFIG_MODULE_SIG_SHA3_384 is not set # CONFIG_MODULE_SIG_SHA3_512 is not set CONFIG_MODULE_SIG_HASH="sha512" -# CONFIG_MODULE_COMPRESS is not set +CONFIG_MODULE_COMPRESS=y +CONFIG_MODULE_COMPRESS_GZIP=n +CONFIG_MODULE_COMPRESS_ZSTD=y # CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS is not set CONFIG_MODPROBE_PATH="/sbin/modprobe" # CONFIG_TRIM_UNUSED_KSYMS is not set @@ -3357,6 +3361,7 @@ CONFIG_DEBUG_INFO_COMPRESSED_NONE=y # CONFIG_DEBUG_INFO_COMPRESSED_ZLIB is not set # CONFIG_DEBUG_INFO_COMPRESSED_ZSTD is not set # CONFIG_DEBUG_INFO_SPLIT is not set +CONFIG_DEBUG_INFO_BTF=y CONFIG_GDB_SCRIPTS=y CONFIG_FRAME_WARN=2048 # CONFIG_STRIP_ASM_SYMS is not set @@ -3377,7 +3382,7 @@ CONFIG_STACK_VALIDATION=y # Generic Kernel Debugging Instruments # # CONFIG_MAGIC_SYSRQ is not set -# CONFIG_DEBUG_FS is not set +CONFIG_DEBUG_FS=y CONFIG_HAVE_ARCH_KGDB=y # CONFIG_KGDB is not set CONFIG_ARCH_HAS_UBSAN=y @@ -3540,7 +3545,7 @@ CONFIG_TRACING=y CONFIG_TRACING_SUPPORT=y CONFIG_FTRACE=y CONFIG_BOOTTIME_TRACING=y -# CONFIG_FUNCTION_TRACER is not set +CONFIG_FUNCTION_TRACER=y # CONFIG_STACK_TRACER is not set # CONFIG_IRQSOFF_TRACER is not set # CONFIG_PREEMPT_TRACER is not set @@ -3550,7 +3555,7 @@ CONFIG_BOOTTIME_TRACING=y # CONFIG_TIMERLAT_TRACER is not set # CONFIG_MMIOTRACE is not set # CONFIG_ENABLE_DEFAULT_TRACERS is not set -# CONFIG_FTRACE_SYSCALLS is not set +CONFIG_FTRACE_SYSCALLS=y # CONFIG_TRACER_SNAPSHOT is not set CONFIG_BRANCH_PROFILE_NONE=y # CONFIG_PROFILE_ANNOTATED_BRANCHES is not set diff --git a/neonvm-runner/Dockerfile b/neonvm-runner/Dockerfile index fff36ef44..f344fdf24 100644 --- a/neonvm-runner/Dockerfile +++ b/neonvm-runner/Dockerfile @@ -31,6 +31,7 @@ RUN apk add --no-cache \ tcpdump \ tini +COPY neonvm-kernel/tools.img /vm/tools.img COPY neonvm-runner/ssh_config /etc/ssh/ssh_config ARG TARGET_ARCH diff --git a/neonvm-runner/cmd/disks.go b/neonvm-runner/cmd/disks.go index 7469b44f4..b7b15cbd7 100644 --- a/neonvm-runner/cmd/disks.go +++ b/neonvm-runner/cmd/disks.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "math" "os" "os/exec" @@ -25,6 +26,8 @@ const ( runtimeDiskPath = "/vm/images/runtime.iso" mountedDiskPath = "/vm/images" + toolsDiskPath = "/vm/tools.img" + sshAuthorizedKeysDiskPath = "/vm/images/ssh-authorized-keys.iso" sshAuthorizedKeysMountPoint = "/vm/ssh" @@ -34,16 +37,21 @@ const ( // setupVMDisks creates the disks for the VM and returns the appropriate QEMU args func setupVMDisks( logger *zap.Logger, - diskCacheSettings string, + cfg *Config, enableSSH bool, swapSize *resource.Quantity, extraDisks []vmv1.Disk, ) ([]string, error) { var qemuCmd []string - qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=rootdisk,file=%s,if=virtio,media=disk,index=0,%s", rootDiskPath, diskCacheSettings)) + qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=rootdisk,file=%s,if=virtio,media=disk,index=0,%s", rootDiskPath, cfg.diskCacheSettings)) qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=runtime,file=%s,if=virtio,media=cdrom,readonly=on,cache=none", runtimeDiskPath)) + { + name := "vm-tools" + qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=%s,file=%s,if=virtio,format=raw,media=disk,readonly=on", name, toolsDiskPath)) + } + if enableSSH { name := "ssh-authorized-keys" if err := createISO9660FromPath(logger, name, sshAuthorizedKeysDiskPath, sshAuthorizedKeysMountPoint); err != nil { @@ -58,7 +66,7 @@ func setupVMDisks( if err := createSwap(dPath, swapSize); err != nil { return nil, fmt.Errorf("failed to create swap disk: %w", err) } - qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=%s,file=%s,if=virtio,media=disk,%s,discard=unmap", swapName, dPath, diskCacheSettings)) + qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=%s,file=%s,if=virtio,media=disk,%s,discard=unmap", swapName, dPath, cfg.diskCacheSettings)) } for _, disk := range extraDisks { @@ -73,7 +81,7 @@ func setupVMDisks( if disk.EmptyDisk.Discard { discard = ",discard=unmap" } - qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=%s,file=%s,if=virtio,media=disk,%s%s", disk.Name, dPath, diskCacheSettings, discard)) + qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=%s,file=%s,if=virtio,media=disk,%s%s", disk.Name, dPath, cfg.diskCacheSettings, discard)) case disk.ConfigMap != nil || disk.Secret != nil: dPath := fmt.Sprintf("%s/%s.iso", mountedDiskPath, disk.Name) mnt := fmt.Sprintf("/vm/mounts%s", disk.MountPath) @@ -175,13 +183,17 @@ func createISO9660runtime( } if enableSSH { mounts = append(mounts, "/neonvm/bin/mkdir -p /mnt/ssh") - mounts = append(mounts, "/neonvm/bin/mount -t iso9660 -o ro,mode=0644 $(/neonvm/bin/blkid -L ssh-authorized-keys) /mnt/ssh") + mounts = append(mounts, "/neonvm/bin/mount -t iso9660 -o ro,mode=0644 $(/neonvm/bin/blkid -L ssh-authorized-keys) /mnt/ssh") } if swapSize != nil { mounts = append(mounts, fmt.Sprintf("/neonvm/bin/sh /neonvm/runtime/resize-swap-internal.sh %d", swapSize.Value())) } + // Add tools. + mounts = append(mounts, "/neonvm/bin/mkdir -p /neonvm/tools") + mounts = append(mounts, "/neonvm/bin/mount -o ro,exec $(/neonvm/bin/blkid -L vm-tools) /neonvm/tools") + if len(disks) != 0 { for _, disk := range disks { if disk.MountPath != "" { @@ -278,7 +290,7 @@ func createISO9660runtime( return nil } -func calcDirUsage(dirPath string) (int64, error) { +func calcDirSize(dirPath string) (int64, error) { stat, err := os.Lstat(dirPath) if err != nil { return 0, err @@ -305,7 +317,7 @@ func calcDirUsage(dirPath string) (int64, error) { if file.Name() == "." || file.Name() == ".." { continue } - s, err := calcDirUsage(dirPath + "/" + file.Name()) + s, err := calcDirSize(dirPath + "/" + file.Name()) if err != nil { return size, err } @@ -348,7 +360,7 @@ func createQCOW2(diskName string, diskPath string, diskSize *resource.Quantity, if diskSize != nil { ext4blockCount = diskSize.Value() / ext4blockSize } else if contentPath != nil { - dirSize, err := calcDirUsage(*contentPath) + dirSize, err := calcDirSize(*contentPath) if err != nil { return err } @@ -403,44 +415,48 @@ func createISO9660FromPath(logger *zap.Logger, diskName string, diskPath string, } defer writer.Cleanup() //nolint:errcheck // Nothing to do with the error, maybe log it ? TODO - dir, err := os.Open(contentPath) - if err != nil { - return err - } - dirEntrys, err := dir.ReadDir(0) - if err != nil { - return err - } - - for _, file := range dirEntrys { - fileName := fmt.Sprintf("%s/%s", contentPath, file.Name()) - outputPath := file.Name() + err = filepath.WalkDir(contentPath, func(filePath string, d fs.DirEntry, err error) error { + if err != nil { + logger.Info("Couldn't walk through dir", zap.String("path", filePath), zap.Error(err)) + return err + } - if file.IsDir() { - continue + if d.IsDir() { + // Skip directories, we only want to add files. + logger.Info("Skipping subdir", zap.String("path", filePath)) + return nil } + // try to resolve symlink and check resolved file IsDir - resolved, err := filepath.EvalSymlinks(fileName) + resolved, err := filepath.EvalSymlinks(filePath) if err != nil { - return err + logger.Info("Couldn't eval symlinks", zap.String("path", filePath), zap.Error(err)) + return nil } - resolvedOpen, err := os.Open(resolved) + + resolvedStat, err := os.Stat(resolved) if err != nil { + logger.Info("Couldn't stat", zap.String("path", filePath), zap.String("resolvedPath", resolved), zap.Error(err)) return err } - resolvedStat, err := resolvedOpen.Stat() + if resolvedStat.IsDir() { + logger.Info("Skipping resolved directory", zap.String("path", filePath), zap.String("resolvedPath", resolved)) + return nil + } + + // Calculate the relative path for the ISO (preserving directory structure) + relPath, err := filepath.Rel(contentPath, filePath) if err != nil { + logger.Info("Failed to get relative path", zap.String("filePath", filePath), zap.Error(err)) return err } - if resolvedStat.IsDir() { - continue - } + outputPath := relPath // Use relative path to maintain directory structure in ISO // run the file handling logic in a closure, so the defers happen within the loop body, // rather than the outer function. err = func() error { - logger.Info("adding file to ISO9660 disk", zap.String("path", outputPath)) - fileToAdd, err := os.Open(fileName) + logger.Info("adding file to ISO9660 disk", zap.String("diskname", diskName), zap.String("path", outputPath)) + fileToAdd, err := os.Open(filePath) if err != nil { return err } @@ -451,6 +467,11 @@ func createISO9660FromPath(logger *zap.Logger, diskName string, diskPath string, if err != nil { return err } + + return nil + }) + if err != nil { + return err } outputFile, err := os.OpenFile(diskPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0o644) diff --git a/neonvm-runner/cmd/main.go b/neonvm-runner/cmd/main.go index 06aebf674..218ffe2d9 100644 --- a/neonvm-runner/cmd/main.go +++ b/neonvm-runner/cmd/main.go @@ -321,7 +321,7 @@ func buildQEMUCmd( "-device", "virtserialport,chardev=log,name=tech.neon.log.0", } - qemuDiskArgs, err := setupVMDisks(logger, cfg.diskCacheSettings, enableSSH, swapSize, vmSpec.Disks) + qemuDiskArgs, err := setupVMDisks(logger, cfg, enableSSH, swapSize, vmSpec.Disks) if err != nil { return nil, err }