neonvm: build and pass kernel tools to neonvm runner

.github/workflows/build-images.yaml

@@ -211,6 +211,7 @@ jobs:
           docker pull --quiet $IMAGE
           ID=$(docker create $IMAGE true)
           docker cp ${ID}:/vmlinuz neonvm-kernel/vmlinuz-$ARCH
+          docker cp ${ID}:/tools.img neonvm-kernel/tools.img
           docker rm -f ${ID}
 
       - name: Get QEMU BIOS for platform

Makefile

@@ -368,8 +368,9 @@ download-qemu-bios:
 kernel: ## Build linux kernel.
 	set -eux; \
 	rm -f neonvm-kernel/vmlinuz; \
+	rm -rf neonvm-kernel/tools.img; \
 	kernel_version="$$(neonvm-kernel/echo-version.sh)"; \
-	version_suffix="-local-$$(date -u '+%FT%TZ')-$$(git describe --dirty)"; \
+	version_suffix="-$$(git describe --tags --dirty= --always)"; \
 	docker buildx build \
 		--tag neonvm-kernel:dev \
 		--build-arg UBUNTU_IMG_TAG=$(UBUNTU_IMG_TAG) \
@@ -386,6 +387,7 @@ kernel: ## Build linux kernel.
 		neonvm-kernel; \
 	id=$$(docker create neonvm-kernel:dev); \
 	docker cp $$id:/vmlinuz neonvm-kernel/vmlinuz-$(TARGET_ARCH); \
+	docker cp $$id:/tools.img neonvm-kernel/tools.img; \
 	docker rm -f $$id
 
 .PHONY: kernel-source
@@ -596,7 +598,7 @@ k3d-setup: k3d kubectl logs-dir-setup ## Create local cluster by k3d tool and pr
 		--config k3d/config.yaml \
 		--volume "$(PWD)/tests/logs:/logs@all" \
 		$(if $(USE_REGISTRIES_FILE),--registry-config=k3d/registries.yaml)
-		
+
 	$(KUBECTL) --context k3d-$(CLUSTER_NAME) apply -f k3d/cilium.yaml
 	$(KUBECTL) --context k3d-$(CLUSTER_NAME) -n kube-system rollout status daemonset  cilium
 	$(KUBECTL) --context k3d-$(CLUSTER_NAME) -n kube-system rollout status deployment cilium-operator

neonvm-kernel/.gitignore

@@ -1,4 +1,5 @@
 vmlinuz*
+tools.img
 
 # Exclude files for local kernel development.
 # Otherwise certain git operations become unexpectedly very expensive.

neonvm-kernel/Dockerfile

@@ -11,9 +11,9 @@ WORKDIR /build
 RUN set -e \
     && echo "Build linux kernel ${KERNEL_VERSION}" \
     && test -n "${KERNEL_VERSION}" \
-    && echo "force this as a requirement for build-deps" > /build/arg-check-succeeded
+    && echo "force this as a requirement for build-kernel-deps" > /build/arg-check-succeeded
 
-FROM ubuntu:$UBUNTU_IMG_TAG$UBUNTU_IMG_SHA AS build-deps
+FROM ubuntu:$UBUNTU_IMG_TAG$UBUNTU_IMG_SHA AS build-kernel-deps
 WORKDIR /build
 
 RUN apt-get update && apt-get -y install \
@@ -25,10 +25,13 @@ RUN apt-get update && apt-get -y install \
     flex \
     bison \
     libelf-dev \
+    dwarves \
+    pahole \
     bc \
     libssl-dev \
     python3 \
     cpio \
+    rsync \
     zstd \
     libncurses-dev
 
@@ -59,37 +62,110 @@ RUN set -e \
     && cd linux-${KERNEL_VERSION} \
     && for f in $(ls -1 ../patches | sort); do echo "Applying $f..."; patch -p1 < ../patches/$f; done
 
+FROM build-kernel-deps AS build-tools-deps
+RUN apt-get update && apt-get install -y \
+    clang \
+    llvm-dev \
+    libdw-dev \
+    libslang2-dev \
+    libbfd-dev \
+    libiberty-dev \
+    libcap-dev \
+    libunwind-dev \
+    zlib1g-dev \
+    libzstd-dev \
+    liblzma-dev \
+    libaio-dev \
+    libtraceevent-dev \
+    libpfm4-dev \
+    systemtap-sdt-dev \
+    musl \
+    musl-dev \
+    musl-tools \
+    pkg-config
 
 ### Cross-compilation related steps
 
 # Build the kernel on amd64
-FROM build-deps AS build_amd64
+FROM build-kernel-deps AS build_kernel_amd64
 ARG KERNEL_VERSION
 ADD linux-config-amd64-${KERNEL_VERSION} linux-${KERNEL_VERSION}/.config
 ARG VERSION_SUFFIX
 RUN cd linux-${KERNEL_VERSION} \
-	&& make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- EXTRAVERSION="$VERSION_SUFFIX" -j `nproc`
+    && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- EXTRAVERSION="$VERSION_SUFFIX" -j `nproc` \
+    && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- prepare -j `nproc` \
+    && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- headers -j `nproc` \
+    && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- modules_prepare -j `nproc` \
+    && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- modules -j `nproc` \
+    && ls -l && ls -l kernel \
+    && zstd -19 kernel/kheaders.ko -o kheaders.ko.zst \
+    && mkdir -p /build/linux-headers/build \
+    && mkdir -p /build/linux-headers/kernel/kernel \
+    && mkdir -p /build/linux-headers/build/arch/x86 \
+    && make headers_install INSTALL_HDR_PATH=/build/linux-headers/build \
+    && rsync -a arch/x86/include /build/linux-headers/build/arch/x86/ \
+    && cp kheaders.ko.zst /build/linux-headers/kernel/kernel/kheaders.ko.zst \
+    && cp -a include arch/x86/include scripts Module.symvers .config Makefile /build/linux-headers/build
+
+FROM build-tools-deps AS build_tools_amd64
+ARG KERNEL_VERSION
+ADD linux-config-amd64-${KERNEL_VERSION} linux-${KERNEL_VERSION}/.config
+RUN cd linux-${KERNEL_VERSION}/tools/perf \
+    && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- LDFLAGS="-static -L/usr/lib/x86_64-linux-gnu -Wl,--whole-archive -lzstd -Wl,--no-whole-archive" NO_LIBPERL=1 NO_GTK2=1 NO_SDT=1 NO_JEVENTS=1
 
 # Copy the kernel image to a separate step
 # Use alpine so that `cp` is available when loading custom kernels for the runner pod.
 # See the neonvm controller's pod creation logic for more detail.
 FROM --platform=linux/amd64 alpine:$ALPINE_IMG_TAG$ALPINE_IMG_SHA_AMD64 AS kernel_amd64
 ARG KERNEL_VERSION
-COPY --from=build_amd64 /build/linux-${KERNEL_VERSION}/arch/x86/boot/bzImage /vmlinuz
+ARG VERSION_SUFFIX
+COPY --from=build_kernel_amd64 /build/linux-${KERNEL_VERSION}/arch/x86/boot/bzImage /vmlinuz
+COPY --from=build_kernel_amd64 /build/linux-headers /tools/lib/modules/${KERNEL_VERSION}${VERSION_SUFFIX}
+COPY --from=build_tools_amd64 /build/linux-${KERNEL_VERSION}/tools/perf/perf /tools/bin/perf
+RUN apk update && apk add e2fsprogs && DIR_SIZE=$(du -sm /tools | cut -f1) \
+    && BUFFER_SIZE=$((DIR_SIZE + DIR_SIZE / 5)) \
+    && dd if=/dev/zero of=/tools.img bs=1M count=$BUFFER_SIZE \
+    && mkfs.ext4 -L vm-tools -d /tools /tools.img
 
 # Build the kernel on arm64
-FROM build-deps AS build_arm64
+FROM build-kernel-deps AS build_kernel_arm64
 ARG KERNEL_VERSION
 ADD linux-config-aarch64-${KERNEL_VERSION} linux-${KERNEL_VERSION}/.config
 RUN cd linux-${KERNEL_VERSION} \
-	&& make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- EXTRAVERSION="$VERSION_SUFFIX" -j `nproc`
+    && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- EXTRAVERSION="$VERSION_SUFFIX" -j `nproc` \
+    && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- prepare -j `nproc` \
+    && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- headers -j `nproc` \
+    && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- modules_prepare -j `nproc` \
+    && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- modules -j `nproc` \
+    && ls -l && ls -l kernel \
+    && zstd -19 kernel/kheaders.ko -o kheaders.ko.zst \
+    && mkdir -p /build/linux-headers/build \
+    && mkdir -p /build/linux-headers/kernel/kernel \
+    && mkdir -p /build/linux-headers/build/arch/arm64 \
+    && make headers_install INSTALL_HDR_PATH=/build/linux-headers/build \
+    && rsync -a arch/arm64/include /build/linux-headers/build/arch/arm64/ \
+    && cp kheaders.ko.zst /build/linux-headers/kernel/kernel/kheaders.ko.zst \
+    && cp -a include arch/arm64/include scripts Module.symvers .config Makefile /build/linux-headers/build
+
+FROM build-tools-deps AS build_tools_arm64
+ARG KERNEL_VERSION
+RUN cd linux-${KERNEL_VERSION}/tools/perf \
+    && make LDFLAGS="-static -L/usr/lib/aarch64-linux-gnu -Wl,--whole-archive -lzstd -Wl,--no-whole-archive" NO_LIBPERL=1 NO_GTK2=1 NO_SDT=1 NO_JEVENTS=1
 
 # Copy the kernel image to a separate step
 # Use alpine so that `cp` is available when loading custom kernels for the runner pod.
 # See the neonvm controller's pod creation logic for more detail.
 FROM --platform=linux/arm64 alpine:$ALPINE_IMG_TAG$ALPINE_IMG_SHA_ARM64 AS kernel_arm64
 ARG KERNEL_VERSION
-COPY --from=build_arm64 /build/linux-${KERNEL_VERSION}/arch/arm64/boot/Image /vmlinuz
+ARG VERSION_SUFFIX
+COPY --from=build_kernel_arm64 /build/linux-${KERNEL_VERSION}/arch/arm64/boot/Image /vmlinuz
+COPY --from=build_kernel_arm64 /build/linux-headers /tools/lib/modules/${KERNEL_VERSION}${VERSION_SUFFIX}
+COPY --from=build_tools_arm64 /build/linux-${KERNEL_VERSION}/tools/perf/perf /tools/bin/perf
+RUN apk update && apk add e2fsprogs \
+    && DIR_SIZE=$(du -sm /tools | cut -f1) \
+    && BUFFER_SIZE=$((DIR_SIZE + DIR_SIZE / 5)) \
+    && dd if=/dev/zero of=/tools.img bs=1M count=$BUFFER_SIZE \
+    && mkfs.ext4 -L vm-tools -d /tools /tools.img
 
 # Dummy default target without target architecture
 FROM ubuntu:$UBUNTU_IMG_TAG$UBUNTU_IMG_SHA

neonvm-kernel/README.md

@@ -26,7 +26,7 @@ make olddefconfig ARCH=x86_64  # or ARCH=arm64
 
 # OR
 
-# Interactively review all new config options 
+# Interactively review all new config options
 make oldconfig ARCH=x86_64  # or ARCH=arm64
 ```
 
@@ -90,3 +90,24 @@ docker build --build-arg KERNEL_VERSION=6.6.64 --platform linux/x86_64 --target
 docker run --rm -v $PWD:/host --name kernel-build -it kernel-build-deps bash
 # inside that bash shell, do the menuconfig, then copy-out the config to /host
 ```
+
+## Tools
+
+The docker image also contains the **tools** - useful stuff which can
+**ONLY** be provided at the same time as the kernel. That means, either
+those are such tools that have a direct dependency on the kernel
+version, or the tools built from the same repository as the kernel
+(like `perf`), or anything else dependent on the kernel.
+
+The `tools` is a **root** directory for this environment which provides
+them. Its structure is the same as if the internals were installed onto
+the host the usual way, so `<tools>/bin` is the path for the binaries,
+`<tools>/include` for the include files, `<tools>/lib` for the libraries
+and so on.
+
+To make it easier to deploy it inside the VM, the tools are packaged
+into an ext4 filesystem and then put into a disk image file with one
+partition with this filesystem. Later it can be simply attached to the
+environment (`qemu`, for example), or mounted to the local filesystem.
+
+The image file is called `tools.img` and is labelled `vm-tools`.

neonvm-kernel/linux-config-aarch64-6.12.26

@@ -143,8 +143,9 @@ CONFIG_RCU_STALL_COMMON=y
 CONFIG_RCU_NEED_SEGCBLIST=y
 # end of RCU Subsystem
 
-# CONFIG_IKCONFIG is not set
-# CONFIG_IKHEADERS is not set
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_IKHEADERS=m
 CONFIG_LOG_BUF_SHIFT=17
 CONFIG_LOG_CPU_MAX_BUF_SHIFT=12
 CONFIG_GENERIC_SCHED_CLOCK=y
@@ -637,7 +638,8 @@ CONFIG_CPU_MITIGATIONS=y
 #
 CONFIG_HOTPLUG_CORE_SYNC=y
 CONFIG_HOTPLUG_CORE_SYNC_DEAD=y
-# CONFIG_KPROBES is not set
+CONFIG_KPROBES=y
+CONFIG_KPROBE_EVENTS=y
 CONFIG_JUMP_LABEL=y
 # CONFIG_STATIC_KEYS_SELFTEST is not set
 CONFIG_UPROBES=y
@@ -778,7 +780,9 @@ CONFIG_MODULE_SIG_SHA512=y
 # CONFIG_MODULE_SIG_SHA3_384 is not set
 # CONFIG_MODULE_SIG_SHA3_512 is not set
 CONFIG_MODULE_SIG_HASH="sha512"
-# CONFIG_MODULE_COMPRESS is not set
+CONFIG_MODULE_COMPRESS=y
+CONFIG_MODULE_COMPRESS_GZIP=n
+CONFIG_MODULE_COMPRESS_ZSTD=y
 # CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS is not set
 CONFIG_MODPROBE_PATH="/sbin/modprobe"
 # CONFIG_TRIM_UNUSED_KSYMS is not set
@@ -3410,6 +3414,7 @@ CONFIG_DEBUG_MISC=y
 # Compile-time checks and compiler options
 #
 CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_BTF=y
 CONFIG_AS_HAS_NON_CONST_ULEB128=y
 # CONFIG_DEBUG_INFO_NONE is not set
 CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
@@ -3438,7 +3443,7 @@ CONFIG_FRAME_POINTER=y
 # Generic Kernel Debugging Instruments
 #
 # CONFIG_MAGIC_SYSRQ is not set
-# CONFIG_DEBUG_FS is not set
+CONFIG_DEBUG_FS=y
 CONFIG_HAVE_ARCH_KGDB=y
 # CONFIG_KGDB is not set
 CONFIG_ARCH_HAS_UBSAN=y
@@ -3591,7 +3596,7 @@ CONFIG_TRACING=y
 CONFIG_TRACING_SUPPORT=y
 CONFIG_FTRACE=y
 CONFIG_BOOTTIME_TRACING=y
-# CONFIG_FUNCTION_TRACER is not set
+CONFIG_FUNCTION_TRACER=y
 # CONFIG_STACK_TRACER is not set
 # CONFIG_IRQSOFF_TRACER is not set
 # CONFIG_PREEMPT_TRACER is not set
@@ -3600,7 +3605,7 @@ CONFIG_BOOTTIME_TRACING=y
 # CONFIG_OSNOISE_TRACER is not set
 # CONFIG_TIMERLAT_TRACER is not set
 # CONFIG_ENABLE_DEFAULT_TRACERS is not set
-# CONFIG_FTRACE_SYSCALLS is not set
+CONFIG_FTRACE_SYSCALLS=y
 # CONFIG_TRACER_SNAPSHOT is not set
 CONFIG_BRANCH_PROFILE_NONE=y
 # CONFIG_PROFILE_ANNOTATED_BRANCHES is not set

neonvm-kernel/linux-config-amd64-6.12.26

@@ -163,8 +163,9 @@ CONFIG_RCU_STALL_COMMON=y
 CONFIG_RCU_NEED_SEGCBLIST=y
 # end of RCU Subsystem
 
-# CONFIG_IKCONFIG is not set
-# CONFIG_IKHEADERS is not set
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_IKHEADERS=m
 CONFIG_LOG_BUF_SHIFT=17
 CONFIG_LOG_CPU_MAX_BUF_SHIFT=12
 CONFIG_HAVE_UNSTABLE_SCHED_CLOCK=y
@@ -440,7 +441,7 @@ CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT=1
 CONFIG_X86_PAT=y
 CONFIG_X86_UMIP=y
 CONFIG_CC_HAS_IBT=y
-# CONFIG_X86_KERNEL_IBT is not set
+CONFIG_X86_KERNEL_IBT=y
 # CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS is not set
 CONFIG_ARCH_PKEY_BITS=4
 CONFIG_X86_INTEL_TSX_MODE_OFF=y
@@ -689,7 +690,8 @@ CONFIG_HOTPLUG_CORE_SYNC_FULL=y
 CONFIG_HOTPLUG_SPLIT_STARTUP=y
 CONFIG_HOTPLUG_PARALLEL=y
 CONFIG_GENERIC_ENTRY=y
-# CONFIG_KPROBES is not set
+CONFIG_KPROBES=y
+CONFIG_KPROBE_EVENTS=y
 # CONFIG_JUMP_LABEL is not set
 # CONFIG_STATIC_CALL_SELFTEST is not set
 CONFIG_UPROBES=y
@@ -855,7 +857,9 @@ CONFIG_MODULE_SIG_SHA512=y
 # CONFIG_MODULE_SIG_SHA3_384 is not set
 # CONFIG_MODULE_SIG_SHA3_512 is not set
 CONFIG_MODULE_SIG_HASH="sha512"
-# CONFIG_MODULE_COMPRESS is not set
+CONFIG_MODULE_COMPRESS=y
+CONFIG_MODULE_COMPRESS_GZIP=n
+CONFIG_MODULE_COMPRESS_ZSTD=y
 # CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS is not set
 CONFIG_MODPROBE_PATH="/sbin/modprobe"
 # CONFIG_TRIM_UNUSED_KSYMS is not set
@@ -3357,6 +3361,7 @@ CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
 # CONFIG_DEBUG_INFO_COMPRESSED_ZLIB is not set
 # CONFIG_DEBUG_INFO_COMPRESSED_ZSTD is not set
 # CONFIG_DEBUG_INFO_SPLIT is not set
+CONFIG_DEBUG_INFO_BTF=y
 CONFIG_GDB_SCRIPTS=y
 CONFIG_FRAME_WARN=2048
 # CONFIG_STRIP_ASM_SYMS is not set
@@ -3377,7 +3382,7 @@ CONFIG_STACK_VALIDATION=y
 # Generic Kernel Debugging Instruments
 #
 # CONFIG_MAGIC_SYSRQ is not set
-# CONFIG_DEBUG_FS is not set
+CONFIG_DEBUG_FS=y
 CONFIG_HAVE_ARCH_KGDB=y
 # CONFIG_KGDB is not set
 CONFIG_ARCH_HAS_UBSAN=y
@@ -3540,7 +3545,7 @@ CONFIG_TRACING=y
 CONFIG_TRACING_SUPPORT=y
 CONFIG_FTRACE=y
 CONFIG_BOOTTIME_TRACING=y
-# CONFIG_FUNCTION_TRACER is not set
+CONFIG_FUNCTION_TRACER=y
 # CONFIG_STACK_TRACER is not set
 # CONFIG_IRQSOFF_TRACER is not set
 # CONFIG_PREEMPT_TRACER is not set
@@ -3550,7 +3555,7 @@ CONFIG_BOOTTIME_TRACING=y
 # CONFIG_TIMERLAT_TRACER is not set
 # CONFIG_MMIOTRACE is not set
 # CONFIG_ENABLE_DEFAULT_TRACERS is not set
-# CONFIG_FTRACE_SYSCALLS is not set
+CONFIG_FTRACE_SYSCALLS=y
 # CONFIG_TRACER_SNAPSHOT is not set
 CONFIG_BRANCH_PROFILE_NONE=y
 # CONFIG_PROFILE_ANNOTATED_BRANCHES is not set

neonvm-runner/Dockerfile

@@ -31,6 +31,7 @@ RUN apk add --no-cache \
     tcpdump \
     tini
 
+COPY neonvm-kernel/tools.img /vm/tools.img
 COPY neonvm-runner/ssh_config /etc/ssh/ssh_config
 
 ARG TARGET_ARCH