add dax support

Signed-off-by: Lai Jiangshan <[email protected]>

Author: laijs

api/descriptions.pb.go

@@ -86,6 +86,7 @@ message VolumeOption {
     string keyring = 3;
     int32 bytesPerSec = 4;
     int32 iops = 5;
+    bool daxBlock = 6;
 }
 
 message UserGroupInfo {
@@ -110,4 +111,4 @@ message Process {
     repeated string Args = 7;
     repeated string Envs = 8;
     string Workdir = 9;
-}
+}

api/descriptions.proto

@@ -3,7 +3,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3.0."* ]]; then
+if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3."* ]]; then
   echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and"
   echo "install the platform appropriate Protobuf package for your OS: "
   echo

hack/update-generated-proto.sh

@@ -18,6 +18,7 @@ import (
 type VmHwStatus struct {
 	PciAddr  int    //next available pci addr for pci hotplug
 	ScsiId   int    //next available scsi id for scsi hotplug
+	PmemId   int    //next available pmem id for nvdimm hotplug
 	AttachId uint64 //next available attachId for attached tty
 	GuestCid uint32 //vsock guest cid
 }
@@ -50,6 +51,7 @@ type VmContext struct {
 
 	pciAddr int //next available pci addr for pci hotplug
 	scsiId  int //next available scsi id for scsi hotplug
+	pmemId  int //next available pmem id for nvdimm hotplug
 
 	//	InterfaceCount int
 
@@ -185,6 +187,14 @@ func (ctx *VmContext) unsetTimeout() {
 	}
 }
 
+func (ctx *VmContext) nextPmemId() int {
+	ctx.idLock.Lock()
+	id := ctx.pmemId
+	ctx.pmemId++
+	ctx.idLock.Unlock()
+	return id
+}
+
 func (ctx *VmContext) nextScsiId() int {
 	ctx.idLock.Lock()
 	id := ctx.scsiId

hypervisor/context.go

@@ -19,6 +19,8 @@ type DiskDescriptor struct {
 	ScsiAddr     string
 	DockerVolume bool
 	Options      map[string]string
+	Dax          bool
+	PmemId       int
 }
 
 func (d *DiskDescriptor) IsDir() bool {
@@ -65,6 +67,8 @@ func NewDiskContext(ctx *VmContext, vol *api.VolumeDescription) *DiskContext {
 			"bytespersec": strconv.Itoa(int(vol.Options.BytesPerSec)),
 			"iops":        strconv.Itoa(int(vol.Options.Iops)),
 		}
+	} else if vol.Options != nil && vol.Options.DaxBlock && ctx.Boot.HotAddCpuMem {
+		dc.DiskDescriptor.Dax = true
 	}
 	return dc
 }
@@ -78,7 +82,11 @@ func (dc *DiskContext) insert(result chan api.Result) {
 		return
 	}
 
-	dc.ScsiId = dc.sandbox.nextScsiId()
+	if dc.Dax {
+		dc.PmemId = dc.sandbox.nextPmemId()
+	} else {
+		dc.ScsiId = dc.sandbox.nextScsiId()
+	}
 	usage := "volume"
 	if dc.isRootVol {
 		usage = "image"

hypervisor/disk.go

@@ -461,7 +461,7 @@ func (lc *LibvirtContext) domainXml(ctx *hypervisor.VmContext) (string, error) {
 	}
 
 	if ctx.Boot.HotAddCpuMem {
-		dom.OS.Type.Machine = "pc-i440fx-2.1"
+		dom.OS.Type.Machine = "pc-i440fx-2.1,nvdimm"
 		dom.VCpu.Content = hypervisor.DefaultMaxCpus
 		dom.MaxMem = &maxmem{Unit: "MiB", Slots: "1", Content: hypervisor.DefaultMaxMem}
 
@@ -918,6 +918,35 @@ func (lc *LibvirtContext) AddDisk(ctx *hypervisor.VmContext, sourceType string,
 		return
 	}
 
+	if blockInfo.Dax {
+		// get the size
+		fi, e := os.Stat(blockInfo.Filename)
+		if e != nil {
+			result <- &hypervisor.DeviceFailed{}
+			return
+		}
+		size := fi.Size()
+		// compose hmp
+		hmp := fmt.Sprintf("object_add memory-backend-file,id=mem%d,share=on,mem-path=%s,size=%d", blockInfo.PmemId, blockInfo.Filename, size)
+		err := exec.Command("virsh", "-c", LibvirtdAddress, "qemu-monitor-command", ctx.Id, "--hmp", hmp).Run()
+		if err != nil {
+			result <- &hypervisor.DeviceFailed{}
+			return
+		}
+		hmp = fmt.Sprintf("device_add nvdimm,id=nvdimm%d,memdev=mem%d", blockInfo.PmemId, blockInfo.PmemId)
+		err = exec.Command("virsh", "-c", LibvirtdAddress, "qemu-monitor-command", ctx.Id, "--hmp", hmp).Run()
+		if err != nil {
+			result <- &hypervisor.DeviceFailed{}
+			return
+		}
+		result <- &hypervisor.BlockdevInsertedEvent{
+			Name:       name,
+			SourceType: sourceType,
+			DeviceName: "pmem" + strconv.Itoa(blockInfo.PmemId),
+		}
+		return
+	}
+
 	secretUUID, err := lc.diskSecretUUID(blockInfo)
 	if err != nil {
 		glog.Error("generate disk-get-secret failed, ", err.Error())

hypervisor/libvirt/libvirt.go

@@ -136,6 +136,7 @@ func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
 	return &VmHwStatus{
 		PciAddr:  ctx.pciAddr,
 		ScsiId:   ctx.scsiId,
+		PmemId:   ctx.pmemId,
 		AttachId: ctx.hyperstart.LastStreamSeq(),
 		GuestCid: ctx.GuestCid,
 	}
@@ -144,6 +145,7 @@ func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
 func (ctx *VmContext) loadHwStatus(pinfo *PersistInfo) error {
 	ctx.pciAddr = pinfo.HwStat.PciAddr
 	ctx.scsiId = pinfo.HwStat.ScsiId
+	ctx.pmemId = pinfo.HwStat.PmemId
 	ctx.GuestCid = pinfo.HwStat.GuestCid
 	if ctx.GuestCid != 0 {
 		if !VsockCidManager.MarkCidInuse(ctx.GuestCid) {

hypervisor/persistence.go

@@ -247,7 +247,58 @@ func (qc *QemuContext) AddDisk(ctx *hypervisor.VmContext, sourceType string, blo
 		}
 	}
 
-	newDiskAddSession(ctx, qc, name, sourceType, filename, format, id, result)
+	commands := make([]*QmpCommand, 2)
+	devName := scsiId2Name(id)
+	if !blockInfo.Dax {
+		commands[0] = &QmpCommand{
+			Execute: "human-monitor-command",
+			Arguments: map[string]interface{}{
+				"command-line": "drive_add dummy file=" +
+					filename + ",if=none,id=" + "drive" + strconv.Itoa(id) + ",format=" + format + ",cache=writeback",
+			},
+		}
+		commands[1] = &QmpCommand{
+			Execute: "device_add",
+			Arguments: map[string]interface{}{
+				"driver": "scsi-hd", "bus": "scsi0.0", "scsi-id": strconv.Itoa(id),
+				"drive": "drive" + strconv.Itoa(id), "id": "scsi-disk" + strconv.Itoa(id),
+			},
+		}
+	} else {
+		// get the size
+		fi, e := os.Stat(filename)
+		if e != nil {
+			result <- &hypervisor.DeviceFailed{}
+			return
+		}
+		size := fi.Size()
+		// compose qmp commands
+		// hmp: object_add memory-backend-file,id=mem2,share=off,mem-path=/path/to/dax.img,size=10G
+		// hmp: device_add nvdimm,id=nvdimm2,memdev=mem2
+		commands[0] = &QmpCommand{
+			Execute: "human-monitor-command",
+			Arguments: map[string]interface{}{
+				"command-line": "object_add memory-backend-file,id=mem" + strconv.Itoa(blockInfo.PmemId) +
+					",share=on,mem-path=" + filename + ",size=" + strconv.FormatInt(size, 10),
+			},
+		}
+		commands[1] = &QmpCommand{
+			Execute: "device_add",
+			Arguments: map[string]interface{}{
+				"driver": "nvdimm", "memdev": "mem" + strconv.Itoa(blockInfo.PmemId), "id": "nvdimm" + strconv.Itoa(blockInfo.PmemId),
+			},
+		}
+		devName = "pmem" + strconv.Itoa(blockInfo.PmemId)
+	}
+	qc.qmp <- &QmpSession{
+		commands: commands,
+		respond: defaultRespond(result, &hypervisor.BlockdevInsertedEvent{
+			Name:       name,
+			SourceType: sourceType,
+			DeviceName: devName,
+			ScsiId:     id,
+		}),
+	}
 }
 
 func (qc *QemuContext) RemoveDisk(ctx *hypervisor.VmContext, blockInfo *hypervisor.DiskDescriptor, callback hypervisor.VmEvent, result chan<- hypervisor.VmEvent) {