Skip to content

Commit d2615bb

Browse files
vishesh92vishesh92
authored
Add support for providing userdata to system VMs (#11654)
This PR adds support for specifying user data (cloud-init) for system VMs via Zone Scoped global settings. This allows the operators to customize the System VMs and setup monitoring, logging or execute any custom commands. We set the user data from the global setting in /var/cache/cloud/cmdline, and use the NoCloud datasource to process user data. cloud-init service is still disabled in the system VMs and it's executed as part of the cloud-postinit service which executes the postinit.sh script. Added global settings: systemvm.userdata.enabled - Disabled by default. Needs to be enabled to utilize the feature. console.proxy.vm.userdata - UUID of the User data to be used for Console Proxy secstorage.vm.userdata - UUID of the User data to be used for Secondary Storage VM virtual.router.userdata - UUID of the User data to be used for Virtual Routers
1 parent 858663f commit d2615bb

File tree

21 files changed

+840
-10
lines changed

21 files changed

+840
-10
lines changed

‎api/src/main/java/org/apache/cloudstack/userdata/UserDataManager.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,23 @@
2222

2323
import com.cloud.utils.component.Manager;
2424

25+
import java.io.IOException;
26+
2527
public interface UserDataManager extends Manager, Configurable {
2628
String VM_USERDATA_MAX_LENGTH_STRING = "vm.userdata.max.length";
2729
ConfigKey<Integer> VM_USERDATA_MAX_LENGTH = new ConfigKey<>("Advanced", Integer.class, VM_USERDATA_MAX_LENGTH_STRING, "32768",
2830
"Max length of vm userdata after base64 encoding. Default is 32768 and maximum is 1048576", true);
2931

3032
String concatenateUserData(String userdata1, String userdata2, String userdataProvider);
3133
String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod);
34+
35+
/**
36+
* This method validates the user data uuid for system VMs and returns the user data
37+
* after compression and base64 encoding for the system VM to consume.
38+
*
39+
* @param userDataUuid
40+
* @return a String containing the user data after compression and base64 encoding
41+
* @throws IOException
42+
*/
43+
String validateAndGetUserDataForSystemVM(String userDataUuid) throws IOException;
3244
}

‎engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,9 @@ public interface VirtualMachineManager extends Manager {
106106
ConfigKey<Boolean> VmSyncPowerStateTransitioning = new ConfigKey<>("Advanced", Boolean.class, "vm.sync.power.state.transitioning", "true",
107107
"Whether to sync power states of the transitioning and stalled VMs while processing VM power reports.", false);
108108

109+
ConfigKey<Boolean> SystemVmEnableUserData = new ConfigKey<>(Boolean.class, "systemvm.userdata.enabled", "Advanced", "false",
110+
"Enable user data for system VMs. When enabled, the CPVM, SSVM, and Router system VMs will use the values from the global settings console.proxy.vm.userdata, secstorage.vm.userdata, and virtual.router.userdata, respectively, to provide cloud-init user data to the VM.",
111+
true, ConfigKey.Scope.Zone, null);
109112

110113
interface Topics {
111114
String VM_POWER_STATE = "vm.powerstate";

‎engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5244,7 +5244,7 @@ public ConfigKey<?>[] getConfigKeys() {
52445244
VmConfigDriveLabel, VmConfigDriveOnPrimaryPool, VmConfigDriveForceHostCacheUse, VmConfigDriveUseHostCacheOnUnsupportedPool,
52455245
HaVmRestartHostUp, ResourceCountRunningVMsonly, AllowExposeHypervisorHostname, AllowExposeHypervisorHostnameAccountLevel, SystemVmRootDiskSize,
52465246
AllowExposeDomainInMetadata, MetadataCustomCloudName, VmMetadataManufacturer, VmMetadataProductName,
5247-
VmSyncPowerStateTransitioning
5247+
VmSyncPowerStateTransitioning, SystemVmEnableUserData
52485248
};
52495249
}
52505250

‎engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,18 @@
1616
// under the License.
1717
package org.apache.cloudstack.userdata;
1818

19+
import java.io.IOException;
1920
import java.io.UnsupportedEncodingException;
2021
import java.net.URLDecoder;
2122
import java.util.HashMap;
2223
import java.util.List;
2324
import java.util.Map;
2425

26+
import com.cloud.domain.Domain;
27+
import com.cloud.user.User;
28+
import com.cloud.user.UserDataVO;
29+
import com.cloud.user.dao.UserDataDao;
30+
import com.cloud.utils.compression.CompressionUtil;
2531
import org.apache.cloudstack.api.BaseCmd;
2632
import org.apache.cloudstack.framework.config.ConfigKey;
2733
import org.apache.commons.codec.binary.Base64;
@@ -31,7 +37,12 @@
3137
import com.cloud.utils.component.ManagerBase;
3238
import com.cloud.utils.exception.CloudRuntimeException;
3339

40+
import javax.inject.Inject;
41+
3442
public class UserDataManagerImpl extends ManagerBase implements UserDataManager {
43+
@Inject
44+
UserDataDao userDataDao;
45+
3546
private static final int MAX_USER_DATA_LENGTH_BYTES = 2048;
3647
private static final int MAX_HTTP_GET_LENGTH = 2 * MAX_USER_DATA_LENGTH_BYTES; // 4KB
3748
private static final int NUM_OF_2K_BLOCKS = 512;
@@ -118,6 +129,25 @@ public String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod) {
118129
return Base64.encodeBase64String(decodedUserData);
119130
}
120131

132+
@Override
133+
public String validateAndGetUserDataForSystemVM(String userDataUuid) throws IOException {
134+
if (StringUtils.isBlank(userDataUuid)) {
135+
return null;
136+
}
137+
UserDataVO userDataVo = userDataDao.findByUuid(userDataUuid);
138+
if (userDataVo == null) {
139+
return null;
140+
}
141+
if (userDataVo.getDomainId() == Domain.ROOT_DOMAIN && userDataVo.getAccountId() == User.UID_ADMIN) {
142+
// Decode base64 user data, compress it, then re-encode to reduce command line length
143+
String plainTextUserData = new String(java.util.Base64.getDecoder().decode(userDataVo.getUserData()));
144+
CompressionUtil compressionUtil = new CompressionUtil();
145+
byte[] compressedUserData = compressionUtil.compressString(plainTextUserData);
146+
return java.util.Base64.getEncoder().encodeToString(compressedUserData);
147+
}
148+
throw new CloudRuntimeException("User data can only be used by system VMs if it belongs to the ROOT domain and ADMIN account.");
149+
}
150+
121151
private byte[] validateAndDecodeByHTTPMethod(String userData, int maxHTTPLength, BaseCmd.HTTPMethod httpMethod) {
122152
byte[] decodedUserData = Base64.decodeBase64(userData.getBytes());
123153
if (decodedUserData == null || decodedUserData.length < 1) {

‎engine/userdata/src/test/java/org/apache/cloudstack/userdata/UserDataManagerImplTest.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,37 @@
1717
package org.apache.cloudstack.userdata;
1818

1919
import static org.junit.Assert.assertEquals;
20+
import static org.junit.Assert.assertFalse;
21+
import static org.junit.Assert.assertNotEquals;
22+
import static org.junit.Assert.assertNotNull;
23+
import static org.junit.Assert.assertNull;
24+
import static org.mockito.Mockito.when;
2025

26+
import java.io.IOException;
2127
import java.nio.charset.StandardCharsets;
28+
import java.util.Base64;
2229

2330
import org.apache.cloudstack.api.BaseCmd;
2431
import org.junit.Test;
2532
import org.junit.runner.RunWith;
2633
import org.mockito.InjectMocks;
34+
import org.mockito.Mock;
35+
import org.mockito.Mockito;
2736
import org.mockito.Spy;
2837
import org.mockito.junit.MockitoJUnitRunner;
2938

39+
import com.cloud.domain.Domain;
40+
import com.cloud.user.User;
41+
import com.cloud.user.UserDataVO;
42+
import com.cloud.user.dao.UserDataDao;
43+
import com.cloud.utils.exception.CloudRuntimeException;
44+
3045
@RunWith(MockitoJUnitRunner.class)
3146
public class UserDataManagerImplTest {
3247

48+
@Mock
49+
private UserDataDao userDataDao;
50+
3351
@Spy
3452
@InjectMocks
3553
private UserDataManagerImpl userDataManager;
@@ -56,4 +74,76 @@ public void testValidateUrlEncodedBase64() {
5674
assertEquals("validate return the value with padding", encodedUserdata, userDataManager.validateUserData(urlEncodedUserdata, BaseCmd.HTTPMethod.GET));
5775
}
5876

77+
@Test
78+
public void testValidateAndGetUserDataForSystemVMWithBlankUuid() throws IOException {
79+
// Test with blank UUID should return null
80+
assertNull("null UUID should return null", userDataManager.validateAndGetUserDataForSystemVM(null));
81+
assertNull("blank UUID should return null", userDataManager.validateAndGetUserDataForSystemVM(""));
82+
assertNull("blank UUID should return null", userDataManager.validateAndGetUserDataForSystemVM(" "));
83+
}
84+
85+
@Test
86+
public void testValidateAndGetUserDataForSystemVMNotFound() throws IOException {
87+
// Test when userDataVo is not found
88+
String testUuid = "test-uuid-123";
89+
when(userDataDao.findByUuid(testUuid)).thenReturn(null);
90+
91+
assertNull("userdata not found should return null", userDataManager.validateAndGetUserDataForSystemVM(testUuid));
92+
}
93+
94+
@Test(expected = CloudRuntimeException.class)
95+
public void testValidateAndGetUserDataForSystemVMInvalidDomain() throws IOException {
96+
// Test with userDataVo that doesn't belong to ROOT domain
97+
String testUuid = "test-uuid-123";
98+
UserDataVO userDataVo = Mockito.mock(UserDataVO.class);
99+
when(userDataVo.getDomainId()).thenReturn(2L); // Not ROOT domain
100+
101+
when(userDataDao.findByUuid(testUuid)).thenReturn(userDataVo);
102+
userDataManager.validateAndGetUserDataForSystemVM(testUuid);
103+
}
104+
105+
@Test(expected = CloudRuntimeException.class)
106+
public void testValidateAndGetUserDataForSystemVMInvalidAccount() throws IOException {
107+
// Test with userDataVo that doesn't belong to ADMIN account
108+
String testUuid = "test-uuid-123";
109+
UserDataVO userDataVo = Mockito.mock(UserDataVO.class);
110+
when(userDataVo.getDomainId()).thenReturn(Domain.ROOT_DOMAIN);
111+
when(userDataVo.getAccountId()).thenReturn(3L);
112+
userDataVo.setUserData("dGVzdCBkYXRh"); // "test data" in base64
113+
114+
when(userDataDao.findByUuid(testUuid)).thenReturn(userDataVo);
115+
userDataManager.validateAndGetUserDataForSystemVM(testUuid);
116+
}
117+
118+
@Test
119+
public void testValidateAndGetUserDataForSystemVMValidSystemVMUserData() throws IOException {
120+
// Test with valid system VM userdata (ROOT domain + ADMIN account)
121+
String testUuid = "test-uuid-123";
122+
String originalText = "#!/bin/bash\necho 'Hello World'";
123+
String base64EncodedUserData = Base64.getEncoder().encodeToString(originalText.getBytes());
124+
125+
UserDataVO userDataVo = Mockito.mock(UserDataVO.class);
126+
when(userDataVo.getDomainId()).thenReturn(Domain.ROOT_DOMAIN);
127+
when(userDataVo.getAccountId()).thenReturn(User.UID_ADMIN);
128+
when(userDataVo.getUserData()).thenReturn(base64EncodedUserData);
129+
130+
when(userDataDao.findByUuid(testUuid)).thenReturn(userDataVo);
131+
132+
String result = userDataManager.validateAndGetUserDataForSystemVM(testUuid);
133+
134+
// Verify result is not null and is base64 encoded
135+
assertNotNull("result should not be null", result);
136+
assertFalse("result should be base64 encoded", result.isEmpty());
137+
138+
// Verify the result is valid base64
139+
try {
140+
Base64.getDecoder().decode(result);
141+
} catch (IllegalArgumentException e) {
142+
throw new AssertionError("Result should be valid base64", e);
143+
}
144+
145+
// The result should be different from input since it's compressed
146+
assertNotEquals("compressed result should be different from original", result, base64EncodedUserData);
147+
}
148+
59149
}

‎framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,64 @@ public class ConfigKey<T> {
4343
public static final String CATEGORY_NETWORK = "Network";
4444
public static final String CATEGORY_SYSTEM = "System";
4545

46+
// Configuration Groups to be used to define group for a config key
47+
// Group name, description, precedence
48+
public static final Ternary<String, String, Long> GROUP_MISCELLANEOUS = new Ternary<>("Miscellaneous", "Miscellaneous configuration", 999L);
49+
public static final Ternary<String, String, Long> GROUP_ACCESS = new Ternary<>("Access", "Identity and Access management configuration", 1L);
50+
public static final Ternary<String, String, Long> GROUP_COMPUTE = new Ternary<>("Compute", "Compute configuration", 2L);
51+
public static final Ternary<String, String, Long> GROUP_STORAGE = new Ternary<>("Storage", "Storage configuration", 3L);
52+
public static final Ternary<String, String, Long> GROUP_NETWORK = new Ternary<>("Network", "Network configuration", 4L);
53+
public static final Ternary<String, String, Long> GROUP_HYPERVISOR = new Ternary<>("Hypervisor", "Hypervisor specific configuration", 5L);
54+
public static final Ternary<String, String, Long> GROUP_MANAGEMENT_SERVER = new Ternary<>("Management Server", "Management Server configuration", 6L);
55+
public static final Ternary<String, String, Long> GROUP_SYSTEM_VMS = new Ternary<>("System VMs", "System VMs related configuration", 7L);
56+
public static final Ternary<String, String, Long> GROUP_INFRASTRUCTURE = new Ternary<>("Infrastructure", "Infrastructure configuration", 8L);
57+
public static final Ternary<String, String, Long> GROUP_USAGE_SERVER = new Ternary<>("Usage Server", "Usage Server related configuration", 9L);
58+
59+
// Configuration Subgroups to be used to define subgroup for a config key
60+
// Subgroup name, description, precedence
61+
public static final Pair<String, Long> SUBGROUP_OTHERS = new Pair<>("Others", 999L);
62+
public static final Pair<String, Long> SUBGROUP_ACCOUNT = new Pair<>("Account", 1L);
63+
public static final Pair<String, Long> SUBGROUP_DOMAIN = new Pair<>("Domain", 2L);
64+
public static final Pair<String, Long> SUBGROUP_PROJECT = new Pair<>("Project", 3L);
65+
public static final Pair<String, Long> SUBGROUP_LDAP = new Pair<>("LDAP", 4L);
66+
public static final Pair<String, Long> SUBGROUP_SAML = new Pair<>("SAML", 5L);
67+
public static final Pair<String, Long> SUBGROUP_VIRTUAL_MACHINE = new Pair<>("Virtual Machine", 1L);
68+
public static final Pair<String, Long> SUBGROUP_KUBERNETES = new Pair<>("Kubernetes", 2L);
69+
public static final Pair<String, Long> SUBGROUP_HIGH_AVAILABILITY = new Pair<>("High Availability", 3L);
70+
public static final Pair<String, Long> SUBGROUP_IMAGES = new Pair<>("Images", 1L);
71+
public static final Pair<String, Long> SUBGROUP_VOLUME = new Pair<>("Volume", 2L);
72+
public static final Pair<String, Long> SUBGROUP_SNAPSHOT = new Pair<>("Snapshot", 3L);
73+
public static final Pair<String, Long> SUBGROUP_VM_SNAPSHOT = new Pair<>("VM Snapshot", 4L);
74+
public static final Pair<String, Long> SUBGROUP_NETWORK = new Pair<>("Network", 1L);
75+
public static final Pair<String, Long> SUBGROUP_DHCP = new Pair<>("DHCP", 2L);
76+
public static final Pair<String, Long> SUBGROUP_VPC = new Pair<>("VPC", 3L);
77+
public static final Pair<String, Long> SUBGROUP_LOADBALANCER = new Pair<>("LoadBalancer", 4L);
78+
public static final Pair<String, Long> SUBGROUP_API = new Pair<>("API", 1L);
79+
public static final Pair<String, Long> SUBGROUP_ALERTS = new Pair<>("Alerts", 2L);
80+
public static final Pair<String, Long> SUBGROUP_EVENTS = new Pair<>("Events", 3L);
81+
public static final Pair<String, Long> SUBGROUP_SECURITY = new Pair<>("Security", 4L);
82+
public static final Pair<String, Long> SUBGROUP_USAGE = new Pair<>("Usage", 1L);
83+
public static final Pair<String, Long> SUBGROUP_LIMITS = new Pair<>("Limits", 6L);
84+
public static final Pair<String, Long> SUBGROUP_JOBS = new Pair<>("Jobs", 7L);
85+
public static final Pair<String, Long> SUBGROUP_AGENT = new Pair<>("Agent", 8L);
86+
public static final Pair<String, Long> SUBGROUP_HYPERVISOR = new Pair<>("Hypervisor", 1L);
87+
public static final Pair<String, Long> SUBGROUP_KVM = new Pair<>("KVM", 2L);
88+
public static final Pair<String, Long> SUBGROUP_VMWARE = new Pair<>("VMware", 3L);
89+
public static final Pair<String, Long> SUBGROUP_XENSERVER = new Pair<>("XenServer", 4L);
90+
public static final Pair<String, Long> SUBGROUP_OVM = new Pair<>("OVM", 5L);
91+
public static final Pair<String, Long> SUBGROUP_BAREMETAL = new Pair<>("Baremetal", 6L);
92+
public static final Pair<String, Long> SUBGROUP_CONSOLE_PROXY_VM = new Pair<>("ConsoleProxyVM", 1L);
93+
public static final Pair<String, Long> SUBGROUP_SEC_STORAGE_VM = new Pair<>("SecStorageVM", 2L);
94+
public static final Pair<String, Long> SUBGROUP_VIRTUAL_ROUTER = new Pair<>("VirtualRouter", 3L);
95+
public static final Pair<String, Long> SUBGROUP_DIAGNOSTICS = new Pair<>("Diagnostics", 4L);
96+
public static final Pair<String, Long> SUBGROUP_PRIMARY_STORAGE = new Pair<>("Primary Storage", 1L);
97+
public static final Pair<String, Long> SUBGROUP_SECONDARY_STORAGE = new Pair<>("Secondary Storage", 2L);
98+
public static final Pair<String, Long> SUBGROUP_BACKUP_AND_RECOVERY = new Pair<>("Backup & Recovery", 1L);
99+
public static final Pair<String, Long> SUBGROUP_CERTIFICATE_AUTHORITY = new Pair<>("Certificate Authority", 2L);
100+
public static final Pair<String, Long> SUBGROUP_QUOTA = new Pair<>("Quota", 3L);
101+
public static final Pair<String, Long> SUBGROUP_CLOUDIAN = new Pair<>("Cloudian", 4L);
102+
public static final Pair<String, Long> SUBGROUP_DRS = new Pair<>("DRS", 4L);
103+
46104
public enum Scope {
47105
Global(null, 1),
48106
Zone(Global, 1 << 1),

‎plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
3636
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
3737
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
38+
import org.apache.cloudstack.userdata.UserDataManager;
39+
import org.apache.commons.lang3.StringUtils;
3840
import org.springframework.stereotype.Component;
3941

4042
import com.cloud.agent.AgentManager;
@@ -101,6 +103,9 @@
101103
import com.cloud.vm.dao.DomainRouterDao;
102104
import com.cloud.vm.dao.NicDao;
103105

106+
import static com.cloud.network.router.VirtualNetworkApplianceManager.VirtualRouterUserData;
107+
import static com.cloud.vm.VirtualMachineManager.SystemVmEnableUserData;
108+
104109
@Component
105110
public class ElasticLoadBalancerManagerImpl extends ManagerBase implements ElasticLoadBalancerManager, VirtualMachineGuru {
106111

@@ -136,6 +141,8 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast
136141
private ElasticLbVmMapDao _elbVmMapDao;
137142
@Inject
138143
private NicDao _nicDao;
144+
@Inject
145+
private UserDataManager userDataManager;
139146

140147
String _instance;
141148

@@ -477,6 +484,19 @@ public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, Depl
477484
}
478485
String msPublicKey = _configDao.getValue("ssh.publickey");
479486
buf.append(" authorized_key=").append(VirtualMachineGuru.getEncodedMsPublicKey(msPublicKey));
487+
488+
if (SystemVmEnableUserData.valueIn(dc.getId())) {
489+
String userDataUuid = VirtualRouterUserData.valueIn(dc.getId());
490+
try {
491+
String userData = userDataManager.validateAndGetUserDataForSystemVM(userDataUuid);
492+
if (StringUtils.isNotBlank(userData)) {
493+
buf.append(" userdata=").append(userData);
494+
}
495+
} catch (Exception e) {
496+
logger.warn("Failed to load user data for the elastic lb vm, ignored", e);
497+
}
498+
}
499+
480500
if (logger.isDebugEnabled()) {
481501
logger.debug("Boot Args for " + profile + ": " + buf.toString());
482502
}

0 commit comments

Comments
 (0)