Skip to content

Commit f0ec553

Browse files
rp-rp-
committed
linstor: record the size of the DRBD device as the volume size
Linstor rounds the storage of a volume up (e.g. to the LVM extent size) and DRBD makes all of it usable, so the device a VM sees is usually a few MiB larger than the size CloudStack requested and records: a 512 MiB volume has a 540909568 byte device. Filesystems the guest creates on the whole disk, or grows to its end (cloud-init growpart), use that space, but the copies CloudStack makes are cut at the recorded size: snapshot backups to secondary storage are shrunk to it, and a copy opened at its own size then holds a filesystem larger than the disk. The driver now records the size of the device after creating a volume (new, cloned from a template or from a snapshot) and after a resize, and notifies the hypervisor of that size. Existing volumes are corrected before a snapshot and before a copy to secondary storage. The recorded size only grows, and the difference is counted against the account's primary storage, except on resize, which counts the recorded size itself. The device size is the smallest usable size of the top layer on the diskful resources; diskless resources only report the size of the volume definition. With DrbdOptions/ExactSize the device has the size of the volume definition.
1 parent d02adcd commit f0ec553

4 files changed

Lines changed: 302 additions & 2 deletions

File tree

‎plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,15 @@
7373
import com.cloud.storage.Volume;
7474
import com.cloud.storage.VolumeDetailVO;
7575
import com.cloud.storage.VolumeVO;
76+
import com.cloud.storage.dao.DiskOfferingDao;
7677
import com.cloud.storage.dao.SnapshotDao;
7778
import com.cloud.storage.dao.SnapshotDetailsDao;
7879
import com.cloud.storage.dao.SnapshotDetailsVO;
7980
import com.cloud.storage.dao.VMTemplateDao;
8081
import com.cloud.storage.dao.VMTemplatePoolDao;
8182
import com.cloud.storage.dao.VolumeDao;
8283
import com.cloud.storage.dao.VolumeDetailsDao;
84+
import com.cloud.user.ResourceLimitService;
8385
import com.cloud.utils.NumbersUtil;
8486
import com.cloud.utils.Pair;
8587
import com.cloud.utils.exception.CloudRuntimeException;
@@ -133,6 +135,8 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
133135
@Inject
134136
private HostDao _hostDao;
135137
@Inject private VMTemplateDao _vmTemplateDao;
138+
@Inject private DiskOfferingDao _diskOfferingDao;
139+
@Inject private ResourceLimitService _resourceLimitMgr;
136140

137141
private final Map<String, Long> volumeStatsLastUpdate = new HashMap<>();
138142
private final Map<String, Pair<Long, Long>> volumeStats = new HashMap<>();
@@ -354,6 +358,33 @@ private void resizeResource(DevelopersApi api, String resourceName, long sizeByt
354358
}
355359
}
356360

361+
/**
362+
* Records the size of the volume's Linstor device as the size of the volume, if it is larger. Linstor rounds the
363+
* storage of a volume up and the VM sees all of it, so copies of the volume (snapshot backups, templates,
364+
* downloads) have to cover all of it too.
365+
* @param adjustResourceCount whether to count the difference against the account's primary storage
366+
* @return the size of the volume
367+
*/
368+
protected long recordDeviceSize(DevelopersApi api, VolumeVO volume, String rscName, boolean adjustResourceCount) {
369+
final long size = volume.getSize() != null ? volume.getSize() : 0;
370+
try {
371+
final long deviceSize = LinstorUtil.getVolumeDeviceSize(api, rscName);
372+
if (deviceSize > size) {
373+
logger.info("Linstor: the device of {} has {} bytes, recording that instead of {}", volume, deviceSize, size);
374+
volume.setSize(deviceSize);
375+
_volumeDao.update(volume.getId(), volume);
376+
if (adjustResourceCount) {
377+
_resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(volume.getAccountId(),
378+
volume.isDisplayVolume(), deviceSize - size, _diskOfferingDao.findById(volume.getDiskOfferingId()));
379+
}
380+
return deviceSize;
381+
}
382+
} catch (ApiException apiEx) {
383+
logger.warn("Linstor: unable to get the device size of {}, keeping {} bytes: {}", rscName, size, apiEx.getBestMessage());
384+
}
385+
return size;
386+
}
387+
357388
/**
358389
* Update resource-definitions resource-group to the correct one if it isn't already the intended.
359390
* @param api Linstor api
@@ -614,6 +645,7 @@ public void createAsync(DataStore dataStore, DataObject vol, AsyncCompletionCall
614645
volume.setPath(vol.getUuid());
615646

616647
_volumeDao.update(volume.getId(), volume);
648+
recordDeviceSize(getLinstorAPI(storagePool), volume, LinstorUtil.RSC_PREFIX + volume.getPath(), true);
617649
break;
618650
case SNAPSHOT:
619651
logger.debug("createAsync - SNAPSHOT");
@@ -975,8 +1007,11 @@ private Answer copyVolume(DataObject srcData, DataObject dstData) {
9751007
final StoragePoolVO pool = _storagePoolDao.findById(srcVolInfo.getDataStore().getId());
9761008
final DevelopersApi api = getLinstorAPI(pool);
9771009
final String rscName = LinstorUtil.RSC_PREFIX + srcVolInfo.getPath();
1010+
// volumes created before their device size was recorded
1011+
final long size = recordDeviceSize(api, _volumeDao.findById(srcVolInfo.getId()), rscName, true);
9781012

9791013
VolumeObjectTO to = (VolumeObjectTO) srcVolInfo.getTO();
1014+
to.setSize(size);
9801015
// patch source format
9811016
// Linstor volumes are stored as RAW, but we can't set the correct format as RAW (we use QCOW2)
9821017
// otherwise create template from snapshot won't work, because this operation
@@ -1205,12 +1240,13 @@ public void copyAsync(DataObject srcData, DataObject destData, Host destHost, As
12051240
private CreateCmdResult notifyResize(
12061241
VolumeObject vol,
12071242
long oldSize,
1243+
long newSize,
12081244
ResizeVolumePayload resizeParameter)
12091245
{
12101246
StoragePool pool = (StoragePool) vol.getDataStore();
12111247

12121248
ResizeVolumeCommand resizeCmd =
1213-
new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), oldSize, resizeParameter.newSize, resizeParameter.shrinkOk,
1249+
new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), oldSize, newSize, resizeParameter.shrinkOk,
12141250
resizeParameter.instanceName, null);
12151251
CreateCmdResult result = new CreateCmdResult(null, null);
12161252
try {
@@ -1244,6 +1280,7 @@ public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> cal
12441280
final long oldSize = vol.getSize();
12451281

12461282
String errMsg = null;
1283+
long newSize = resizeParameter.newSize;
12471284
VolumeDefinitionModify dfm = new VolumeDefinitionModify();
12481285
dfm.setSizeKib(resizeParameter.newSize / 1024);
12491286
try
@@ -1257,6 +1294,8 @@ public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> cal
12571294
volume.setMaxIops(resizeParameter.newMaxIops);
12581295
volume.setSize(resizeParameter.newSize);
12591296
_volumeDao.update(volume.getId(), volume);
1297+
// the resize counts the recorded size against the account's primary storage
1298+
newSize = recordDeviceSize(api, volume, rscName, false);
12601299
}
12611300
} catch (ApiException apiExc)
12621301
{
@@ -1270,7 +1309,7 @@ public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> cal
12701309
result.setResult(errMsg);
12711310
} else {
12721311
// notify guests
1273-
result = notifyResize(vol, oldSize, resizeParameter);
1312+
result = notifyResize(vol, oldSize, newSize, resizeParameter);
12741313
}
12751314

12761315
callback.complete(result);
@@ -1313,6 +1352,8 @@ public void takeSnapshot(SnapshotInfo snapshotInfo, AsyncCompletionCallback<Crea
13131352
final StoragePoolVO storagePool = _storagePoolDao.findById(storagePoolId);
13141353
final DevelopersApi api = getLinstorAPI(storagePool);
13151354
final String rscName = LinstorUtil.RSC_PREFIX + volumeVO.getPath();
1355+
// volumes created before their device size was recorded, a backup of the snapshot covers the whole device
1356+
recordDeviceSize(api, volumeVO, rscName, true);
13161357

13171358
Snapshot snapshot = new Snapshot();
13181359
snapshot.setName(getSnapshotName(snapshotInfo.getUuid()));

‎plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,11 @@
2323
import com.linbit.linstor.api.model.ApiCallRc;
2424
import com.linbit.linstor.api.model.ApiCallRcList;
2525
import com.linbit.linstor.api.model.AutoSelectFilter;
26+
import com.linbit.linstor.api.model.DrbdVolume;
27+
import com.linbit.linstor.api.model.LUKSVolume;
2628
import com.linbit.linstor.api.model.LayerType;
2729
import com.linbit.linstor.api.model.Node;
30+
import com.linbit.linstor.api.model.OneOfVolumeLayerData;
2831
import com.linbit.linstor.api.model.Properties;
2932
import com.linbit.linstor.api.model.ProviderKind;
3033
import com.linbit.linstor.api.model.Resource;
@@ -34,6 +37,7 @@
3437
import com.linbit.linstor.api.model.ResourceGroupSpawn;
3538
import com.linbit.linstor.api.model.ResourceWithVolumes;
3639
import com.linbit.linstor.api.model.StoragePool;
40+
import com.linbit.linstor.api.model.StorageVolume;
3741
import com.linbit.linstor.api.model.Volume;
3842
import com.linbit.linstor.api.model.VolumeDefinition;
3943
import com.linbit.linstor.api.model.VolumeDefinitionModify;
@@ -72,6 +76,8 @@ public class LinstorUtil {
7276
public static final String RSC_PREFIX = "cs-";
7377
public static final String RSC_GROUP = "resourceGroup";
7478
public static final String CS_TEMPLATE_FOR_PREFIX = "_cs-template-for-";
79+
// the DRBD device has exactly the size of the volume definition
80+
public static final String PROP_DRBD_EXACT_SIZE = ApiConsts.NAMESPC_DRBD_OPTIONS + "/ExactSize";
7581

7682
public static final String LIN_PROP_DRBDOPT_EXACT_SIZE = "DrbdOptions/ExactSize";
7783

@@ -545,6 +551,63 @@ public static boolean isRscDiskless(ResourceWithVolumes rsc) {
545551
return rsc.getFlags() != null && rsc.getFlags().contains(ApiConsts.FLAG_DISKLESS);
546552
}
547553

554+
/**
555+
* The usable size of the topmost layer of a volume, the size of its block device on that node.
556+
*/
557+
private static Long getTopLayerUsableSizeKib(Volume vlm) {
558+
if (CollectionUtils.isNotEmpty(vlm.getLayerDataList())) {
559+
OneOfVolumeLayerData data = vlm.getLayerDataList().get(0).getData();
560+
if (data instanceof DrbdVolume) {
561+
return ((DrbdVolume) data).getUsableSizeKib();
562+
} else if (data instanceof LUKSVolume) {
563+
return ((LUKSVolume) data).getUsableSizeKib();
564+
} else if (data instanceof StorageVolume) {
565+
return ((StorageVolume) data).getUsableSizeKib();
566+
}
567+
}
568+
return vlm.getUsableSizeKib();
569+
}
570+
571+
/**
572+
* The size of the block device of a resource's (only) volume in bytes, the size a VM sees. Linstor rounds the
573+
* storage of a volume up, e.g. to the LVM extent size, and DRBD makes all of it usable, so the device is usually
574+
* a few MiB larger than the volume definition. A DRBD device has the smallest usable size of its diskful
575+
* resources; diskless resources only report the size of the volume definition. With the property
576+
* DrbdOptions/ExactSize the device has exactly the size of the volume definition.
577+
* @return the device size, never less than the size of the volume definition, 0 if the resource definition
578+
* has no volume
579+
*/
580+
public static long getVolumeDeviceSize(DevelopersApi api, String rscName) throws ApiException {
581+
List<ResourceDefinition> rscDfns = api.resourceDefinitionList(
582+
Collections.singletonList(rscName), true, null, null, null);
583+
if (rscDfns.isEmpty() || CollectionUtils.isEmpty(rscDfns.get(0).getVolumeDefinitions())) {
584+
return 0;
585+
}
586+
ResourceDefinition rscDfn = rscDfns.get(0);
587+
final long definedSize = rscDfn.getVolumeDefinitions().get(0).getSizeKib() * 1024;
588+
if (rscDfn.getProps() != null && Boolean.parseBoolean(rscDfn.getProps().get(PROP_DRBD_EXACT_SIZE))) {
589+
return definedSize;
590+
}
591+
592+
List<ResourceWithVolumes> resources = api.viewResources(
593+
Collections.emptyList(),
594+
Collections.singletonList(rscName),
595+
Collections.emptyList(),
596+
null,
597+
null,
598+
null);
599+
Long deviceKib = null;
600+
for (ResourceWithVolumes rsc : resources) {
601+
if (!isRscDiskless(rsc) && !rsc.getVolumes().isEmpty()) {
602+
Long usableKib = getTopLayerUsableSizeKib(rsc.getVolumes().get(0));
603+
if (usableKib != null && usableKib > 0 && (deviceKib == null || usableKib < deviceKib)) {
604+
deviceKib = usableKib;
605+
}
606+
}
607+
}
608+
return deviceKib != null ? Math.max(definedSize, deviceKib * 1024) : definedSize;
609+
}
610+
548611
/**
549612
* Checks if all diskful resource are on a zeroed block device.
550613
* @param pool Linstor pool to use

‎plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,15 @@
1919
import com.linbit.linstor.api.ApiException;
2020
import com.linbit.linstor.api.DevelopersApi;
2121
import com.linbit.linstor.api.model.AutoSelectFilter;
22+
import com.linbit.linstor.api.model.DrbdVolume;
2223
import com.linbit.linstor.api.model.LayerType;
24+
import com.linbit.linstor.api.model.Properties;
25+
import com.linbit.linstor.api.model.ResourceDefinition;
2326
import com.linbit.linstor.api.model.ResourceGroup;
27+
import com.linbit.linstor.api.model.ResourceWithVolumes;
28+
import com.linbit.linstor.api.model.Volume;
29+
import com.linbit.linstor.api.model.VolumeDefinition;
30+
import com.linbit.linstor.api.model.VolumeLayer;
2431

2532
import java.util.Arrays;
2633
import java.util.Collections;
@@ -42,6 +49,11 @@
4249
import org.mockito.junit.MockitoJUnitRunner;
4350

4451
import com.cloud.storage.DataStoreRole;
52+
import com.cloud.storage.DiskOfferingVO;
53+
import com.cloud.storage.VolumeVO;
54+
import com.cloud.storage.dao.DiskOfferingDao;
55+
import com.cloud.storage.dao.VolumeDao;
56+
import com.cloud.user.ResourceLimitService;
4557

4658
import static org.mockito.Mockito.mock;
4759
import static org.mockito.Mockito.when;
@@ -54,6 +66,15 @@ public class LinstorPrimaryDataStoreDriverImplTest {
5466
@Mock
5567
private SnapshotDataStoreDao snapshotStoreDao;
5668

69+
@Mock
70+
private VolumeDao volumeDao;
71+
72+
@Mock
73+
private DiskOfferingDao diskOfferingDao;
74+
75+
@Mock
76+
private ResourceLimitService resourceLimitMgr;
77+
5778
@InjectMocks
5879
private LinstorPrimaryDataStoreDriverImpl linstorPrimaryDataStoreDriver;
5980

@@ -178,4 +199,87 @@ public void testGetEncryptedLayerList() throws ApiException {
178199
layers = LinstorUtil.getEncryptedLayerList(api, "EncryptedGrp");
179200
Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers);
180201
}
202+
203+
/**
204+
* A 512 MiB volume definition whose DRBD device has 540909568 bytes (LVM-thin rounds the storage up).
205+
*/
206+
private void mockLinstorVolume(String rscName) throws ApiException {
207+
VolumeDefinition vlmDfn = new VolumeDefinition();
208+
vlmDfn.setSizeKib(524288L);
209+
ResourceDefinition rscDfn = new ResourceDefinition();
210+
rscDfn.setName(rscName);
211+
rscDfn.addVolumeDefinitionsItem(vlmDfn);
212+
rscDfn.setProps(new Properties());
213+
when(api.resourceDefinitionList(Collections.singletonList(rscName), true, null, null, null))
214+
.thenReturn(Collections.singletonList(rscDfn));
215+
216+
DrbdVolume drbdVolume = new DrbdVolume();
217+
drbdVolume.setUsableSizeKib(528232L);
218+
VolumeLayer layer = new VolumeLayer();
219+
layer.setType(LayerType.DRBD);
220+
layer.setData(drbdVolume);
221+
Volume volume = new Volume();
222+
volume.addLayerDataListItem(layer);
223+
ResourceWithVolumes rsc = new ResourceWithVolumes();
224+
rsc.setNodeName("nodeA");
225+
rsc.setVolumes(Collections.singletonList(volume));
226+
when(api.viewResources(Collections.emptyList(), Collections.singletonList(rscName), Collections.emptyList(),
227+
null, null, null)).thenReturn(Collections.singletonList(rsc));
228+
}
229+
230+
private VolumeVO mockVolume(long size) {
231+
VolumeVO volume = mock(VolumeVO.class);
232+
Mockito.lenient().when(volume.getId()).thenReturn(7L);
233+
Mockito.lenient().when(volume.getAccountId()).thenReturn(2L);
234+
Mockito.lenient().when(volume.isDisplayVolume()).thenReturn(true);
235+
Mockito.lenient().when(volume.getDiskOfferingId()).thenReturn(3L);
236+
when(volume.getSize()).thenReturn(size);
237+
return volume;
238+
}
239+
240+
@Test
241+
public void testRecordDeviceSizeRecordsTheLargerDevice() throws ApiException {
242+
mockLinstorVolume("cs-vol");
243+
VolumeVO volume = mockVolume(536870912L);
244+
DiskOfferingVO diskOffering = mock(DiskOfferingVO.class);
245+
when(diskOfferingDao.findById(3L)).thenReturn(diskOffering);
246+
247+
long size = linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", true);
248+
249+
Assert.assertEquals(540909568L, size);
250+
Mockito.verify(volume).setSize(540909568L);
251+
Mockito.verify(volumeDao).update(7L, volume);
252+
Mockito.verify(resourceLimitMgr).incrementVolumePrimaryStorageResourceCount(2L, true, 4038656L, diskOffering);
253+
}
254+
255+
@Test
256+
public void testRecordDeviceSizeWithoutResourceCount() throws ApiException {
257+
// a resize counts the recorded size itself
258+
mockLinstorVolume("cs-vol");
259+
VolumeVO volume = mockVolume(536870912L);
260+
261+
Assert.assertEquals(540909568L, linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", false));
262+
Mockito.verify(volumeDao).update(7L, volume);
263+
Mockito.verifyNoInteractions(resourceLimitMgr);
264+
}
265+
266+
@Test
267+
public void testRecordDeviceSizeKeepsRecordedDeviceSize() throws ApiException {
268+
mockLinstorVolume("cs-vol");
269+
VolumeVO volume = mockVolume(540909568L);
270+
271+
Assert.assertEquals(540909568L, linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", true));
272+
Mockito.verify(volume, Mockito.never()).setSize(Mockito.anyLong());
273+
Mockito.verifyNoInteractions(volumeDao, resourceLimitMgr);
274+
}
275+
276+
@Test
277+
public void testRecordDeviceSizeKeepsSizeIfLinstorFails() throws ApiException {
278+
when(api.resourceDefinitionList(Collections.singletonList("cs-vol"), true, null, null, null))
279+
.thenThrow(new ApiException(500, "controller unreachable"));
280+
VolumeVO volume = mockVolume(536870912L);
281+
282+
Assert.assertEquals(536870912L, linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", true));
283+
Mockito.verifyNoInteractions(volumeDao, resourceLimitMgr);
284+
}
181285
}

0 commit comments

Comments
 (0)