From 8868f0927b7e5cbe9cdc6a593b4ddbfe16519d58 Mon Sep 17 00:00:00 2001 From: hamistao Date: Fri, 6 Dec 2024 03:39:07 -0300 Subject: [PATCH 1/9] api: Add `storage_volumes_instance_rootfs_size` Signed-off-by: hamistao --- doc/api-extensions.md | 5 +++++ shared/version/api.go | 1 + 2 files changed, 6 insertions(+) diff --git a/doc/api-extensions.md b/doc/api-extensions.md index c1aa436cae7c..7f48f5fc41c5 100644 --- a/doc/api-extensions.md +++ b/doc/api-extensions.md @@ -2538,3 +2538,8 @@ Adds a new {config:option}`device-unix-hotplug-device-conf:subsystem` configurat ## `storage_ceph_osd_pool_size` This introduces the configuration keys {config:option}`storage-ceph-pool-conf:ceph.osd.pool_size`, and {config:option}`storage-cephfs-pool-conf:cephfs.osd_pool_size` to be used when adding or updating a `ceph` or `cephfs` storage pool to instruct LXD to create set the replication size for the underlying OSD pools. + +## `storage_volumes_instance_rootfs_size` + +Introduces a `volatile.rootfs.size` key, previously only supported for images, for container and virtual machine volumes. This key is used to store a volume's current size at any point. +Necessary to get accurate root disk sizes for instances. diff --git a/shared/version/api.go b/shared/version/api.go index 7eff819c9332..9115cfe0562d 100644 --- a/shared/version/api.go +++ b/shared/version/api.go @@ -428,6 +428,7 @@ var APIExtensions = []string{ "unix_device_hotplug_ownership_inherit", "unix_device_hotplug_subsystem_device_option", "storage_ceph_osd_pool_size", + "storage_volumes_instance_rootfs_size", } // APIExtensionsCount returns the number of available API extensions. From d6a62f66fe126c701be8b1fbe9f5b61fc3ef77de Mon Sep 17 00:00:00 2001 From: hamistao Date: Fri, 6 Dec 2024 03:34:56 -0300 Subject: [PATCH 2/9] lxd/storage: Allow `volatile.rootfs.size` for instance volumes Signed-off-by: hamistao --- lxd/storage/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lxd/storage/utils.go b/lxd/storage/utils.go index e6684a0bda5f..c886b3df9f6f 100644 --- a/lxd/storage/utils.go +++ b/lxd/storage/utils.go @@ -724,8 +724,8 @@ func validateVolumeCommonRules(vol drivers.Volume) map[string]func(string) error rules["block.filesystem"] = validate.IsAny } - // volatile.rootfs.size is only used for image volumes. - if vol.Type() == drivers.VolumeTypeImage { + // volatile.rootfs.size is only used for image and instance volumes. + if shared.ValueInSlice(vol.Type(), []drivers.VolumeType{drivers.VolumeTypeImage, drivers.VolumeTypeContainer, drivers.VolumeTypeContainer}) { rules["volatile.rootfs.size"] = validate.Optional(validate.IsInt64) } From e2689b29f84857574bf8be186bf56334a1e11d0c Mon Sep 17 00:00:00 2001 From: hamistao Date: Mon, 9 Dec 2024 02:42:13 -0300 Subject: [PATCH 3/9] lxd/storage: Validate `volatile.rootfs.size` as size All the handling of `volatile.rootfs.size` for images considers that it can be a '123 GiB' type string as well, so this also considers these strings when validating the key Signed-off-by: hamistao --- lxd/storage/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lxd/storage/utils.go b/lxd/storage/utils.go index c886b3df9f6f..19791c4a83c8 100644 --- a/lxd/storage/utils.go +++ b/lxd/storage/utils.go @@ -726,7 +726,7 @@ func validateVolumeCommonRules(vol drivers.Volume) map[string]func(string) error // volatile.rootfs.size is only used for image and instance volumes. if shared.ValueInSlice(vol.Type(), []drivers.VolumeType{drivers.VolumeTypeImage, drivers.VolumeTypeContainer, drivers.VolumeTypeContainer}) { - rules["volatile.rootfs.size"] = validate.Optional(validate.IsInt64) + rules["volatile.rootfs.size"] = validate.Optional(validate.IsSize) } return rules From 7db540fc53330f16a23067a4ec4e58ff68db1f49 Mon Sep 17 00:00:00 2001 From: hamistao Date: Wed, 4 Dec 2024 02:30:16 -0300 Subject: [PATCH 4/9] lxd/db/storage_volumes: Create `UpdateStoragePoolVolumeConfig` Signed-off-by: hamistao --- lxd/db/storage_volumes.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lxd/db/storage_volumes.go b/lxd/db/storage_volumes.go index 4b3ca03d6db2..c6bbb20443b1 100644 --- a/lxd/db/storage_volumes.go +++ b/lxd/db/storage_volumes.go @@ -372,6 +372,23 @@ func storageVolumeSnapshotConfig(ctx context.Context, tx *ClusterTx, volumeSnaps }, volumeSnapshotID) } +// UpdateStoragePoolVolumeConfig updates a storage volume's config on the database. +func (c *ClusterTx) UpdateStoragePoolVolumeConfig(volumeName string, volumeID int64, volumeConfig map[string]string) error { + isSnapshot := strings.Contains(volumeName, shared.SnapshotDelimiter) + + err := storageVolumeConfigClear(c.tx, volumeID, isSnapshot) + if err != nil { + return err + } + + err = storageVolumeConfigAdd(c.tx, volumeID, volumeConfig, isSnapshot) + if err != nil { + return err + } + + return nil +} + // UpdateStoragePoolVolume updates the storage volume attached to a given storage pool. func (c *ClusterTx) UpdateStoragePoolVolume(ctx context.Context, projectName string, volumeName string, volumeType int, poolID int64, volumeDescription string, volumeConfig map[string]string) error { isSnapshot := strings.Contains(volumeName, shared.SnapshotDelimiter) From 1c2680f490f05e85805e7d833e16e976bc60704a Mon Sep 17 00:00:00 2001 From: hamistao Date: Wed, 4 Dec 2024 02:30:57 -0300 Subject: [PATCH 5/9] lxd/db/storage_volumes: Use `UpdateStoragePoolVolumeConfig` Signed-off-by: hamistao --- lxd/db/storage_volumes.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lxd/db/storage_volumes.go b/lxd/db/storage_volumes.go index c6bbb20443b1..a045d074cf13 100644 --- a/lxd/db/storage_volumes.go +++ b/lxd/db/storage_volumes.go @@ -398,12 +398,7 @@ func (c *ClusterTx) UpdateStoragePoolVolume(ctx context.Context, projectName str return err } - err = storageVolumeConfigClear(c.tx, volume.ID, isSnapshot) - if err != nil { - return err - } - - err = storageVolumeConfigAdd(c.tx, volume.ID, volumeConfig, isSnapshot) + err = c.UpdateStoragePoolVolumeConfig(volume.Name, volume.ID, volumeConfig) if err != nil { return err } From d73157f5407d3cf57eb8bb55db80a9157dd677db Mon Sep 17 00:00:00 2001 From: hamistao Date: Wed, 4 Dec 2024 02:34:36 -0300 Subject: [PATCH 6/9] lxd/storage: Update instance volume config on resize This makes it so that the an instance volume's `size` config is aligned with the volume's actual size in any point in time. i.e. some configs like disk device `size` and pool `volume.config` propagate into `volume.size`, similarly to what happens with custom volumes. Signed-off-by: hamistao --- lxd/storage/backend_lxd.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lxd/storage/backend_lxd.go b/lxd/storage/backend_lxd.go index ee111d241e05..17936bc74176 100644 --- a/lxd/storage/backend_lxd.go +++ b/lxd/storage/backend_lxd.go @@ -3302,7 +3302,7 @@ func (b *lxdBackend) GetInstanceUsage(inst instance.Instance) (*VolumeUsage, err // SetInstanceQuota sets the quota on the instance's root volume. // Returns ErrInUse if the instance is running and the storage driver doesn't support online resizing. -func (b *lxdBackend) SetInstanceQuota(inst instance.Instance, size string, vmStateSize string, op *operations.Operation) error { +func (b *lxdBackend) SetInstanceQuota(inst instance.Instance, size string, vmStateSize string, op *operations.Operation) (err error) { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "size": size, "vm_state_size": vmStateSize}) l.Debug("SetInstanceQuota started") defer l.Debug("SetInstanceQuota finished") @@ -3322,6 +3322,18 @@ func (b *lxdBackend) SetInstanceQuota(inst instance.Instance, size string, vmSta return err } + // If the volume resize succeeds, update the instance volume's 'size' config on the database. + defer func() { + if err == nil { + newConfig := dbVol.Config + newConfig["volatile.rootfs.size"] = size + + err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { + return tx.UpdateStoragePoolVolumeConfig(dbVol.Name, dbVol.ID, newConfig) + }) + } + }() + // Apply the main volume quota. vol := b.GetVolume(volType, contentVolume, volStorageName, dbVol.Config) err = b.driver.SetVolumeQuota(vol, size, false, op) From 5bfe5521fc39923377df5e1ba2fcd133dfcbf8f9 Mon Sep 17 00:00:00 2001 From: hamistao Date: Wed, 4 Dec 2024 04:00:09 -0300 Subject: [PATCH 7/9] lxd/storage: Derive disk size from volume for running instances For running instances, the instance volume 'size' config key is the most accurate representation of the instance root disk size. The device's own 'size' may be inaccurate when the instance does not support online resizing. When getting instance disk total size of a stopped instance, you want to see what is the size of the root disk you will be dealing with when you start the instance, even if the volume quota is only applied on instance start. Signed-off-by: hamistao --- lxd/storage/backend_lxd.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/lxd/storage/backend_lxd.go b/lxd/storage/backend_lxd.go index 17936bc74176..b1b33a24ada0 100644 --- a/lxd/storage/backend_lxd.go +++ b/lxd/storage/backend_lxd.go @@ -3271,14 +3271,27 @@ func (b *lxdBackend) GetInstanceUsage(inst instance.Instance) (*VolumeUsage, err // If driver does not support getting volume usage, this value would be -1. val.Used = size - // Get the total size. - _, rootDiskConf, err := instancetype.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) - if err != nil { - return nil, err + // The instance volume 'volatile.rootfs.size' config key is the most accurate representation of the instance root disk size. + // The device's own 'volatile.rootfs.size' is not influenced by 'volume.size' on the pool level and may be inaccurate when + // the instance does not support online resizing. + sizeStr := dbVol.Config["volatile.rootfs.size"] + + // If the instance isn't running, return the value its volume will assume when it starts even if the + // volume quota hasn't been set yet. + if !inst.IsRunning() { + // Get the total size. + _, rootDiskConf, err := instancetype.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) + if err != nil { + return nil, err + } + + rootDiskSize, ok := rootDiskConf["size"] + if ok { + sizeStr = rootDiskSize + } } - sizeStr, ok := rootDiskConf["size"] - if ok { + if sizeStr != "" { total, err := units.ParseByteSizeString(sizeStr) if err != nil { return nil, err From eef859b997fddf860018abd1590618ec2198b415 Mon Sep 17 00:00:00 2001 From: hamistao Date: Thu, 5 Dec 2024 11:54:25 -0300 Subject: [PATCH 8/9] lxd/storage_volumes_state: Return actual volume size for stopped intances When getting instance disk total size of a stopped instance, you want to see what size is the root disk you will be dealing with when you start it. But it could be the case that the current volume size is different from the instance configuration as the volume when gets resizes (if needed) when the instance starts. So for the volume state, we always get the actual current size of the volume. Signed-off-by: hamistao --- lxd/storage_volumes_state.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lxd/storage_volumes_state.go b/lxd/storage_volumes_state.go index 2027f4e254ea..0610191296b5 100644 --- a/lxd/storage_volumes_state.go +++ b/lxd/storage_volumes_state.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" "github.com/gorilla/mux" @@ -146,6 +147,27 @@ func storagePoolVolumeTypeStateGet(d *Daemon, r *http.Request) response.Response if err != nil && err != storageDrivers.ErrNotSupported { return response.SmartError(err) } + + // If the instance is stopped and its size was recently changed, the volume may not have been resized yet + // as this only happens on instance start. + // When this is the case, fallback to getting the total size from the volume's config. + if !inst.IsRunning() { + volType, err := storagePools.InstanceTypeToVolumeType(inst.Type()) + if err != nil { + return response.SmartError(err) + } + + // Load storage volume from database. + dbVol, err := storagePools.VolumeDBGet(pool, inst.Project().Name, inst.Name(), volType) + if err != nil { + return response.SmartError(err) + } + + // If size is not set, move on with total 0. + volumeSize, _ := strconv.ParseInt(dbVol.Config["volatile.rootfs.size"], 10, 64) + + usage.Total = volumeSize + } } // Prepare the state struct. From 4b13a2a49df83375a9717ab2835232e6f3fba59d Mon Sep 17 00:00:00 2001 From: hamistao Date: Mon, 25 Nov 2024 23:40:54 -0300 Subject: [PATCH 9/9] test: Test getting instance size Signed-off-by: hamistao --- test/suites/storage.sh | 77 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/test/suites/storage.sh b/test/suites/storage.sh index ea49cef64963..fe80341e243a 100644 --- a/test/suites/storage.sh +++ b/test/suites/storage.sh @@ -952,6 +952,15 @@ EOF LXD_DIR="${LXD_DIR}" storage_pool="lxdtest-$(basename "${LXD_DIR}")-pool26" lxc storage create "$storage_pool" "$lxd_backend" + + if [ "${lxd_backend}" = "zfs" ]; then + get_instance_size "zfs" + lxc storage set "$storage_pool" volume.zfs.block_mode true + get_instance_size "zfs-block" + else + get_instance_size "${lxd_backend}" + fi + lxc init -s "${storage_pool}" testimage c1 # The storage pool will not be removed since it has c1 attached to it ! lxc storage delete "${storage_pool}" || false @@ -1025,3 +1034,71 @@ EOF LXD_DIR="${LXD_DIR}" kill_lxd "${LXD_STORAGE_DIR}" } + +get_instance_size() { + driver="{$1}" + + lxc init -s "${storage_pool}" testimage c1 + # Set 'volume.size' and create new container + lxc storage set "${storage_pool}" volume.size="7GiB" + lxc init -s "${storage_pool}" testimage c2 + + # Non block-based drivers. + if [ "${driver}" = "btrfs" ] || [ "${driver}" = "dir" ] || [ "${driver}" = "zfs" ]; then + # Client doesn't show totals for unbounded containers + [ -z "$(lxc info c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" ] + [ -z "$(lxc info c2 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" ] + # Check volume also has empty 'volatile.rootfs.size' and volume state also don't show total + [ -z "$(lxc storage volume get container/c1 volatile.rootfs.size)" ] + ! lxc storage volume info container/c1 | grep "Total:" || false + [ -z "$(lxc storage volume get container/c2 volatile.rootfs.size)" ] + ! lxc storage volume info container/c2 | grep "Total:" || false + fi + + # Block-based drivers + if [ "${driver}" = "lvm" ] || [ "${driver}" = "ceph" ] || [ "${driver}" = "zfs-block" ]; then + [ "$(lxc info c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "10GiB" ] + [ "$(lxc info c2 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "7GiB" ] + + # Check volume 'volatile.rootfs.size' and volume state also show correct total size + [ "$(lxc storage volume get container/c1 volatile.rootfs.size)" = "10GiB" ] + [ "$(lxc storage volume info container/c1 | grep "Total:")" = "10GiB" ] + [ "$(lxc storage volume get container/c2 volatile.rootfs.size)" = "7GiB" ] + [ "$(lxc storage volume info container/c2 | grep "Total:")" = "7GiB" ] + fi + + # Shrink c1 and grow c2 + # Device 'size' should override default size and 'volume.size' + lxc start c1 + lxc start c2 + ! lxc storage volume set container/c1 size="7GiB" || false + lxc config device set c1 root size="7GiB" + lxc config device set c2 root size="10GiB" + + # Non block-based drivers. + if [ "${driver}" = "btrfs" ] || [ "${driver}" = "dir" ] || [ "${driver}" = "zfs" ]; then + # Device 'size' always applies right away + [ "$(lxc info c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "7GiB" ] + [ "$(lxc info c2 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "10GiB" ] + + # Also visible from volume 'volatile.rootfs.size' config and volume state + [ "$(lxc storage volume get container/c1 volatile.rootfs.size)" = "7GiB" ] + [ "$(lxc storage volume info container/c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "7GiB" ] + fi + + # Block-based drivers + if [ "${driver}" = "lvm" ] || [ "${driver}" = "ceph" ] || [ "${driver}" = "zfs-block" ]; then + [ "$(lxc info c2 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "7GiB" ] + # Shrinking does not apply while instance is running + [ "$(lxc info c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "10GiB" ] + lxc stop -f c1 + # While instance is stopped, show new instance root disk size even though + # The volume itself only gets resized on instance start + [ "$(lxc info c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "7GiB" ] + [ "$(lxc storage volume get container/c1 volatile.rootfs.size)" = "7GiB" ] + lxc start c1 + [ "$(lxc info c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "7GiB" ] + fi + + lxc delete -f c1 c2 +}