new feature qemu-scheduler

Makefile

@@ -57,8 +57,9 @@ generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
 .PHONY: fmt
-fmt: ## Run go fmt against code.
+fmt: goimports ## Run go fmt against code.
 	go fmt ./...
+	$(GOIMPORTS) -w ./
 
 .PHONY: vet
 vet: ## Run go vet against code.
@@ -232,6 +233,7 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest
 ENVSUBST ?= $(LOCALBIN)/envsubst
 KUBECTL ?= $(LOCALBIN)/kubectl
 GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint
+GOIMPORTS ?= $(LOCALBIN)/goimports
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.0.0
@@ -280,3 +282,8 @@ $(SETUP_ENVTEST): go.mod # Build setup-envtest from tools folder.
 golangci-lint: $(GOLANGCI_LINT)
 $(GOLANGCI_LINT): $(LOCALBIN)
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(LOCALBIN) v1.54.0
+
+.PHONY: goimports
+goimports: $(GOIMPORTS)
+$(GOIMPORTS): $(LOCALBIN)
+	GOBIN=$(LOCALBIN) go install golang.org/x/tools/cmd/goimports@latest

README.md

@@ -63,6 +63,8 @@ kubectl delete cluster cappx-test
 
 - Supports custom cloud-config (user data). CAPPX uses VNC websockert for bootstrapping nodes so it can applies custom cloud-config that can not be achieved by only Proxmox API.
 
+- Flexible vmid/node assigning. You can flexibly assign vmid to your qemu and flexibly schedule qemus to proxmox nodes. For more details please check [qemu-scheduler](./cloud/scheduler/).
+
 ### Node Images
 
 CAPPX is compatible with `iso`, `qcow2`, `qed`, `raw`, `vdi`, `vpc`, `vmdk` format of image. You can build your own node image and use it for `ProxmoxMachine`.

cloud/interfaces.go

@@ -8,6 +8,7 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
 	infrav1 "github.com/sp-yduck/cluster-api-provider-proxmox/api/v1beta1"
+	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler"
 )
 
 type Reconciler interface {
@@ -45,8 +46,10 @@ type ClusterSettter interface {
 // MachineGetter is an interface which can get machine information.
 type MachineGetter interface {
 	Client
+	GetScheduler(client *proxmox.Service) *scheduler.Scheduler
 	Name() string
 	Namespace() string
+	Annotations() map[string]string
 	// Zone() string
 	// Role() string
 	// IsControlPlane() bool

cloud/scheduler/README.md

@@ -0,0 +1,67 @@
+# qemu-scheduler
+
+Scheduling refers to making sure that VM(QEMU) are matched to Proxmox Nodes.
+
+## How qemu-scheduler select proxmox node to run qemu
+
+Basic flow of the node selection process is `filter => score => select one node which has highest score`
+
+### Filter Plugins
+
+Filter plugins filter the node based on nodename, overcommit ratio etc.
+
+#### regex plugin
+
+Regex plugin is a one of the default Filter Plugin of qemu-scheduler. You can specify node name as regex format. 
+```sh
+key: node.qemu-scheduler/regex
+value(example): node[0-9]+
+```
+
+### Score Plugins
+
+Score plugins score the nodes based on resource etc.
+
+## How to specify vmid
+qemu-scheduler reads context and find key registerd to scheduler. If the context has any value of the registerd key, qemu-scheduler uses the plugin that matchies the key.
+
+### Range Plugin
+You can specify vmid range with `(start id)-(end id)` format.
+```sh
+key: vmid.qemu-scheduler/range
+value(example): 100-150
+```
+
+### Regex Plugin
+```sh
+key: vmid.qemu-scheduler/regex
+value(example): (12[0-9]|130)
+```
+
+## How qemu-scheduler works with CAPPX
+CAPPX passes all the annotation (of `ProxmoxMachine`) key-values to scheduler's context. So if you will use Range Plugin for your `ProxmoxMachine`, your manifest must look like following.
+```sh
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: ProxmoxMachine
+metadata:
+    name: sample-machine
+    annotations:
+        vmid.qemu-scheduler/range: 100-150 # this means your vmid will be chosen from the range of 100 to 150.
+```
+
+Also, you can specifies these annotations via `MachineDeployment` since Cluster API propagates some metadatas (ref: [metadata-propagation](https://cluster-api.sigs.k8s.io/developer/architecture/controllers/metadata-propagation.html#metadata-propagation)).
+
+For example, your `MachineDeployment` may look like following.
+```sh
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  annotations:
+    caution: "# do not use here, because this annotation won't be propagated to your ProxmoxMachine"
+  name: sample-machine-deployment
+spec:
+  template:
+    metadata:
+      annotations:
+        node.qemu-scheduler/regex: node[0-9]+ # this annotation will be propagated to your ProxmoxMachine via MachineSet
+```

cloud/scheduler/framework/cycle_state.go

@@ -0,0 +1,77 @@
+package framework
+
+import (
+	"github.com/sp-yduck/proxmox-go/api"
+	"github.com/sp-yduck/proxmox-go/proxmox"
+)
+
+type CycleState struct {
+	completed bool
+	err       error
+	messages  map[string]string
+	result    SchedulerResult
+}
+
+type SchedulerResult struct {
+	vmid     int
+	node     string
+	instance *proxmox.VirtualMachine
+}
+
+func NewCycleState() CycleState {
+	return CycleState{completed: false, err: nil, messages: map[string]string{}}
+}
+
+func (c *CycleState) SetComplete() {
+	c.completed = true
+}
+
+func (c *CycleState) IsCompleted() bool {
+	return c.completed
+}
+
+func (c *CycleState) SetError(err error) {
+	c.err = err
+}
+
+func (c *CycleState) Error() error {
+	return c.err
+}
+
+func (c *CycleState) SetMessage(pluginName, message string) {
+	c.messages[pluginName] = message
+}
+
+func (c *CycleState) Messages() map[string]string {
+	return c.messages
+}
+
+func (c *CycleState) QEMU() *api.VirtualMachine {
+	return c.result.instance.VM
+}
+
+func (c *CycleState) UpdateState(completed bool, err error, result SchedulerResult) {
+	c.completed = completed
+	c.err = err
+	c.result = result
+}
+
+func NewSchedulerResult(vmid int, node string, instance *proxmox.VirtualMachine) SchedulerResult {
+	return SchedulerResult{vmid: vmid, node: node, instance: instance}
+}
+
+func (c *CycleState) Result() SchedulerResult {
+	return c.result
+}
+
+func (r *SchedulerResult) Node() string {
+	return r.node
+}
+
+func (r *SchedulerResult) VMID() int {
+	return r.vmid
+}
+
+func (r *SchedulerResult) Instance() *proxmox.VirtualMachine {
+	return r.instance
+}

cloud/scheduler/framework/interface.go

@@ -0,0 +1,28 @@
+package framework
+
+import (
+	"context"
+
+	"github.com/sp-yduck/proxmox-go/api"
+)
+
+type Plugin interface {
+	// return plugin name
+	Name() string
+}
+
+type NodeFilterPlugin interface {
+	Plugin
+	Filter(ctx context.Context, state *CycleState, config api.VirtualMachineCreateOptions, nodeInfo *NodeInfo) *Status
+}
+
+type NodeScorePlugin interface {
+	Plugin
+	Score(ctx context.Context, state *CycleState, config api.VirtualMachineCreateOptions, nodeInfo *NodeInfo) (int64, *Status)
+}
+
+type VMIDPlugin interface {
+	Plugin
+	PluginKey() CtxKey
+	Select(ctx context.Context, state *CycleState, config api.VirtualMachineCreateOptions, nextid int, usedID map[int]bool) (int, error)
+}

cloud/scheduler/framework/suite_test.go

@@ -0,0 +1,45 @@
+package framework_test
+
+import (
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/sp-yduck/proxmox-go/proxmox"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+)
+
+var (
+	proxmoxSvc *proxmox.Service
+)
+
+func TestFrameworks(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Scheduler Framework Suite")
+}
+
+var _ = BeforeSuite(func() {
+	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+
+	if GinkgoLabelFilter() != "unit" {
+		By("setup proxmox client to do integration test")
+		url := os.Getenv("PROXMOX_URL")
+		user := os.Getenv("PROXMOX_USER")
+		password := os.Getenv("PROXMOX_PASSWORD")
+		tokenid := os.Getenv("PROXMOX_TOKENID")
+		secret := os.Getenv("PROXMOX_SECRET")
+
+		authConfig := proxmox.AuthConfig{
+			Username: user,
+			Password: password,
+			TokenID:  tokenid,
+			Secret:   secret,
+		}
+		param := proxmox.NewParams(url, authConfig, proxmox.ClientConfig{InsecureSkipVerify: true})
+		var err error
+		proxmoxSvc, err = proxmox.GetOrCreateService(param)
+		Expect(err).NotTo(HaveOccurred())
+	}
+})

cloud/scheduler/framework/types.go

@@ -0,0 +1,91 @@
+package framework
+
+import (
+	"context"
+
+	"github.com/sp-yduck/proxmox-go/api"
+	"github.com/sp-yduck/proxmox-go/proxmox"
+)
+
+type Status struct {
+	code         int
+	reasons      []string
+	err          error
+	failedPlugin string
+}
+
+func NewStatus() *Status {
+	return &Status{code: 0}
+}
+
+func (s *Status) Code() int {
+	return s.code
+}
+
+func (s *Status) SetCode(code int) {
+	s.code = code
+}
+
+func (s *Status) Reasons() []string {
+	if s.err != nil {
+		return append([]string{s.err.Error()}, s.reasons...)
+	}
+	return s.reasons
+}
+
+func (s *Status) FailedPlugin() string {
+	return s.failedPlugin
+}
+
+func (s *Status) SetFailedPlugin(name string) {
+	s.failedPlugin = name
+}
+
+func (s *Status) IsSuccess() bool {
+	return s.code == 0
+}
+
+func (s *Status) Error() error {
+	return s.err
+}
+
+// NodeInfo is node level aggregated information
+type NodeInfo struct {
+	node *api.Node
+
+	// qemus assigned to the node
+	qemus []*api.VirtualMachine
+}
+
+func GetNodeInfoList(ctx context.Context, client *proxmox.Service) ([]*NodeInfo, error) {
+	nodes, err := client.Nodes(ctx)
+	if err != nil {
+		return nil, err
+	}
+	nodeInfos := []*NodeInfo{}
+	for _, node := range nodes {
+		qemus, err := client.RESTClient().GetVirtualMachines(ctx, node.Node)
+		if err != nil {
+			return nil, err
+		}
+		nodeInfos = append(nodeInfos, &NodeInfo{node: node, qemus: qemus})
+	}
+	return nodeInfos, nil
+}
+
+func (n NodeInfo) Node() *api.Node {
+	return n.node
+}
+
+func (n NodeInfo) QEMUs() []*api.VirtualMachine {
+	return n.qemus
+}
+
+// NodeScoreList declares a list of nodes and their scores.
+type NodeScoreList []NodeScore
+
+// NodeScore is a struct with node name and score.
+type NodeScore struct {
+	Name  string
+	Score int64
+}

cloud/scheduler/framework/types_test.go

@@ -0,0 +1,21 @@
+package framework_test
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/framework"
+)
+
+var _ = Describe("GetNodeInfoList", Label("integration", "framework"), func() {
+	ctx := context.Background()
+
+	It("should not error", func() {
+		nodes, err := framework.GetNodeInfoList(ctx, proxmoxSvc)
+		Expect(err).To(BeNil())
+		Expect(len(nodes)).ToNot(Equal(0))
+	})
+
+})