From 1cf0efb41368a82ceddacf1cf223e9b2badef1cf Mon Sep 17 00:00:00 2001
From: ks-ci-bot <47586280+ks-ci-bot@users.noreply.github.com>
Date: Thu, 18 Apr 2024 01:03:20 +0000
Subject: [PATCH 01/31] Add new kubernetes version

---
 docs/kubernetes-versions.md |  4 ++++
 version/components.json     | 38 ++++++++++++++++++++++++++++++-------
 2 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/docs/kubernetes-versions.md b/docs/kubernetes-versions.md
index d98d3982a..6cbbc67bb 100644
--- a/docs/kubernetes-versions.md
+++ b/docs/kubernetes-versions.md
@@ -123,6 +123,7 @@
 | v1.27.10 | :white_check_mark: |
 | v1.27.11 | :white_check_mark: |
 | v1.27.12 | :white_check_mark: |
+| v1.27.13 | :white_check_mark: |
 | v1.28.0 | :white_check_mark: |
 | v1.28.1 | :white_check_mark: |
 | v1.28.2 | :white_check_mark: |
@@ -132,7 +133,10 @@
 | v1.28.6 | :white_check_mark: |
 | v1.28.7 | :white_check_mark: |
 | v1.28.8 | :white_check_mark: |
+| v1.28.9 | :white_check_mark: |
 | v1.29.0 | :white_check_mark: |
 | v1.29.1 | :white_check_mark: |
 | v1.29.2 | :white_check_mark: |
 | v1.29.3 | :white_check_mark: |
+| v1.29.4 | :white_check_mark: |
+| v1.30.0 | :white_check_mark: |
diff --git a/version/components.json b/version/components.json
index 84a22b92c..72bccef3a 100644
--- a/version/components.json
+++ b/version/components.json
@@ -123,6 +123,7 @@
             "v1.27.10": "23985e958443ac1aabdbeeedc675358abc0638eb580707829fd42b0996a0aae5",
             "v1.27.11": "31bf446a712fb08190838c35d1f4c93b0f975708c59634a5dc3d8915a241c83e",
             "v1.27.12": "06ee36cc80cfdfc01c937d750783d3ca6169a3da76382c7af3dd172d9f6bfa4e",
+            "v1.27.13": "b88c30b7067f095b7fa02c5560cc50d6e69a5a9fecc606ef477dc7efc86453b9",
             "v1.28.0": "12ea68bfef0377ccedc1a7c98a05ea76907decbcf1e1ec858a60a7b9b73211bb",
             "v1.28.1": "6134dbc92dcb83c3bae1a8030f7bb391419b5d13ea94badd3a79b7ece75b2736",
             "v1.28.2": "6a4808230661c69431143db2e200ea2d021c7f1b1085e6353583075471310d00",
@@ -132,10 +133,13 @@
             "v1.28.6": "bda3eda8d51e8746a42b535b7eab7df52b091a796227c3212dc30909a8f1b431",
             "v1.28.7": "8aa005bdf6af43e47fc818b26f4cb9f361aae8ec4390519e8d4033be65fbef2b",
             "v1.28.8": "c11946cbfd962e1197062534514226cfd70230349e6343ff3ecebfca5476ee64",
+            "v1.28.9": "a4d8acf0a74cb1d07d96a1a34148f54c6420874221af16d8ec902d9bffc7ef89",
             "v1.29.0": "629d4630657caace9c819fd3797f4a70c397fbd41a2a7e464a0507dad675d52c",
             "v1.29.1": "d4d81d9020b550c896376fb9e0586a9f15a332175890d061619b52b3e9bc6cbd",
             "v1.29.2": "2d4e4fa8685bcbfb661cb41050cd4756f50a7aa147f68492d51a99f9cdfd69ac",
-            "v1.29.3": "6abaa1208bf40b6d1f49e518bd68c8ae4a1be0c5b7d3e45d87979999ab070d8b"
+            "v1.29.3": "6abaa1208bf40b6d1f49e518bd68c8ae4a1be0c5b7d3e45d87979999ab070d8b",
+            "v1.29.4": "ea20ab064f716ab7f69a36d72df340257b31c9721ea86e1cf9d70b35999ddeea",
+            "v1.30.0": "29f4232c50e6524abba3443ff3b9948d386964d79eb8dfefb409e1f8a8434c14"
         },
         "arm64": {
             "v1.19.0": "db1c432646e6e6484989b6f7191f3610996ac593409f12574290bfc008ea11f5",
@@ -260,6 +264,7 @@
             "v1.27.10": "ed0447155a7e967ae23480b06b31b2c0aaa871e7c59dfd82ae25b03a1eccf6e6",
             "v1.27.11": "b8452d6c3f1331beb3d5fa42466a9bc96638a76c40980dba9822300f230c0858",
             "v1.27.12": "e74d47c14b5a251cff961dcce92cd632abcfd0fba4a07e78f0a5a5b2796e4b84",
+            "v1.27.13": "f334ba0612fada50e98a7ea56b686b35c22f0e3243ec2210f2a6a87e841a139f",
             "v1.28.0": "b9b473d2d9136559b19eb465006af77df45c09862cd7ce6673a33aae517ff5ab",
             "v1.28.1": "7d2f68917470a5d66bd2a7d62897f59cb4afaeffb2f26c028afa119acd8c3fc8",
             "v1.28.2": "010789a94cf512d918ec4a3ef8ec734dea0061d89a8293059ef9101ca1bf6bff",
@@ -269,10 +274,13 @@
             "v1.28.6": "4298cad464e92eec19cdf3e6a607a82a1d626ae70fedba7956175152ab983457",
             "v1.28.7": "f556e49494737f97a15bf15bb4b27d45f8747b477302cdfd22dd61816bc02203",
             "v1.28.8": "e0f47adc69ef84e2f6c42cc341b8a790904a929ad10ed1c23c2e822ec804e247",
+            "v1.28.9": "cd6aefad8144a9771fd470529ff14be2675df7b561f7c56dee3fed4f81332dc4",
             "v1.29.0": "bbddee2d46d2e1643ae3623698b45b13aa2e858616d61c642f2f49e5bb14c980",
             "v1.29.1": "3bff8c50c104c45e416cce9991706c6ac46365f0defbcd54f8cf4ace0fa68dcf",
             "v1.29.2": "e05720feb9d2d67eff25b0156a5c22e2de37be2ffab4e1f4d31e8c526fafd0e1",
-            "v1.29.3": "ce2e4c230f954e59ae77e34c4ff2ae08cad3970505ae1e21b6337e6d83b21682"
+            "v1.29.3": "ce2e4c230f954e59ae77e34c4ff2ae08cad3970505ae1e21b6337e6d83b21682",
+            "v1.29.4": "438287a91e08cbefecab79be8ac893a935c3dbf6e87bea895fb99f2bc38cf06e",
+            "v1.30.0": "c36afd28921303e6db8e58274de16c60a80a1e75030fc3c4e9c4ed6249b6b696"
         }
     },
     "kubelet": {
@@ -399,6 +407,7 @@
             "v1.27.10": "25a34bf98bb8a296ea07f1ebbcb496b1e6b6c6da3247695288a7c99fc8c1be2c",
             "v1.27.11": "2ce92a5d8985b93bd8ffc4f5519cd79bf2f844590aa38228a3d809c5bf5986e0",
             "v1.27.12": "aae861a21913c274228ccdad1609b370e5198c9f4b39b8924b20a7ffe7f148e0",
+            "v1.27.13": "ed68df2a77f3057ab47f57eacb6e9310e91731e4f43c58a3c3b5c857d78d0080",
             "v1.28.0": "bfb6b977100963f2879a33e5fbaa59a5276ba829a957a6819c936e9c1465f981",
             "v1.28.1": "2bc22332f44f8fcd3fce57879fd873f977949ebd261571fbae31fbb2713a5dd3",
             "v1.28.2": "17edb866636f14eceaad58c56eab12af7ab3be3c78400aff9680635d927f1185",
@@ -408,10 +417,13 @@
             "v1.28.6": "8506df1f20a5f8bba0592f5a4cf5d0cc541047708e664cb88580735400d0b26f",
             "v1.28.7": "120b1495babc4364f7e16a9d0f8b8e6b6f78316d047e4f6de77b5569b05813c7",
             "v1.28.8": "049b412a5861255cd3922f612acb79ab51135e166c5d80acf12fba9179eebf0c",
+            "v1.28.9": "f3af46cff11c675a80d91ebb38ebc4e85a9f813ce93e56ee131e7fea1491b786",
             "v1.29.0": "e1c38137db8d8777eed8813646b59bf4d22d19b9011ab11dc28e2e34f6b80a05",
             "v1.29.1": "1b1975c58d38be1a99a8bcba4564ac489afd223b0abe9f2ab08bbde89d2412a3",
             "v1.29.2": "f71a85039b71fe08f1c063a93d61a1c952dc8f9a8c6be9b13fbdac8f0d9ff960",
-            "v1.29.3": "d8b55a2f8a87c8cd2cbf867d76d1d7f98b7198a740db19bad6ed7b8b813de771"
+            "v1.29.3": "d8b55a2f8a87c8cd2cbf867d76d1d7f98b7198a740db19bad6ed7b8b813de771",
+            "v1.29.4": "58571f0ed62543a9bbac541e52c15d8385083113a463e23aec1341d0b5043939",
+            "v1.30.0": "32a32ec3d7e7f8b2648c9dd503ce9ef63b4af1d1677f5b5aed7846fb02d66f18"
         },
         "arm64": {
             "v1.19.0": "d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf",
@@ -536,6 +548,7 @@
             "v1.27.10": "0edadc44ef36be8d8106cad9972360c0477540e2d8c0bbeb38fd97fd1d7801d5",
             "v1.27.11": "e81987a864fb47afe14f65fa4e93760bc19c424335e0f0540c6c725b727ce22a",
             "v1.27.12": "0d7d2d25c8b909d6cec7c1c2a5bfe51428ec33eaa5e8b209c718b77983e9dcba",
+            "v1.27.13": "d7bfb14d0b0fc2c41074baf02617cf98589fd029fb3539ea017825e36371f19c",
             "v1.28.0": "05dd12e35783cab4960e885ec0e7d0e461989b94297e7bea9018ccbd15c4dce9",
             "v1.28.1": "9b7fa64b2785da4a38768377961e227f8da629c56a5df43ca1b665dd07b56f3c",
             "v1.28.2": "32269e9ec38c561d028b65c3048ea6a100e1292cbe9e505565222455c8096577",
@@ -545,10 +558,13 @@
             "v1.28.6": "ee2c060deff330d3338e24aec9734c9e5d5aea4fea1905c0795bccff6997a65e",
             "v1.28.7": "e2c98b39b0b0745ef3e30febaeb8eaaf31ec721012405bd0dcf25e84026c221e",
             "v1.28.8": "90d61f40b7bb061b0fc6d08b8b9ddae51f90863c899b098e19eaa89dc855f2c0",
+            "v1.28.9": "312471ad255acfcdeea2c5849b171467af4518e96d69d727a3197ff334e9299d",
             "v1.29.0": "0e0e4544c2a0a3475529154b7534d0d58683466efa04a2bb2e763b476db0bb16",
             "v1.29.1": "e46417ab1ceae995f0e00d4177959a36ed34b807829422bc9dda70b263fe5c5d",
             "v1.29.2": "9b4aa572d4cd51a41b1067161d961423d0d12b120fb636ea887a12a975d4b19a",
-            "v1.29.3": "891dce19ed0eae34050c2eca0454204892e97bfe1a926f988cd044a987a9c7c9"
+            "v1.29.3": "891dce19ed0eae34050c2eca0454204892e97bfe1a926f988cd044a987a9c7c9",
+            "v1.29.4": "dc4bb6ea6cd35b024d63cc20d1c1800a9c695bd6f70411c57358d7c407513b00",
+            "v1.30.0": "fa887647422d34f3c7cc5b30fefcf97084d2c3277eff237c5808685ba8e4b15a"
         }
     },
     "kubectl": {
@@ -675,6 +691,7 @@
             "v1.27.10": "bfb219643c28d9842fceae51590776f06987835d93fc3cb9b0149c9111c741ac",
             "v1.27.11": "7ae327978a1edb43700070c86f5fd77215792c6b58a7ea70192647e0da848e29",
             "v1.27.12": "d639eda39be2dce42fbec21e038942ab5734541715e3ea5fb29c9ad76686bd7f",
+            "v1.27.13": "e991f163197cbd85bbff22f656a74d48b69db5addfa43cc04cca0cf5328f57f1",
             "v1.28.0": "4717660fd1466ec72d59000bb1d9f5cdc91fac31d491043ca62b34398e0799ce",
             "v1.28.1": "e7a7d6f9d06fab38b4128785aa80f65c54f6675a0d2abef655259ddd852274e1",
             "v1.28.2": "c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5",
@@ -684,10 +701,13 @@
             "v1.28.6": "c8351fe0611119fd36634dd3f53eb94ec1a2d43ef9e78b92b4846df5cc7aa7e3",
             "v1.28.7": "aff42d3167685e4d8e86fda0ad9c6ce6ec6c047bc24d608041d54717a18192ba",
             "v1.28.8": "e02aad5c0bac52c970700b814645b62c4f18b634144398ac344875dbaf1072f8",
+            "v1.28.9": "b4693d0b22f509250694b10c7727c42b427d570af04f2065fe23a55d6c0051f1",
             "v1.29.0": "0e03ab096163f61ab610b33f37f55709d3af8e16e4dcc1eb682882ef80f96fd5",
             "v1.29.1": "69ab3a931e826bf7ac14d38ba7ca637d66a6fcb1ca0e3333a2cafdf15482af9f",
             "v1.29.2": "7816d067740f47f949be826ac76943167b7b3a38c4f0c18b902fffa8779a5afa",
-            "v1.29.3": "89c0435cec75278f84b62b848b8c0d3e15897d6947b6c59a49ddccd93d7312bf"
+            "v1.29.3": "89c0435cec75278f84b62b848b8c0d3e15897d6947b6c59a49ddccd93d7312bf",
+            "v1.29.4": "10e343861c3cb0010161e703307ba907add2aeeeaffc6444779ad915f9889c88",
+            "v1.30.0": "7c3807c0f5c1b30110a2ff1e55da1d112a6d0096201f1beb81b269f582b5d1c5"
         },
         "arm64": {
             "v1.19.0": "d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f",
@@ -812,6 +832,7 @@
             "v1.27.10": "2e1996379d5a8b132e0606fcd3df3c8689e11882630b75cca3b7135126847871",
             "v1.27.11": "d30e1aa873e78eb376ddee3c785aa78c44eddc56ce2ef901dac1ce0c2c4f50b0",
             "v1.27.12": "bfc6cb71041ebc0f048402988eccc107cfff2b866c864231c9ada05ab328e5bf",
+            "v1.27.13": "4838ad8f3902c928f6139d69eba962b1cc2471511c828885650d728d970594d1",
             "v1.28.0": "f5484bd9cac66b183c653abed30226b561f537d15346c605cc81d98095f1717c",
             "v1.28.1": "46954a604b784a8b0dc16754cfc3fa26aabca9fd4ffd109cd028bfba99d492f6",
             "v1.28.2": "ea6d89b677a8d9df331a82139bb90d9968131530b94eab26cee561531eff4c53",
@@ -821,10 +842,13 @@
             "v1.28.6": "0de705659a80c3fef01df43cc0926610fe31482f728b0f992818abd9bdcd2cb9",
             "v1.28.7": "13d547495bdea49b223fe06bffb6d2bef96436634847f759107655aa80fc990e",
             "v1.28.8": "93d60dd36093b4c719f1f1bafcf59437c17cb2209341c7c94771e7dd9acdab33",
+            "v1.28.9": "e0341d3973213f8099e7fcbbf6d1d506967bc2b7a4faac3fb3b4340f226e9b2f",
             "v1.29.0": "8f7a4bd6bae900a4ddab12bd1399aa652c0d59ea508f39b910e111d248893ff7",
             "v1.29.1": "96d6dc7b2bdcd344ce58d17631c452225de5bbf59b83fd3c89c33c6298fb5d8b",
             "v1.29.2": "3507ecb4224cf05ae2151a98d4932253624e7762159936d5347b19fe037655ca",
-            "v1.29.3": "191a96b27e3c6ae28b330da4c9bfefc9592762670727df4fcf124c9f1d5a466a"
+            "v1.29.3": "191a96b27e3c6ae28b330da4c9bfefc9592762670727df4fcf124c9f1d5a466a",
+            "v1.29.4": "61537408eedcad064d7334384aed508a8aa1ea786311b87b505456a2e0535d36",
+            "v1.30.0": "669af0cf520757298ea60a8b6eb6b719ba443a9c7d35f36d3fb2fd7513e8c7d2"
         }
     },
     "etcd": {
@@ -1059,4 +1083,4 @@
             "v3.27.3": "1fc5f58a18d8b1c487b4663fc5cbe23b45bd9d31617debd309f6dfac7c11a8ef"
         }
     }
-}
+}
\ No newline at end of file

From 0d36fbdce7a6b03fc7cfde546514f772c4c6241f Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Thu, 1 Feb 2024 13:47:15 +0800
Subject: [PATCH 02/31] feat: add CustomScripts.Role property

---
 cmd/kk/apis/kubekey/v1alpha2/cluster_types.go |  1 +
 cmd/kk/pkg/bootstrap/customscripts/module.go  | 11 ++++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go
index b5b3bc2af..e5b442055 100644
--- a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go
+++ b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go
@@ -89,6 +89,7 @@ type KubeVip struct {
 type CustomScripts struct {
 	Name      string   `yaml:"name" json:"name,omitempty"`
 	Bash      string   `yaml:"bash" json:"bash,omitempty"`
+	Role      string   `yaml:"role" json:"role,omitempty"`
 	Materials []string `yaml:"materials" json:"materials,omitempty"`
 }
 
diff --git a/cmd/kk/pkg/bootstrap/customscripts/module.go b/cmd/kk/pkg/bootstrap/customscripts/module.go
index 1c98a178d..8873b328d 100644
--- a/cmd/kk/pkg/bootstrap/customscripts/module.go
+++ b/cmd/kk/pkg/bootstrap/customscripts/module.go
@@ -18,6 +18,7 @@ package customscripts
 
 import (
 	"fmt"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
 
 	kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module"
@@ -38,10 +39,18 @@ func (m *CustomScriptsModule) Init() {
 
 		taskName := fmt.Sprintf("Phase:%s(%d/%d) script:%s", m.Phase, idx, len(m.Scripts), script.Name)
 		taskDir := fmt.Sprintf("%s-%d-script", m.Phase, idx)
+
+		var hosts []connector.Host
+		if len(script.Role) > 0 {
+			hosts = m.Runtime.GetHostsByRole(script.Role)
+		} else {
+			hosts = m.Runtime.GetAllHosts()
+		}
+
 		task := &task.RemoteTask{
 			Name:     taskName,
 			Desc:     taskName,
-			Hosts:    m.Runtime.GetAllHosts(),
+			Hosts:    hosts,
 			Action:   &CustomScriptTask{taskDir: taskDir, script: script},
 			Parallel: true,
 			Retry:    1,

From fa354283706e7b69ba2546498bac07f77120d7ed Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Wed, 24 Apr 2024 00:08:55 +0800
Subject: [PATCH 03/31] feat: reformat import order

---
 cmd/kk/pkg/bootstrap/customscripts/module.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/kk/pkg/bootstrap/customscripts/module.go b/cmd/kk/pkg/bootstrap/customscripts/module.go
index 8873b328d..336d35d9b 100644
--- a/cmd/kk/pkg/bootstrap/customscripts/module.go
+++ b/cmd/kk/pkg/bootstrap/customscripts/module.go
@@ -18,9 +18,9 @@ package customscripts
 
 import (
 	"fmt"
-	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
 
 	kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task"
 )

From 08e36795d58090c552a7c3bc2c7c70fafb73d119 Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Thu, 1 Feb 2024 21:22:28 +0800
Subject: [PATCH 04/31] feat: add phase apply addons

---
 cmd/kk/cmd/create/phase/addons.go | 48 ++++++++++++++++++++++++++
 cmd/kk/cmd/create/phase/phase.go  |  1 +
 cmd/kk/pkg/phase/addons/addons.go | 57 +++++++++++++++++++++++++++++++
 3 files changed, 106 insertions(+)
 create mode 100644 cmd/kk/cmd/create/phase/addons.go
 create mode 100644 cmd/kk/pkg/phase/addons/addons.go

diff --git a/cmd/kk/cmd/create/phase/addons.go b/cmd/kk/cmd/create/phase/addons.go
new file mode 100644
index 000000000..236c86992
--- /dev/null
+++ b/cmd/kk/cmd/create/phase/addons.go
@@ -0,0 +1,48 @@
+package phase
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/addons"
+)
+
+type ApplyAddonsOptions struct {
+	CommonOptions  *options.CommonOptions
+	ClusterCfgFile string
+}
+
+func NewApplyAddonsOptions() *ApplyAddonsOptions {
+	return &ApplyAddonsOptions{
+		CommonOptions: options.NewCommonOptions(),
+	}
+}
+
+func NewCmdApplyAddons() *cobra.Command {
+	o := NewApplyAddonsOptions()
+	cmd := &cobra.Command{
+		Use:   "addons",
+		Short: "Apply cluster addons",
+		Run: func(cmd *cobra.Command, args []string) {
+			util.CheckErr(o.Run())
+		},
+	}
+
+	o.CommonOptions.AddCommonFlag(cmd)
+	o.AddFlags(cmd)
+	return cmd
+}
+
+func (o *ApplyAddonsOptions) Run() error {
+	arg := common.Argument{
+		FilePath: o.ClusterCfgFile,
+		Debug:    o.CommonOptions.Verbose,
+	}
+	return addons.ApplyClusterAddons(arg)
+}
+
+func (o *ApplyAddonsOptions) AddFlags(cmd *cobra.Command) {
+	cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file")
+}
diff --git a/cmd/kk/cmd/create/phase/phase.go b/cmd/kk/cmd/create/phase/phase.go
index 655fddeae..862a8a346 100755
--- a/cmd/kk/cmd/create/phase/phase.go
+++ b/cmd/kk/cmd/create/phase/phase.go
@@ -34,6 +34,7 @@ func NewPhaseCommand() *cobra.Command {
 	cmds.AddCommand(NewCmdCreateJoinNodes())
 	cmds.AddCommand(NewCmdCreateConfigureKubernetes())
 	cmds.AddCommand(NewCmdCreateKubeSphere())
+	cmds.AddCommand(NewCmdApplyAddons())
 
 	return cmds
 }
diff --git a/cmd/kk/pkg/phase/addons/addons.go b/cmd/kk/pkg/phase/addons/addons.go
new file mode 100644
index 000000000..ee2660d00
--- /dev/null
+++ b/cmd/kk/pkg/phase/addons/addons.go
@@ -0,0 +1,57 @@
+package addons
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/addons"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes"
+)
+
+func ApplyClusterAddons(args common.Argument) error {
+
+	var loaderType string
+	if args.FilePath != "" {
+		loaderType = common.File
+	} else {
+		loaderType = common.AllInOne
+	}
+
+	runtime, err := common.NewKubeRuntime(loaderType, args)
+	if err != nil {
+		return err
+	}
+
+	switch runtime.Cluster.Kubernetes.Type {
+	case common.Kubernetes:
+		if err := ApplyClusterAddonsPipeline(runtime); err != nil {
+			return err
+		}
+	default:
+		return errors.New("unsupported cluster kubernetes type")
+	}
+
+	return nil
+}
+
+func ApplyClusterAddonsPipeline(runtime *common.KubeRuntime) error {
+	m := []module.Module{
+		&precheck.GreetingsModule{},
+		&precheck.NodePreCheckModule{},
+		&kubernetes.StatusModule{},
+		&addons.AddonsModule{},
+	}
+
+	p := pipeline.Pipeline{
+		Name:    "ApplyClusterAddonsPipeline",
+		Modules: m,
+		Runtime: runtime,
+	}
+	if err := p.Start(); err != nil {
+		return err
+	}
+	return nil
+}

From 934315165b7f87df6939ca0bb340fc9771e5befb Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Tue, 12 Mar 2024 20:12:28 +0800
Subject: [PATCH 05/31] feat: add arg SkipInstallAddons

---
 cmd/kk/cmd/create/cluster.go           | 3 +++
 cmd/kk/pkg/common/kube_runtime.go      | 1 +
 cmd/kk/pkg/pipelines/create_cluster.go | 6 +++---
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/cmd/kk/cmd/create/cluster.go b/cmd/kk/cmd/create/cluster.go
index 3ca16ec03..3e3bf6cf0 100644
--- a/cmd/kk/cmd/create/cluster.go
+++ b/cmd/kk/cmd/create/cluster.go
@@ -38,6 +38,7 @@ type CreateClusterOptions struct {
 	EnableKubeSphere    bool
 	KubeSphere          string
 	LocalStorage        bool
+	SkipInstallAddons   bool
 	SkipPullImages      bool
 	SkipPushImages      bool
 	SecurityEnhancement bool
@@ -112,6 +113,7 @@ func (o *CreateClusterOptions) Run() error {
 		KubernetesVersion:   o.Kubernetes,
 		KsEnable:            o.EnableKubeSphere,
 		KsVersion:           o.KubeSphere,
+		SkipInstallAddons:   o.SkipInstallAddons,
 		SkipPullImages:      o.SkipPullImages,
 		SkipPushImages:      o.SkipPushImages,
 		SecurityEnhancement: o.SecurityEnhancement,
@@ -137,6 +139,7 @@ func (o *CreateClusterOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes")
 	cmd.Flags().BoolVarP(&o.LocalStorage, "with-local-storage", "", false, "Deploy a local PV provisioner")
 	cmd.Flags().BoolVarP(&o.EnableKubeSphere, "with-kubesphere", "", false, fmt.Sprintf("Deploy a specific version of kubesphere (default %s)", kubesphere.Latest().Version))
+	cmd.Flags().BoolVarP(&o.SkipInstallAddons, "skip-install-addons", "", false, "Skip install addons")
 	cmd.Flags().BoolVarP(&o.SkipPullImages, "skip-pull-images", "", false, "Skip pre pull images")
 	cmd.Flags().BoolVarP(&o.SkipPushImages, "skip-push-images", "", false, "Skip pre push images")
 	cmd.Flags().BoolVarP(&o.SecurityEnhancement, "with-security-enhancement", "", false, "Security enhancement")
diff --git a/cmd/kk/pkg/common/kube_runtime.go b/cmd/kk/pkg/common/kube_runtime.go
index c7727474b..fdf4fc657 100644
--- a/cmd/kk/pkg/common/kube_runtime.go
+++ b/cmd/kk/pkg/common/kube_runtime.go
@@ -37,6 +37,7 @@ type Argument struct {
 	KsVersion           string
 	Debug               bool
 	IgnoreErr           bool
+	SkipInstallAddons   bool
 	SkipPullImages      bool
 	SkipPushImages      bool
 	SkipDependencyCheck bool
diff --git a/cmd/kk/pkg/pipelines/create_cluster.go b/cmd/kk/pkg/pipelines/create_cluster.go
index 4f5935f6c..1ed3c331c 100644
--- a/cmd/kk/pkg/pipelines/create_cluster.go
+++ b/cmd/kk/pkg/pipelines/create_cluster.go
@@ -92,7 +92,7 @@ func NewCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&kubernetes.SecurityEnhancementModule{Skip: !runtime.Arg.SecurityEnhancement},
 		&kubernetes.SaveKubeConfigModule{},
 		&plugins.DeployPluginsModule{},
-		&addons.AddonsModule{},
+		&addons.AddonsModule{Skip: runtime.Arg.SkipInstallAddons},
 		&storage.DeployLocalVolumeModule{Skip: skipLocalStorage},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
 		&kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
@@ -166,7 +166,7 @@ func NewK3sCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&filesystem.ChownModule{},
 		&certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()},
 		&k3s.SaveKubeConfigModule{},
-		&addons.AddonsModule{},
+		&addons.AddonsModule{Skip: runtime.Arg.SkipInstallAddons},
 		&storage.DeployLocalVolumeModule{Skip: skipLocalStorage},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
 		&kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
@@ -240,7 +240,7 @@ func NewK8eCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&filesystem.ChownModule{},
 		&certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()},
 		&k8e.SaveKubeConfigModule{},
-		&addons.AddonsModule{},
+		&addons.AddonsModule{Skip: runtime.Arg.SkipInstallAddons},
 		&storage.DeployLocalVolumeModule{Skip: skipLocalStorage},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
 		&kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled},

From 9dba9c84b05d7dd93249b7b4b57bb8d4bc09175a Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Wed, 13 Mar 2024 10:30:23 +0800
Subject: [PATCH 06/31] feat: support addons from args

---
 cmd/kk/cmd/create/phase/addons.go |  9 +++---
 cmd/kk/pkg/addons/tasks.go        | 48 +++++++++++++++++++++++++++++++
 cmd/kk/pkg/common/kube_runtime.go |  1 +
 3 files changed, 54 insertions(+), 4 deletions(-)

diff --git a/cmd/kk/cmd/create/phase/addons.go b/cmd/kk/cmd/create/phase/addons.go
index 236c86992..ead549ccb 100644
--- a/cmd/kk/cmd/create/phase/addons.go
+++ b/cmd/kk/cmd/create/phase/addons.go
@@ -26,7 +26,7 @@ func NewCmdApplyAddons() *cobra.Command {
 		Use:   "addons",
 		Short: "Apply cluster addons",
 		Run: func(cmd *cobra.Command, args []string) {
-			util.CheckErr(o.Run())
+			util.CheckErr(o.Run(args))
 		},
 	}
 
@@ -35,10 +35,11 @@ func NewCmdApplyAddons() *cobra.Command {
 	return cmd
 }
 
-func (o *ApplyAddonsOptions) Run() error {
+func (o *ApplyAddonsOptions) Run(args []string) error {
 	arg := common.Argument{
-		FilePath: o.ClusterCfgFile,
-		Debug:    o.CommonOptions.Verbose,
+		FilePath:      o.ClusterCfgFile,
+		Debug:         o.CommonOptions.Verbose,
+		EnabledAddons: args,
 	}
 	return addons.ApplyClusterAddons(arg)
 }
diff --git a/cmd/kk/pkg/addons/tasks.go b/cmd/kk/pkg/addons/tasks.go
index cd79d589f..ce19ffe79 100644
--- a/cmd/kk/pkg/addons/tasks.go
+++ b/cmd/kk/pkg/addons/tasks.go
@@ -18,7 +18,9 @@ package addons
 
 import (
 	"fmt"
+	"github.com/pkg/errors"
 	"path/filepath"
+	"strings"
 
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
@@ -31,7 +33,18 @@ type Install struct {
 
 func (i *Install) Execute(runtime connector.Runtime) error {
 	nums := len(i.KubeConf.Cluster.Addons)
+
+	enabledAddons, err := i.enabledAddons()
+	if err != nil {
+		return err
+	}
+
+	logger.Log.Messagef(runtime.RemoteHost().GetName(), "[%v/%v] enabled addons", len(enabledAddons), nums)
+
 	for index, addon := range i.KubeConf.Cluster.Addons {
+		if _, ok := enabledAddons[addon.Name]; !ok {
+			continue
+		}
 		logger.Log.Messagef(runtime.RemoteHost().GetName(), "Install addon [%v-%v]: %s", nums, index, addon.Name)
 		if err := InstallAddons(i.KubeConf, &addon, filepath.Join(runtime.GetWorkDir(), fmt.Sprintf("config-%s", runtime.GetObjName()))); err != nil {
 			return err
@@ -39,3 +52,38 @@ func (i *Install) Execute(runtime connector.Runtime) error {
 	}
 	return nil
 }
+
+func (i *Install) enabledAddons() (map[string]struct{}, error) {
+	enabledAddons := make(map[string]struct{}, len(i.KubeConf.Cluster.Addons))
+	for _, addon := range i.KubeConf.Cluster.Addons {
+		enabledAddons[addon.Name] = struct{}{}
+	}
+
+	if len(i.KubeConf.Arg.EnabledAddons) == 0 {
+		return enabledAddons, nil
+	}
+
+	enabledAddonsConfig := make(map[string]struct{}, len(i.KubeConf.Arg.EnabledAddons))
+	for _, config := range i.KubeConf.Arg.EnabledAddons {
+		enabledAddonsConfig[config] = struct{}{}
+	}
+
+	for addon := range enabledAddons {
+		if _, ok := enabledAddonsConfig[addon]; !ok {
+			// drop addons not in input args
+			delete(enabledAddons, addon)
+		} else {
+			// drop input args validated
+			delete(enabledAddonsConfig, addon)
+		}
+	}
+	if len(enabledAddonsConfig) > 0 {
+		// exists invalid input args
+		keys := make([]string, 0, len(enabledAddonsConfig))
+		for key := range enabledAddonsConfig {
+			keys = append(keys, key)
+		}
+		return nil, errors.New(fmt.Sprintf("Addons not exists: %s", strings.Join(keys, ",")))
+	}
+	return enabledAddons, nil
+}
diff --git a/cmd/kk/pkg/common/kube_runtime.go b/cmd/kk/pkg/common/kube_runtime.go
index fdf4fc657..132158240 100644
--- a/cmd/kk/pkg/common/kube_runtime.go
+++ b/cmd/kk/pkg/common/kube_runtime.go
@@ -38,6 +38,7 @@ type Argument struct {
 	Debug               bool
 	IgnoreErr           bool
 	SkipInstallAddons   bool
+	EnabledAddons       []string
 	SkipPullImages      bool
 	SkipPushImages      bool
 	SkipDependencyCheck bool

From d3d4eee5d4ed6edd039048753eb956e8e384181d Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Wed, 13 Mar 2024 10:37:07 +0800
Subject: [PATCH 07/31] feat: support preInstall & postInstall of addons

---
 cmd/kk/apis/kubekey/v1alpha2/addons_types.go | 12 ++++---
 cmd/kk/pkg/addons/module.go                  | 23 ++++++++++++++
 cmd/kk/pkg/addons/tasks.go                   | 33 ++++++++++++++++++--
 cmd/kk/pkg/core/pipeline/pipeline.go         |  5 ++-
 4 files changed, 65 insertions(+), 8 deletions(-)

diff --git a/cmd/kk/apis/kubekey/v1alpha2/addons_types.go b/cmd/kk/apis/kubekey/v1alpha2/addons_types.go
index 5500317b1..8066524cf 100644
--- a/cmd/kk/apis/kubekey/v1alpha2/addons_types.go
+++ b/cmd/kk/apis/kubekey/v1alpha2/addons_types.go
@@ -17,11 +17,13 @@
 package v1alpha2
 
 type Addon struct {
-	Name      string  `yaml:"name" json:"name,omitempty"`
-	Namespace string  `yaml:"namespace" json:"namespace,omitempty"`
-	Sources   Sources `yaml:"sources" json:"sources,omitempty"`
-	Retries   int     `yaml:"retries" json:"retries,omitempty"`
-	Delay     int     `yaml:"delay" json:"delay,omitempty"`
+	Name        string          `yaml:"name" json:"name,omitempty"`
+	Namespace   string          `yaml:"namespace" json:"namespace,omitempty"`
+	PreInstall  []CustomScripts `yaml:"preInstall" json:"preInstall,omitempty"`
+	Sources     Sources         `yaml:"sources" json:"sources,omitempty"`
+	PostInstall []CustomScripts `yaml:"postInstall" json:"postInstall,omitempty"`
+	Retries     int             `yaml:"retries" json:"retries,omitempty"`
+	Delay       int             `yaml:"delay" json:"delay,omitempty"`
 }
 
 type Sources struct {
diff --git a/cmd/kk/pkg/addons/module.go b/cmd/kk/pkg/addons/module.go
index ab833c23e..4b08fede1 100644
--- a/cmd/kk/pkg/addons/module.go
+++ b/cmd/kk/pkg/addons/module.go
@@ -17,6 +17,9 @@
 package addons
 
 import (
+	"fmt"
+
+	kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task"
 )
@@ -44,3 +47,23 @@ func (a *AddonsModule) Init() {
 		install,
 	}
 }
+
+type AddonModule struct {
+	common.KubeModule
+	addon *kubekeyapiv1alpha2.Addon
+}
+
+func (s *AddonModule) Init() {
+	s.Name = "AddonModule"
+	s.Desc = fmt.Sprintf("Install addon %s", s.addon.Name)
+
+	install := &task.LocalTask{
+		Name:   "InstallAddon",
+		Desc:   fmt.Sprintf("Install addon %s", s.addon.Name),
+		Action: &InstallAddon{addon: s.addon},
+	}
+
+	s.Tasks = []task.Interface{
+		install,
+	}
+}
diff --git a/cmd/kk/pkg/addons/tasks.go b/cmd/kk/pkg/addons/tasks.go
index ce19ffe79..547796410 100644
--- a/cmd/kk/pkg/addons/tasks.go
+++ b/cmd/kk/pkg/addons/tasks.go
@@ -18,13 +18,18 @@ package addons
 
 import (
 	"fmt"
-	"github.com/pkg/errors"
 	"path/filepath"
 	"strings"
 
+	"github.com/pkg/errors"
+
+	kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/customscripts"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline"
 )
 
 type Install struct {
@@ -46,7 +51,19 @@ func (i *Install) Execute(runtime connector.Runtime) error {
 			continue
 		}
 		logger.Log.Messagef(runtime.RemoteHost().GetName(), "Install addon [%v-%v]: %s", nums, index, addon.Name)
-		if err := InstallAddons(i.KubeConf, &addon, filepath.Join(runtime.GetWorkDir(), fmt.Sprintf("config-%s", runtime.GetObjName()))); err != nil {
+
+		m := []module.Module{
+			&customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: addon.PreInstall},
+			&AddonModule{addon: &addon},
+			&customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: addon.PostInstall},
+		}
+		p := pipeline.Pipeline{
+			Name:          "InstallAddonsPipeline",
+			Modules:       m,
+			Runtime:       runtime,
+			SkipPrintLogo: true,
+		}
+		if err := p.Start(); err != nil {
 			return err
 		}
 	}
@@ -87,3 +104,15 @@ func (i *Install) enabledAddons() (map[string]struct{}, error) {
 	}
 	return enabledAddons, nil
 }
+
+type InstallAddon struct {
+	common.KubeAction
+	addon *kubekeyapiv1alpha2.Addon
+}
+
+func (i *InstallAddon) Execute(runtime connector.Runtime) error {
+	if err := InstallAddons(i.KubeConf, i.addon, filepath.Join(runtime.GetWorkDir(), fmt.Sprintf("config-%s", runtime.GetObjName()))); err != nil {
+		return err
+	}
+	return nil
+}
diff --git a/cmd/kk/pkg/core/pipeline/pipeline.go b/cmd/kk/pkg/core/pipeline/pipeline.go
index 9afce39d9..0f6525560 100644
--- a/cmd/kk/pkg/core/pipeline/pipeline.go
+++ b/cmd/kk/pkg/core/pipeline/pipeline.go
@@ -51,10 +51,13 @@ type Pipeline struct {
 	PipelineCache   *cache.Cache
 	ModuleCachePool sync.Pool
 	ModulePostHooks []module.PostHookInterface
+	SkipPrintLogo   bool
 }
 
 func (p *Pipeline) Init() error {
-	fmt.Print(logo)
+	if !p.SkipPrintLogo {
+		fmt.Print(logo)
+	}
 	p.PipelineCache = cache.NewCache()
 	p.SpecHosts = len(p.Runtime.GetAllHosts())
 	//if err := p.Runtime.GenerateWorkDir(); err != nil {

From 59f49e8f3e6b9a72dd2d6473828b66dda132450c Mon Sep 17 00:00:00 2001
From: knowmost <knowmost@outlook.com>
Date: Sun, 28 Apr 2024 11:59:32 +0800
Subject: [PATCH 08/31] chore: fix function name in comment

Signed-off-by: knowmost <knowmost@outlook.com>
---
 cmd/kk/cmd/create/phase/init.go    | 2 +-
 cmd/kk/cmd/upgrade/phase/images.go | 2 +-
 cmd/kk/cmd/upgrade/phase/nodes.go  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/kk/cmd/create/phase/init.go b/cmd/kk/cmd/create/phase/init.go
index 60f5c83b3..abcc2fd8a 100644
--- a/cmd/kk/cmd/create/phase/init.go
+++ b/cmd/kk/cmd/create/phase/init.go
@@ -39,7 +39,7 @@ func NewCreateInitClusterOptions() *CreateInitClusterOptions {
 	}
 }
 
-// NewCmdUpgrade creates a new upgrade command
+// NewCmdCreateInitCluster creates a new upgrade command
 func NewCmdCreateInitCluster() *cobra.Command {
 	o := NewCreateInitClusterOptions()
 	cmd := &cobra.Command{
diff --git a/cmd/kk/cmd/upgrade/phase/images.go b/cmd/kk/cmd/upgrade/phase/images.go
index 86bd97a44..a726f7265 100755
--- a/cmd/kk/cmd/upgrade/phase/images.go
+++ b/cmd/kk/cmd/upgrade/phase/images.go
@@ -39,7 +39,7 @@ func NewUpgradeImagesOptions() *UpgradeImagesOptions {
 	}
 }
 
-// NewCmdUpgrade creates a new upgrade command
+// NewCmdUpgradeImages creates a new upgrade command
 func NewCmdUpgradeImages() *cobra.Command {
 	o := NewUpgradeImagesOptions()
 	cmd := &cobra.Command{
diff --git a/cmd/kk/cmd/upgrade/phase/nodes.go b/cmd/kk/cmd/upgrade/phase/nodes.go
index 83b0eaa29..b00d54ee4 100755
--- a/cmd/kk/cmd/upgrade/phase/nodes.go
+++ b/cmd/kk/cmd/upgrade/phase/nodes.go
@@ -39,7 +39,7 @@ func NewUpgradeNodesOptions() *UpgradeNodesOptions {
 	}
 }
 
-// NewCmdUpgrade creates a new upgrade command
+// NewCmdUpgradeNodes creates a new upgrade command
 func NewCmdUpgradeNodes() *cobra.Command {
 	o := NewUpgradeNodesOptions()
 	cmd := &cobra.Command{

From 0f6ff74727f6df31a56fc586dc069dc73510bc0d Mon Sep 17 00:00:00 2001
From: hellocn9 <zhangyancn9@126.com>
Date: Mon, 29 Apr 2024 15:34:27 +0800
Subject: [PATCH 09/31] add a flag --with-buildx to install buildx when
 container runtime is docker

---
 cmd/kk/apis/kubekey/v1alpha2/default.go |  1 +
 cmd/kk/cmd/create/cluster.go            |  3 +++
 cmd/kk/pkg/binaries/kubernetes.go       |  5 ++++
 cmd/kk/pkg/common/common.go             |  2 ++
 cmd/kk/pkg/common/kube_runtime.go       |  1 +
 cmd/kk/pkg/container/docker.go          | 33 +++++++++++++++++++++++++
 cmd/kk/pkg/container/module.go          | 14 +++++++++++
 cmd/kk/pkg/container/prepares.go        | 12 +++++++++
 cmd/kk/pkg/files/file.go                |  6 +++++
 version/components.json                 |  8 ++++++
 10 files changed, 85 insertions(+)

diff --git a/cmd/kk/apis/kubekey/v1alpha2/default.go b/cmd/kk/apis/kubekey/v1alpha2/default.go
index 0f802cf7b..b02d41fc9 100644
--- a/cmd/kk/apis/kubekey/v1alpha2/default.go
+++ b/cmd/kk/apis/kubekey/v1alpha2/default.go
@@ -43,6 +43,7 @@ const (
 	DefaultEtcdPort                = "2379"
 	DefaultDockerVersion           = "24.0.9"
 	DefaultCriDockerdVersion       = "0.3.10"
+	DefaultBuildxVersion           = "v0.14.0"
 	DefaultContainerdVersion       = "1.7.13"
 	DefaultRuncVersion             = "v1.1.12"
 	DefaultCrictlVersion           = "v1.29.0"
diff --git a/cmd/kk/cmd/create/cluster.go b/cmd/kk/cmd/create/cluster.go
index 3ca16ec03..2a6db5f3b 100644
--- a/cmd/kk/cmd/create/cluster.go
+++ b/cmd/kk/cmd/create/cluster.go
@@ -45,6 +45,7 @@ type CreateClusterOptions struct {
 	DownloadCmd         string
 	Artifact            string
 	InstallPackages     bool
+	WithBuildx          bool
 
 	localStorageChanged bool
 }
@@ -122,6 +123,7 @@ func (o *CreateClusterOptions) Run() error {
 		Artifact:            o.Artifact,
 		InstallPackages:     o.InstallPackages,
 		Namespace:           o.CommonOptions.Namespace,
+		WithBuildx:          o.WithBuildx,
 	}
 
 	if o.localStorageChanged {
@@ -145,6 +147,7 @@ func (o *CreateClusterOptions) AddFlags(cmd *cobra.Command) {
 		`The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`)
 	cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact")
 	cmd.Flags().BoolVarP(&o.InstallPackages, "with-packages", "", false, "install operation system packages by artifact")
+	cmd.Flags().BoolVarP(&o.WithBuildx, "with-buildx", "", false, "install buildx when Container runtime is docker")
 }
 
 func completionSetting(cmd *cobra.Command) (err error) {
diff --git a/cmd/kk/pkg/binaries/kubernetes.go b/cmd/kk/pkg/binaries/kubernetes.go
index 5f16c9c98..832183e59 100644
--- a/cmd/kk/pkg/binaries/kubernetes.go
+++ b/cmd/kk/pkg/binaries/kubernetes.go
@@ -45,6 +45,8 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string,
 	runc := files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, kubeConf.Arg.DownloadCommand)
 	calicoctl := files.NewKubeBinary("calicoctl", arch, kubekeyapiv1alpha2.DefaultCalicoVersion, path, kubeConf.Arg.DownloadCommand)
 
+	buildx := files.NewKubeBinary(common.Buildx, arch, kubekeyapiv1alpha2.DefaultBuildxVersion, path, kubeConf.Arg.DownloadCommand)
+
 	binaries := []*files.KubeBinary{kubeadm, kubelet, kubectl, helm, kubecni, crictl, etcd}
 
 	if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Docker {
@@ -52,6 +54,9 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string,
 		if kubeConf.Cluster.Kubernetes.IsAtLeastV124() && kubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
 			binaries = append(binaries, criDockerd)
 		}
+		if kubeConf.Arg.WithBuildx {
+			binaries = append(binaries, buildx)
+		}
 	} else if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Containerd {
 		binaries = append(binaries, containerd, runc)
 	}
diff --git a/cmd/kk/pkg/common/common.go b/cmd/kk/pkg/common/common.go
index a57ee1ec2..cfbec5384 100644
--- a/cmd/kk/pkg/common/common.go
+++ b/cmd/kk/pkg/common/common.go
@@ -69,6 +69,8 @@ const (
 	Isula      = "isula"
 	Runc       = "runc"
 
+	Buildx = "buildx"
+
 	// global cache key
 	// PreCheckModule
 	NodePreCheck           = "nodePreCheck"
diff --git a/cmd/kk/pkg/common/kube_runtime.go b/cmd/kk/pkg/common/kube_runtime.go
index c7727474b..32cad9e92 100644
--- a/cmd/kk/pkg/common/kube_runtime.go
+++ b/cmd/kk/pkg/common/kube_runtime.go
@@ -55,6 +55,7 @@ type Argument struct {
 	Role                string
 	Type                string
 	EtcdUpgrade         bool
+	WithBuildx          bool
 }
 
 func NewKubeRuntime(flag string, arg Argument) (*KubeRuntime, error) {
diff --git a/cmd/kk/pkg/container/docker.go b/cmd/kk/pkg/container/docker.go
index 94641b167..cbe8de6e6 100644
--- a/cmd/kk/pkg/container/docker.go
+++ b/cmd/kk/pkg/container/docker.go
@@ -30,6 +30,39 @@ import (
 	"github.com/pkg/errors"
 )
 
+type SyncDockerBuildxPluginBinaries struct {
+	common.KubeAction
+}
+
+func (s *SyncDockerBuildxPluginBinaries) Execute(runtime connector.Runtime) error {
+	if err := utils.ResetTmpDir(runtime); err != nil {
+		return err
+	}
+
+	binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch())
+	if !ok {
+		return errors.New("get KubeBinary by pipeline cache failed")
+	}
+	binariesMap := binariesMapObj.(map[string]*files.KubeBinary)
+
+	buildx, ok := binariesMap[common.Buildx]
+	if !ok {
+		return errors.New("get KubeBinary key buildx by pipeline cache failed")
+	}
+
+	dst := filepath.Join(common.TmpDir, buildx.FileName)
+	if err := runtime.GetRunner().Scp(buildx.Path(), dst); err != nil {
+		return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync docker binaries failed"))
+	}
+
+	if _, err := runtime.GetRunner().SudoCmd(
+		fmt.Sprintf("mkdir -p /usr/local/lib/docker/cli-plugins && mv %s /usr/local/lib/docker/cli-plugins/docker-buildx && chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx && rm -rf %s", dst, dst),
+		false); err != nil {
+		return errors.Wrap(errors.WithStack(err), fmt.Sprintf("install docker-buildx binaries failed"))
+	}
+	return nil
+}
+
 type SyncDockerBinaries struct {
 	common.KubeAction
 }
diff --git a/cmd/kk/pkg/container/module.go b/cmd/kk/pkg/container/module.go
index 36eaddeac..a432eb4dc 100644
--- a/cmd/kk/pkg/container/module.go
+++ b/cmd/kk/pkg/container/module.go
@@ -60,6 +60,19 @@ func (i *InstallContainerModule) Init() {
 }
 
 func InstallDocker(m *InstallContainerModule) []task.Interface {
+
+	syncBuildxPluginBinaries := &task.RemoteTask{
+		Name:  "SyncDockerBuildxBinaries",
+		Desc:  "Sync docker buildx binaries",
+		Hosts: m.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&WithBuildxPlugin{},
+		},
+		Action:   new(SyncDockerBuildxPluginBinaries),
+		Parallel: true,
+		Retry:    2,
+	}
+
 	syncBinaries := &task.RemoteTask{
 		Name:  "SyncDockerBinaries",
 		Desc:  "Sync docker binaries",
@@ -169,6 +182,7 @@ func InstallDocker(m *InstallContainerModule) []task.Interface {
 		enableContainerdForDocker,
 		enableDocker,
 		dockerLoginRegistry,
+		syncBuildxPluginBinaries,
 	}
 }
 
diff --git a/cmd/kk/pkg/container/prepares.go b/cmd/kk/pkg/container/prepares.go
index d0f0ba829..4e6782bcb 100644
--- a/cmd/kk/pkg/container/prepares.go
+++ b/cmd/kk/pkg/container/prepares.go
@@ -23,6 +23,18 @@ import (
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
 )
 
+type WithBuildxPlugin struct {
+	common.KubePrepare
+}
+
+func (w *WithBuildxPlugin) PreCheck(runtime connector.Runtime) (bool, error) {
+	if w.KubeConf.Arg.WithBuildx {
+		return true, nil
+	} else {
+		return false, nil
+	}
+}
+
 type DockerExist struct {
 	common.KubePrepare
 	Not bool
diff --git a/cmd/kk/pkg/files/file.go b/cmd/kk/pkg/files/file.go
index 9079837ad..d874d3ee2 100644
--- a/cmd/kk/pkg/files/file.go
+++ b/cmd/kk/pkg/files/file.go
@@ -52,6 +52,7 @@ const (
 	containerd = "containerd"
 	runc       = "runc"
 	calicoctl  = "calicoctl"
+	buildx     = "buildx"
 )
 
 // KubeBinary Type field const
@@ -66,6 +67,7 @@ const (
 	REGISTRY   = "registry"
 	CONTAINERD = "containerd"
 	RUNC       = "runc"
+	BUILD      = "buildx"
 )
 
 var (
@@ -225,6 +227,10 @@ func NewKubeBinary(name, arch, version, prePath string, getCmd func(path, url st
 		if component.Zone == "cn" {
 			component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/projectcalico/calico/releases/download/%s/calicoctl-linux-%s", version, arch)
 		}
+	case buildx:
+		component.Type = BUILD
+		component.FileName = fmt.Sprintf("buildx-%s.linux-%s", version, arch)
+		component.Url = fmt.Sprintf("https://github.com/docker/buildx/releases/download/%s/buildx-%s.linux-%s", version, version, arch)
 	default:
 		logger.Log.Fatalf("unsupported kube binaries %s", name)
 	}
diff --git a/version/components.json b/version/components.json
index 72bccef3a..18f925b25 100644
--- a/version/components.json
+++ b/version/components.json
@@ -1082,5 +1082,13 @@
             "v3.27.2": "0fd1f65a511338cf9940835987d420c94ab95b5386288ba9673b736a4d347463",
             "v3.27.3": "1fc5f58a18d8b1c487b4663fc5cbe23b45bd9d31617debd309f6dfac7c11a8ef"
         }
+    },
+    "buildx": {
+        "amd64": {
+            "v0.14.0": "32f8f17eca35bf2efe6c0e47f40e4692a876f34531b421efc984799a5b41226e"
+        },
+        "arm64": {
+            "v0.14.0": "38bf0ea9c48743edb8243f14272be65a2bad7092228068337aea584309ea664c"
+        }
     }
 }
\ No newline at end of file

From 1e36c205ccd5a5aeb35234d95dc162c23f46224f Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Thu, 1 Feb 2024 21:25:18 +0800
Subject: [PATCH 10/31] feat: add phase PostClusterInstall

---
 cmd/kk/apis/kubekey/v1alpha2/cluster_types.go | 15 ++++++++-------
 cmd/kk/pkg/pipelines/create_cluster.go        |  3 +++
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go
index b5b3bc2af..ba79c6520 100644
--- a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go
+++ b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go
@@ -94,13 +94,14 @@ type CustomScripts struct {
 
 // System defines the system config for each node in cluster.
 type System struct {
-	NtpServers      []string        `yaml:"ntpServers" json:"ntpServers,omitempty"`
-	Timezone        string          `yaml:"timezone" json:"timezone,omitempty"`
-	Rpms            []string        `yaml:"rpms" json:"rpms,omitempty"`
-	Debs            []string        `yaml:"debs" json:"debs,omitempty"`
-	PreInstall      []CustomScripts `yaml:"preInstall" json:"preInstall,omitempty"`
-	PostInstall     []CustomScripts `yaml:"postInstall" json:"postInstall,omitempty"`
-	SkipConfigureOS bool            `yaml:"skipConfigureOS" json:"skipConfigureOS,omitempty"`
+	NtpServers         []string        `yaml:"ntpServers" json:"ntpServers,omitempty"`
+	Timezone           string          `yaml:"timezone" json:"timezone,omitempty"`
+	Rpms               []string        `yaml:"rpms" json:"rpms,omitempty"`
+	Debs               []string        `yaml:"debs" json:"debs,omitempty"`
+	PreInstall         []CustomScripts `yaml:"preInstall" json:"preInstall,omitempty"`
+	PostClusterInstall []CustomScripts `yaml:"postClusterInstall" json:"postClusterInstall,omitempty"`
+	PostInstall        []CustomScripts `yaml:"postInstall" json:"postInstall,omitempty"`
+	SkipConfigureOS    bool            `yaml:"skipConfigureOS" json:"skipConfigureOS,omitempty"`
 }
 
 // RegistryConfig defines the configuration information of the image's repository.
diff --git a/cmd/kk/pkg/pipelines/create_cluster.go b/cmd/kk/pkg/pipelines/create_cluster.go
index 4f5935f6c..5a510334c 100644
--- a/cmd/kk/pkg/pipelines/create_cluster.go
+++ b/cmd/kk/pkg/pipelines/create_cluster.go
@@ -92,6 +92,7 @@ func NewCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&kubernetes.SecurityEnhancementModule{Skip: !runtime.Arg.SecurityEnhancement},
 		&kubernetes.SaveKubeConfigModule{},
 		&plugins.DeployPluginsModule{},
+		&customscripts.CustomScriptsModule{Phase: "PostClusterInstall", Scripts: runtime.Cluster.System.PostClusterInstall},
 		&addons.AddonsModule{},
 		&storage.DeployLocalVolumeModule{Skip: skipLocalStorage},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
@@ -166,6 +167,7 @@ func NewK3sCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&filesystem.ChownModule{},
 		&certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()},
 		&k3s.SaveKubeConfigModule{},
+		&customscripts.CustomScriptsModule{Phase: "PostClusterInstall", Scripts: runtime.Cluster.System.PostClusterInstall},
 		&addons.AddonsModule{},
 		&storage.DeployLocalVolumeModule{Skip: skipLocalStorage},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
@@ -240,6 +242,7 @@ func NewK8eCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&filesystem.ChownModule{},
 		&certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()},
 		&k8e.SaveKubeConfigModule{},
+		&customscripts.CustomScriptsModule{Phase: "PostClusterInstall", Scripts: runtime.Cluster.System.PostClusterInstall},
 		&addons.AddonsModule{},
 		&storage.DeployLocalVolumeModule{Skip: skipLocalStorage},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},

From 12d15135be8ace32bc05d3582b78f53230e9fc26 Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Fri, 2 Feb 2024 16:24:05 +0800
Subject: [PATCH 11/31] feat: set CustomScriptTask workdir to remoteTaskHome

---
 cmd/kk/pkg/bootstrap/customscripts/tasks.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmd/kk/pkg/bootstrap/customscripts/tasks.go b/cmd/kk/pkg/bootstrap/customscripts/tasks.go
index 439e4dc0d..314698f9d 100644
--- a/cmd/kk/pkg/bootstrap/customscripts/tasks.go
+++ b/cmd/kk/pkg/bootstrap/customscripts/tasks.go
@@ -96,8 +96,9 @@ func (t *CustomScriptTask) Execute(runtime connector.Runtime) error {
 		RunBash = "/bin/bash " + targetPath
 	}
 
+	cd_cmd := fmt.Sprintf("cd %s;", remoteTaskHome)
 	start := time.Now()
-	out, err := runtime.GetRunner().SudoCmd(RunBash, false)
+	out, err := runtime.GetRunner().SudoCmd(cd_cmd+RunBash, false)
 	if err != nil {
 		return errors.Errorf("Exec Bash: %s err:%s", RunBash, err)
 	}

From 967083c0ca885461e23e3c52c3c052cee36b686c Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Sat, 4 May 2024 10:23:54 +0800
Subject: [PATCH 12/31] feat: support image transport when save images

---
 cmd/kk/cmd/artifact/export.go           |  3 +++
 cmd/kk/pkg/common/artifact_runtime.go   |  1 +
 cmd/kk/pkg/common/common.go             |  3 +++
 cmd/kk/pkg/images/module.go             |  3 ++-
 cmd/kk/pkg/images/tasks.go              |  3 ++-
 cmd/kk/pkg/images/utils.go              | 12 ++++++++++++
 cmd/kk/pkg/pipelines/artifact_export.go |  2 +-
 7 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/cmd/kk/cmd/artifact/export.go b/cmd/kk/cmd/artifact/export.go
index 0b8b1d6d6..945fba146 100644
--- a/cmd/kk/cmd/artifact/export.go
+++ b/cmd/kk/cmd/artifact/export.go
@@ -34,6 +34,7 @@ type ArtifactExportOptions struct {
 	Output             string
 	CriSocket          string
 	DownloadCmd        string
+	ImageTransport     string
 	SkipRemoveArtifact bool
 }
 
@@ -80,6 +81,7 @@ func (o *ArtifactExportOptions) Run() error {
 		ManifestFile:       o.ManifestFile,
 		Output:             o.Output,
 		CriSocket:          o.CriSocket,
+		ImageTransport:     o.ImageTransport,
 		Debug:              o.CommonOptions.Verbose,
 		IgnoreErr:          o.CommonOptions.IgnoreErr,
 		SkipRemoveArtifact: o.SkipRemoveArtifact,
@@ -93,6 +95,7 @@ func (o *ArtifactExportOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&o.Output, "output", "o", "", "Path to a output path")
 	cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s",
 		`The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`)
+	cmd.Flags().StringVarP(&o.ImageTransport, "image-transport", "", "", "Image transport to pull from, take values from [docker, docker-daemon]")
 	cmd.Flags().BoolVarP(&o.SkipRemoveArtifact, "skip-remove-artifact", "", false, "Skip remove artifact")
 
 }
diff --git a/cmd/kk/pkg/common/artifact_runtime.go b/cmd/kk/pkg/common/artifact_runtime.go
index c9ad51170..c77f94367 100644
--- a/cmd/kk/pkg/common/artifact_runtime.go
+++ b/cmd/kk/pkg/common/artifact_runtime.go
@@ -35,6 +35,7 @@ type ArtifactArgument struct {
 	Debug              bool
 	IgnoreErr          bool
 	DownloadCommand    func(path, url string) string
+	ImageTransport     string
 	SkipRemoveArtifact bool
 }
 
diff --git a/cmd/kk/pkg/common/common.go b/cmd/kk/pkg/common/common.go
index a57ee1ec2..a14cd312e 100644
--- a/cmd/kk/pkg/common/common.go
+++ b/cmd/kk/pkg/common/common.go
@@ -97,4 +97,7 @@ const (
 
 	// Artifact pipeline
 	Artifact = "artifact"
+
+	// Image Copy Transports
+	DockerDaemon = "docker-daemon"
 )
diff --git a/cmd/kk/pkg/images/module.go b/cmd/kk/pkg/images/module.go
index cc4843e02..5d5d783d2 100644
--- a/cmd/kk/pkg/images/module.go
+++ b/cmd/kk/pkg/images/module.go
@@ -49,6 +49,7 @@ func (p *PullModule) Init() {
 
 type CopyImagesToLocalModule struct {
 	common.ArtifactModule
+	ImageTransport string
 }
 
 func (c *CopyImagesToLocalModule) Init() {
@@ -58,7 +59,7 @@ func (c *CopyImagesToLocalModule) Init() {
 	copyImage := &task.LocalTask{
 		Name:   "SaveImages",
 		Desc:   "Copy images to a local OCI path from registries",
-		Action: new(SaveImages),
+		Action: &SaveImages{ImageTransport: c.ImageTransport},
 	}
 
 	c.Tasks = []task.Interface{
diff --git a/cmd/kk/pkg/images/tasks.go b/cmd/kk/pkg/images/tasks.go
index f431b51db..887e9ad76 100644
--- a/cmd/kk/pkg/images/tasks.go
+++ b/cmd/kk/pkg/images/tasks.go
@@ -155,6 +155,7 @@ func GetImage(runtime connector.ModuleRuntime, kubeConf *common.KubeConf, name s
 
 type SaveImages struct {
 	common.ArtifactAction
+	ImageTransport string
 }
 
 func (s *SaveImages) Execute(runtime connector.Runtime) error {
@@ -175,7 +176,7 @@ func (s *SaveImages) Execute(runtime connector.Runtime) error {
 			auth = v
 		}
 
-		srcName := fmt.Sprintf("docker://%s", image)
+		srcName := formatImageName(s.ImageTransport, image)
 		for _, platform := range s.Manifest.Spec.Arches {
 			arch, variant := ParseArchVariant(platform)
 			// placeholder
diff --git a/cmd/kk/pkg/images/utils.go b/cmd/kk/pkg/images/utils.go
index 7e7a3ea66..aebdd989f 100644
--- a/cmd/kk/pkg/images/utils.go
+++ b/cmd/kk/pkg/images/utils.go
@@ -26,6 +26,7 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
 )
 
@@ -188,3 +189,14 @@ func suffixImageName(imageFullName []string) string {
 	}
 	return imageFullName[0]
 }
+
+func formatImageName(transport string, imageFullName string) string {
+	switch transport {
+	case common.DockerDaemon:
+		return fmt.Sprintf("docker-daemon:%s", imageFullName)
+	case common.Docker:
+		fallthrough
+	default:
+		return fmt.Sprintf("docker://%s", imageFullName)
+	}
+}
diff --git a/cmd/kk/pkg/pipelines/artifact_export.go b/cmd/kk/pkg/pipelines/artifact_export.go
index a47db6857..8d9dc2db9 100644
--- a/cmd/kk/pkg/pipelines/artifact_export.go
+++ b/cmd/kk/pkg/pipelines/artifact_export.go
@@ -34,7 +34,7 @@ import (
 func NewArtifactExportPipeline(runtime *common.ArtifactRuntime) error {
 	m := []module.Module{
 		&confirm.CheckFileExistModule{FileName: runtime.Arg.Output},
-		&images.CopyImagesToLocalModule{},
+		&images.CopyImagesToLocalModule{ImageTransport: runtime.Arg.ImageTransport},
 		&binaries.ArtifactBinariesModule{},
 		&artifact.RepositoryModule{},
 		&artifact.ArchiveModule{},

From 9cbd5c01da71b59ebf3e9ad75f4f026fbae4cbc3 Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Sat, 4 May 2024 10:24:29 +0800
Subject: [PATCH 13/31] feat: support image transport when import images

---
 cmd/kk/cmd/artifact/images/push.go | 16 ++++++++++------
 cmd/kk/pkg/common/kube_runtime.go  |  1 +
 cmd/kk/pkg/images/module.go        |  7 ++++---
 cmd/kk/pkg/images/tasks.go         |  5 +++--
 4 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/cmd/kk/cmd/artifact/images/push.go b/cmd/kk/cmd/artifact/images/push.go
index d337bdd14..3a5f7ad4e 100644
--- a/cmd/kk/cmd/artifact/images/push.go
+++ b/cmd/kk/cmd/artifact/images/push.go
@@ -37,6 +37,7 @@ type ArtifactImagesPushOptions struct {
 	CommonOptions *options.CommonOptions
 
 	ImageDirPath   string
+	ImageTransport string
 	Artifact       string
 	ClusterCfgFile string
 }
@@ -89,11 +90,12 @@ func (o *ArtifactImagesPushOptions) Validate(_ []string) error {
 
 func (o *ArtifactImagesPushOptions) Run() error {
 	arg := common.Argument{
-		ImagesDir: o.ImageDirPath,
-		Artifact:  o.Artifact,
-		FilePath:  o.ClusterCfgFile,
-		Debug:     o.CommonOptions.Verbose,
-		IgnoreErr: o.CommonOptions.IgnoreErr,
+		ImagesDir:      o.ImageDirPath,
+		Artifact:       o.Artifact,
+		FilePath:       o.ClusterCfgFile,
+		ImageTransport: o.ImageTransport,
+		Debug:          o.CommonOptions.Verbose,
+		IgnoreErr:      o.CommonOptions.IgnoreErr,
 	}
 	return runPush(arg)
 }
@@ -102,6 +104,7 @@ func (o *ArtifactImagesPushOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&o.ImageDirPath, "images-dir", "", "", "Path to a KubeKey artifact images directory")
 	cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact")
 	cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file")
+	cmd.Flags().StringVarP(&o.ImageTransport, "image-transport", "", "", "Image transport to push to, take values from [docker, docker-daemon]")
 }
 
 func runPush(arg common.Argument) error {
@@ -122,7 +125,8 @@ func newImagesPushPipeline(runtime *common.KubeRuntime) error {
 
 	m := []module.Module{
 		&artifact.UnArchiveModule{Skip: noArtifact},
-		&images.CopyImagesToRegistryModule{ImagePath: runtime.Arg.ImagesDir},
+		&images.CopyImagesToRegistryModule{ImagePath: runtime.Arg.ImagesDir,
+			ImageTransport: runtime.Arg.ImageTransport},
 		&filesystem.ChownWorkDirModule{},
 	}
 
diff --git a/cmd/kk/pkg/common/kube_runtime.go b/cmd/kk/pkg/common/kube_runtime.go
index c7727474b..b67010356 100644
--- a/cmd/kk/pkg/common/kube_runtime.go
+++ b/cmd/kk/pkg/common/kube_runtime.go
@@ -48,6 +48,7 @@ type Argument struct {
 	FromCluster         bool
 	KubeConfig          string
 	Artifact            string
+	ImageTransport      string
 	InstallPackages     bool
 	ImagesDir           string
 	Namespace           string
diff --git a/cmd/kk/pkg/images/module.go b/cmd/kk/pkg/images/module.go
index 5d5d783d2..c64f85218 100644
--- a/cmd/kk/pkg/images/module.go
+++ b/cmd/kk/pkg/images/module.go
@@ -69,8 +69,9 @@ func (c *CopyImagesToLocalModule) Init() {
 
 type CopyImagesToRegistryModule struct {
 	common.KubeModule
-	Skip      bool
-	ImagePath string
+	Skip           bool
+	ImagePath      string
+	ImageTransport string
 }
 
 func (c *CopyImagesToRegistryModule) IsSkip() bool {
@@ -84,7 +85,7 @@ func (c *CopyImagesToRegistryModule) Init() {
 	copyImage := &task.LocalTask{
 		Name:   "CopyImagesToRegistry",
 		Desc:   "Copy images to a private registry from an artifact OCI Path",
-		Action: &CopyImagesToRegistry{ImagesPath: c.ImagePath},
+		Action: &CopyImagesToRegistry{ImagesPath: c.ImagePath, ImageTransport: c.ImageTransport},
 	}
 
 	pushManifest := &task.LocalTask{
diff --git a/cmd/kk/pkg/images/tasks.go b/cmd/kk/pkg/images/tasks.go
index 887e9ad76..d33041c42 100644
--- a/cmd/kk/pkg/images/tasks.go
+++ b/cmd/kk/pkg/images/tasks.go
@@ -233,7 +233,8 @@ func (s *SaveImages) Execute(runtime connector.Runtime) error {
 
 type CopyImagesToRegistry struct {
 	common.KubeAction
-	ImagesPath string
+	ImagesPath     string
+	ImageTransport string
 }
 
 func (c *CopyImagesToRegistry) Execute(runtime connector.Runtime) error {
@@ -306,7 +307,7 @@ func (c *CopyImagesToRegistry) Execute(runtime connector.Runtime) error {
 		}
 
 		srcName := fmt.Sprintf("oci:%s:%s", imagesPath, ref)
-		destName := fmt.Sprintf("docker://%s", image.ImageName())
+		destName := formatImageName(c.ImageTransport, image.ImageName())
 		logger.Log.Infof("Source: %s", srcName)
 		logger.Log.Infof("Destination: %s", destName)
 

From dd5b0f971da964cf35330559b1c15da50d5fbec5 Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Sat, 4 May 2024 11:33:31 +0800
Subject: [PATCH 14/31] feat: support skip download of os iso file by sha256
 checksum

---
 cmd/kk/apis/kubekey/v1alpha2/manifest_types.go |  1 +
 cmd/kk/pkg/artifact/tasks.go                   | 11 +++++++++++
 cmd/kk/pkg/files/file.go                       | 17 +++++++++++++++++
 3 files changed, 29 insertions(+)

diff --git a/cmd/kk/apis/kubekey/v1alpha2/manifest_types.go b/cmd/kk/apis/kubekey/v1alpha2/manifest_types.go
index 66aab5844..2ba3ee18e 100644
--- a/cmd/kk/apis/kubekey/v1alpha2/manifest_types.go
+++ b/cmd/kk/apis/kubekey/v1alpha2/manifest_types.go
@@ -24,6 +24,7 @@ import (
 type Iso struct {
 	LocalPath string `yaml:"localPath" json:"localPath"`
 	Url       string `yaml:"url" json:"url"`
+	Checksum  string `yaml:"checksum" json:"checksum"`
 }
 
 type Repository struct {
diff --git a/cmd/kk/pkg/artifact/tasks.go b/cmd/kk/pkg/artifact/tasks.go
index a10889961..3e7e34202 100644
--- a/cmd/kk/pkg/artifact/tasks.go
+++ b/cmd/kk/pkg/artifact/tasks.go
@@ -30,6 +30,7 @@ import (
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
 	coreutil "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files"
 )
 
 type DownloadISOFile struct {
@@ -44,6 +45,16 @@ func (d *DownloadISOFile) Execute(runtime connector.Runtime) error {
 
 		fileName := fmt.Sprintf("%s-%s-%s.iso", sys.Id, sys.Version, sys.Arch)
 		filePath := filepath.Join(runtime.GetWorkDir(), fileName)
+
+		checksumEqual, err := files.SHA256CheckEqual(filePath, sys.Repository.Iso.Checksum)
+		if err != nil {
+			return err
+		}
+		if checksumEqual {
+			logger.Log.Infof("Skip download exists iso file %s", fileName)
+			continue
+		}
+
 		getCmd := d.Manifest.Arg.DownloadCommand(filePath, sys.Repository.Iso.Url)
 
 		cmd := exec.Command("/bin/sh", "-c", getCmd)
diff --git a/cmd/kk/pkg/files/file.go b/cmd/kk/pkg/files/file.go
index 9079837ad..50f327758 100644
--- a/cmd/kk/pkg/files/file.go
+++ b/cmd/kk/pkg/files/file.go
@@ -322,6 +322,23 @@ func (b *KubeBinary) SHA256Check() error {
 	return nil
 }
 
+func SHA256CheckEqual(filePath string, checksum string) (bool, error) {
+	if strings.TrimSpace(checksum) == "" {
+		return false, nil
+	}
+
+	if !util.IsExist(filePath) {
+		return false, nil
+	}
+
+	output, err := sha256sum(filePath)
+	if err != nil {
+		return false, errors.Wrap(err, fmt.Sprintf("Failed to check SHA256 of %s", filePath))
+	}
+
+	return output == checksum, nil
+}
+
 func sha256sum(path string) (string, error) {
 	file, err := os.Open(path)
 	if err != nil {

From 2e4448e00ed7451f85e0bb8a263225b6a899570a Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Sun, 5 May 2024 12:15:29 +0800
Subject: [PATCH 15/31] feat: support save images from index

---
 cmd/kk/cmd/artifact/export.go           |  2 ++
 cmd/kk/pkg/common/artifact_runtime.go   |  1 +
 cmd/kk/pkg/images/module.go             |  3 ++-
 cmd/kk/pkg/images/tasks.go              | 10 +++++++---
 cmd/kk/pkg/pipelines/artifact_export.go |  6 +++---
 5 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/cmd/kk/cmd/artifact/export.go b/cmd/kk/cmd/artifact/export.go
index 0b8b1d6d6..0ccff69cd 100644
--- a/cmd/kk/cmd/artifact/export.go
+++ b/cmd/kk/cmd/artifact/export.go
@@ -34,6 +34,7 @@ type ArtifactExportOptions struct {
 	Output             string
 	CriSocket          string
 	DownloadCmd        string
+	ImageStartIndex    int
 	SkipRemoveArtifact bool
 }
 
@@ -93,6 +94,7 @@ func (o *ArtifactExportOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&o.Output, "output", "o", "", "Path to a output path")
 	cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s",
 		`The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`)
+	cmd.Flags().IntVarP(&o.ImageStartIndex, "image-start-index", "", 0, "Save images from specific index, default to 0")
 	cmd.Flags().BoolVarP(&o.SkipRemoveArtifact, "skip-remove-artifact", "", false, "Skip remove artifact")
 
 }
diff --git a/cmd/kk/pkg/common/artifact_runtime.go b/cmd/kk/pkg/common/artifact_runtime.go
index c9ad51170..847774dec 100644
--- a/cmd/kk/pkg/common/artifact_runtime.go
+++ b/cmd/kk/pkg/common/artifact_runtime.go
@@ -35,6 +35,7 @@ type ArtifactArgument struct {
 	Debug              bool
 	IgnoreErr          bool
 	DownloadCommand    func(path, url string) string
+	ImageStartIndex    int
 	SkipRemoveArtifact bool
 }
 
diff --git a/cmd/kk/pkg/images/module.go b/cmd/kk/pkg/images/module.go
index cc4843e02..15650c85c 100644
--- a/cmd/kk/pkg/images/module.go
+++ b/cmd/kk/pkg/images/module.go
@@ -49,6 +49,7 @@ func (p *PullModule) Init() {
 
 type CopyImagesToLocalModule struct {
 	common.ArtifactModule
+	ImageStartIndex int
 }
 
 func (c *CopyImagesToLocalModule) Init() {
@@ -58,7 +59,7 @@ func (c *CopyImagesToLocalModule) Init() {
 	copyImage := &task.LocalTask{
 		Name:   "SaveImages",
 		Desc:   "Copy images to a local OCI path from registries",
-		Action: new(SaveImages),
+		Action: &SaveImages{ImageStartIndex: c.ImageStartIndex},
 	}
 
 	c.Tasks = []task.Interface{
diff --git a/cmd/kk/pkg/images/tasks.go b/cmd/kk/pkg/images/tasks.go
index f431b51db..8ddf40490 100644
--- a/cmd/kk/pkg/images/tasks.go
+++ b/cmd/kk/pkg/images/tasks.go
@@ -155,6 +155,7 @@ func GetImage(runtime connector.ModuleRuntime, kubeConf *common.KubeConf, name s
 
 type SaveImages struct {
 	common.ArtifactAction
+	ImageStartIndex int
 }
 
 func (s *SaveImages) Execute(runtime connector.Runtime) error {
@@ -164,7 +165,10 @@ func (s *SaveImages) Execute(runtime connector.Runtime) error {
 	if err := coreutil.Mkdir(dirName); err != nil {
 		return errors.Wrapf(errors.WithStack(err), "mkdir %s failed", dirName)
 	}
-	for _, image := range s.Manifest.Spec.Images {
+	for index, image := range s.Manifest.Spec.Images {
+		if s.ImageStartIndex > index {
+			continue
+		}
 		if err := validateImageName(image); err != nil {
 			return err
 		}
@@ -186,8 +190,8 @@ func (s *SaveImages) Execute(runtime connector.Runtime) error {
 			// oci:./kubekey/artifact/images:kubesphere:kube-apiserver:v1.21.5-amd64
 			// oci:./kubekey/artifact/images:kubesphere:kube-apiserver:v1.21.5-arm-v7
 			destName := fmt.Sprintf("oci:%s:%s:%s-%s%s", dirName, imageFullName[1], suffixImageName(imageFullName[2:]), arch, variant)
-			logger.Log.Infof("Source: %s", srcName)
-			logger.Log.Infof("Destination: %s", destName)
+			logger.Log.Infof("[%d]Source: %s", index, srcName)
+			logger.Log.Infof("[%d]Destination: %s", index, destName)
 
 			o := &CopyImageOptions{
 				srcImage: &srcImageOptions{
diff --git a/cmd/kk/pkg/pipelines/artifact_export.go b/cmd/kk/pkg/pipelines/artifact_export.go
index a47db6857..cb9de98e3 100644
--- a/cmd/kk/pkg/pipelines/artifact_export.go
+++ b/cmd/kk/pkg/pipelines/artifact_export.go
@@ -34,7 +34,7 @@ import (
 func NewArtifactExportPipeline(runtime *common.ArtifactRuntime) error {
 	m := []module.Module{
 		&confirm.CheckFileExistModule{FileName: runtime.Arg.Output},
-		&images.CopyImagesToLocalModule{},
+		&images.CopyImagesToLocalModule{ImageStartIndex: runtime.Arg.ImageStartIndex},
 		&binaries.ArtifactBinariesModule{},
 		&artifact.RepositoryModule{},
 		&artifact.ArchiveModule{},
@@ -58,7 +58,7 @@ func NewArtifactExportPipeline(runtime *common.ArtifactRuntime) error {
 func NewK3sArtifactExportPipeline(runtime *common.ArtifactRuntime) error {
 	m := []module.Module{
 		&confirm.CheckFileExistModule{FileName: runtime.Arg.Output},
-		&images.CopyImagesToLocalModule{},
+		&images.CopyImagesToLocalModule{ImageStartIndex: runtime.Arg.ImageStartIndex},
 		&binaries.K3sArtifactBinariesModule{},
 		&artifact.RepositoryModule{},
 		&artifact.ArchiveModule{},
@@ -82,7 +82,7 @@ func NewK3sArtifactExportPipeline(runtime *common.ArtifactRuntime) error {
 func NewK8eArtifactExportPipeline(runtime *common.ArtifactRuntime) error {
 	m := []module.Module{
 		&confirm.CheckFileExistModule{FileName: runtime.Arg.Output},
-		&images.CopyImagesToLocalModule{},
+		&images.CopyImagesToLocalModule{ImageStartIndex: runtime.Arg.ImageStartIndex},
 		&binaries.K8eArtifactBinariesModule{},
 		&artifact.RepositoryModule{},
 		&artifact.ArchiveModule{},

From 50b016faf2c4f268327b0d6750d8af08efbcd33d Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Sun, 5 May 2024 12:28:59 +0800
Subject: [PATCH 16/31] fix: image start index not work

---
 cmd/kk/cmd/artifact/export.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmd/kk/cmd/artifact/export.go b/cmd/kk/cmd/artifact/export.go
index 0ccff69cd..3e27bfd28 100644
--- a/cmd/kk/cmd/artifact/export.go
+++ b/cmd/kk/cmd/artifact/export.go
@@ -81,6 +81,7 @@ func (o *ArtifactExportOptions) Run() error {
 		ManifestFile:       o.ManifestFile,
 		Output:             o.Output,
 		CriSocket:          o.CriSocket,
+		ImageStartIndex:    o.ImageStartIndex,
 		Debug:              o.CommonOptions.Verbose,
 		IgnoreErr:          o.CommonOptions.IgnoreErr,
 		SkipRemoveArtifact: o.SkipRemoveArtifact,

From 975b7140ea99633ec1a67bcbaf53c9c8b2c2ea95 Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Sun, 5 May 2024 23:03:33 +0800
Subject: [PATCH 17/31] feat: support docker daemon push

---
 cmd/kk/pkg/images/module.go   |  7 ++++---
 cmd/kk/pkg/images/prepares.go | 34 ++++++++++++++++++++++++++++++++++
 cmd/kk/pkg/images/tasks.go    | 25 ++++++++++++++-----------
 cmd/kk/pkg/images/utils.go    | 14 +++++++++++++-
 4 files changed, 65 insertions(+), 15 deletions(-)
 create mode 100644 cmd/kk/pkg/images/prepares.go

diff --git a/cmd/kk/pkg/images/module.go b/cmd/kk/pkg/images/module.go
index c64f85218..97b4fa46b 100644
--- a/cmd/kk/pkg/images/module.go
+++ b/cmd/kk/pkg/images/module.go
@@ -89,9 +89,10 @@ func (c *CopyImagesToRegistryModule) Init() {
 	}
 
 	pushManifest := &task.LocalTask{
-		Name:   "PushManifest",
-		Desc:   "Push multi-arch manifest to private registry",
-		Action: new(PushManifest),
+		Name:    "PushManifest",
+		Desc:    "Push multi-arch manifest to private registry",
+		Prepare: &ShouldPushManifest{ImageTransport: c.ImageTransport},
+		Action:  new(PushManifest),
 	}
 
 	c.Tasks = []task.Interface{
diff --git a/cmd/kk/pkg/images/prepares.go b/cmd/kk/pkg/images/prepares.go
new file mode 100644
index 000000000..a61c2dde4
--- /dev/null
+++ b/cmd/kk/pkg/images/prepares.go
@@ -0,0 +1,34 @@
+/*
+ Copyright 2024 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package images
+
+import (
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
+)
+
+type ShouldPushManifest struct {
+	common.KubePrepare
+	ImageTransport string
+}
+
+func (m *ShouldPushManifest) PreCheck(_ connector.Runtime) (bool, error) {
+	if m.ImageTransport != common.DockerDaemon {
+		return true, nil
+	}
+	return false, nil
+}
diff --git a/cmd/kk/pkg/images/tasks.go b/cmd/kk/pkg/images/tasks.go
index d33041c42..f66af2b5e 100644
--- a/cmd/kk/pkg/images/tasks.go
+++ b/cmd/kk/pkg/images/tasks.go
@@ -184,9 +184,9 @@ func (s *SaveImages) Execute(runtime connector.Runtime) error {
 				variant = "-" + variant
 			}
 			// Ex:
-			// oci:./kubekey/artifact/images:kubesphere:kube-apiserver:v1.21.5-amd64
-			// oci:./kubekey/artifact/images:kubesphere:kube-apiserver:v1.21.5-arm-v7
-			destName := fmt.Sprintf("oci:%s:%s:%s-%s%s", dirName, imageFullName[1], suffixImageName(imageFullName[2:]), arch, variant)
+			// oci:./kubekey/artifact/images:docker.io/kubesphere/kube-apiserver:v1.21.5-amd64
+			// oci:./kubekey/artifact/images:docker.io/kubesphere/kube-apiserver:v1.21.5-arm-v7
+			destName := fmt.Sprintf("oci:%s:%s-%s%s", dirName, image, arch, variant)
 			logger.Log.Infof("Source: %s", srcName)
 			logger.Log.Infof("Destination: %s", destName)
 
@@ -262,18 +262,21 @@ func (c *CopyImagesToRegistry) Execute(runtime connector.Runtime) error {
 		ref := m.Annotations.RefName
 
 		// Ex:
-		// calico:cni:v3.20.0-amd64
-		nameArr := strings.Split(ref, ":")
-		if len(nameArr) != 3 {
+		// docker.io/calico/cni:v3.20.0-amd64
+		repoAddr, namespace, imageName, imageTag, err := parseImageFullName(ref)
+		if err != nil {
 			return errors.Errorf("invalid ref name: %s", ref)
 		}
 
+		if c.ImageTransport != common.DockerDaemon {
+			repoAddr = c.KubeConf.Cluster.Registry.PrivateRegistry
+		}
 		image := Image{
-			RepoAddr:          c.KubeConf.Cluster.Registry.PrivateRegistry,
-			Namespace:         nameArr[0],
+			RepoAddr:          repoAddr,
+			Namespace:         namespace,
 			NamespaceOverride: c.KubeConf.Cluster.Registry.NamespaceOverride,
-			Repo:              nameArr[1],
-			Tag:               nameArr[2],
+			Repo:              imageName,
+			Tag:               imageTag,
 		}
 
 		uniqueImage, p := ParseImageWithArchTag(image.ImageName())
@@ -307,7 +310,7 @@ func (c *CopyImagesToRegistry) Execute(runtime connector.Runtime) error {
 		}
 
 		srcName := fmt.Sprintf("oci:%s:%s", imagesPath, ref)
-		destName := formatImageName(c.ImageTransport, image.ImageName())
+		destName := formatImageName(c.ImageTransport, uniqueImage)
 		logger.Log.Infof("Source: %s", srcName)
 		logger.Log.Infof("Destination: %s", destName)
 
diff --git a/cmd/kk/pkg/images/utils.go b/cmd/kk/pkg/images/utils.go
index aebdd989f..ad35edd95 100644
--- a/cmd/kk/pkg/images/utils.go
+++ b/cmd/kk/pkg/images/utils.go
@@ -183,7 +183,19 @@ func validateImageName(imageFullName string) error {
 	return nil
 }
 
-func suffixImageName(imageFullName []string) string {
+func parseImageFullName(imageFullName string) (string, string, string, string, error) {
+	if err := validateImageName(imageFullName); err != nil {
+		return "", "", "", "", err
+	}
+	image := strings.Split(imageFullName, "/")
+	partsNum := len(image)
+	repoAddr := image[0]
+	namespace := concatImageName(image[1 : partsNum-1])
+	nameAndTag := strings.Split(image[partsNum-1], ":")
+	return repoAddr, namespace, nameAndTag[0], nameAndTag[1], nil
+}
+
+func concatImageName(imageFullName []string) string {
 	if len(imageFullName) >= 2 {
 		return strings.Join(imageFullName, "/")
 	}

From c74e784ac1f51edbb9d64c4df3c86ad8eb9f27e5 Mon Sep 17 00:00:00 2001
From: guangwu <guoguangwug@gmail.com>
Date: Wed, 8 May 2024 15:10:46 +0800
Subject: [PATCH 18/31] fix: close file

---
 pkg/service/operation/file/template.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/service/operation/file/template.go b/pkg/service/operation/file/template.go
index 500a5ce05..46c8b1833 100644
--- a/pkg/service/operation/file/template.go
+++ b/pkg/service/operation/file/template.go
@@ -71,6 +71,7 @@ func (t *Template) RenderToLocal() error {
 	if err != nil {
 		return err
 	}
+	defer f.Close()
 
 	if err := t.template.Execute(f, t.data); err != nil {
 		return err

From 6301f9b44ec4a963e35a98032456a50a0698b9a8 Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Thu, 9 May 2024 18:39:44 +0800
Subject: [PATCH 19/31] fix: iso local path miss

---
 cmd/kk/pkg/artifact/tasks.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmd/kk/pkg/artifact/tasks.go b/cmd/kk/pkg/artifact/tasks.go
index 3e7e34202..180b9ef09 100644
--- a/cmd/kk/pkg/artifact/tasks.go
+++ b/cmd/kk/pkg/artifact/tasks.go
@@ -52,6 +52,7 @@ func (d *DownloadISOFile) Execute(runtime connector.Runtime) error {
 		}
 		if checksumEqual {
 			logger.Log.Infof("Skip download exists iso file %s", fileName)
+			d.Manifest.Spec.OperatingSystems[i].Repository.Iso.LocalPath = filePath
 			continue
 		}
 

From b8f7b4f60711b441a4f0273057ab8f975042983b Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Fri, 10 May 2024 00:36:28 +0000
Subject: [PATCH 20/31] update README.md [skip ci]

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 9600381e6..22df4da32 100644
--- a/README.md
+++ b/README.md
@@ -402,6 +402,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/xrwang8"><img src="https://avatars.githubusercontent.com/u/68765051?v=4?s=100" width="100px;" alt="Ronald Fletcher"/><br /><sub><b>Ronald Fletcher</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=xrwang8" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/baikjy0215"><img src="https://avatars.githubusercontent.com/u/110450904?v=4?s=100" width="100px;" alt="baikjy0215"/><br /><sub><b>baikjy0215</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=baikjy0215" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/knowmost"><img src="https://avatars.githubusercontent.com/u/167442703?v=4?s=100" width="100px;" alt="knowmost"/><br /><sub><b>knowmost</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=knowmost" title="Documentation">📖</a></td>
     </tr>
   </tbody>
 </table>

From 4760811f5e8a3bec48a6f05350fe6fb22600c1ec Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Fri, 10 May 2024 00:36:29 +0000
Subject: [PATCH 21/31] update CONTRIBUTORS.md [skip ci]

---
 CONTRIBUTORS.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index d882bdd97..e127b4c01 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -118,6 +118,7 @@ Contributions of any kind are welcome! Thanks goes to these wonderful contributo
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/xrwang8"><img src="https://avatars.githubusercontent.com/u/68765051?v=4?s=100" width="100px;" alt="Ronald Fletcher"/><br /><sub><b>Ronald Fletcher</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=xrwang8" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/baikjy0215"><img src="https://avatars.githubusercontent.com/u/110450904?v=4?s=100" width="100px;" alt="baikjy0215"/><br /><sub><b>baikjy0215</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=baikjy0215" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/knowmost"><img src="https://avatars.githubusercontent.com/u/167442703?v=4?s=100" width="100px;" alt="knowmost"/><br /><sub><b>knowmost</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=knowmost" title="Documentation">📖</a></td>
     </tr>
   </tbody>
 </table>

From f135c1dde678a5986181e8c114fabdd1f196ddba Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Fri, 10 May 2024 00:36:30 +0000
Subject: [PATCH 22/31] update README_zh-CN.md [skip ci]

---
 README_zh-CN.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README_zh-CN.md b/README_zh-CN.md
index 39619500c..64f3cced8 100644
--- a/README_zh-CN.md
+++ b/README_zh-CN.md
@@ -419,6 +419,7 @@ kubectl completion bash >/etc/bash_completion.d/kubectl
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/xrwang8"><img src="https://avatars.githubusercontent.com/u/68765051?v=4?s=100" width="100px;" alt="Ronald Fletcher"/><br /><sub><b>Ronald Fletcher</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=xrwang8" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/baikjy0215"><img src="https://avatars.githubusercontent.com/u/110450904?v=4?s=100" width="100px;" alt="baikjy0215"/><br /><sub><b>baikjy0215</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=baikjy0215" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/knowmost"><img src="https://avatars.githubusercontent.com/u/167442703?v=4?s=100" width="100px;" alt="knowmost"/><br /><sub><b>knowmost</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=knowmost" title="Documentation">📖</a></td>
     </tr>
   </tbody>
 </table>

From 3d112fb8d2bf702ee89bb7b228a428c4d36f2ade Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Fri, 10 May 2024 00:36:31 +0000
Subject: [PATCH 23/31] update .all-contributorsrc [skip ci]

---
 .all-contributorsrc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.all-contributorsrc b/.all-contributorsrc
index ea78cea7f..ced0e693b 100644
--- a/.all-contributorsrc
+++ b/.all-contributorsrc
@@ -796,6 +796,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "knowmost",
+      "name": "knowmost",
+      "avatar_url": "https://avatars.githubusercontent.com/u/167442703?v=4",
+      "profile": "https://github.com/knowmost",
+      "contributions": [
+        "doc"
+      ]
     }
   ],
   "contributorsPerLine": 7,

From 0084709116937c2c1b5e4dd721961e19f378a6ca Mon Sep 17 00:00:00 2001
From: ks-ci-bot <47586280+ks-ci-bot@users.noreply.github.com>
Date: Wed, 15 May 2024 00:52:14 +0000
Subject: [PATCH 24/31] Add new kubernetes version

---
 docs/kubernetes-versions.md | 1 +
 version/components.json     | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/docs/kubernetes-versions.md b/docs/kubernetes-versions.md
index 6cbbc67bb..7db3a39ba 100644
--- a/docs/kubernetes-versions.md
+++ b/docs/kubernetes-versions.md
@@ -134,6 +134,7 @@
 | v1.28.7 | :white_check_mark: |
 | v1.28.8 | :white_check_mark: |
 | v1.28.9 | :white_check_mark: |
+| v1.28.10 | :white_check_mark: |
 | v1.29.0 | :white_check_mark: |
 | v1.29.1 | :white_check_mark: |
 | v1.29.2 | :white_check_mark: |
diff --git a/version/components.json b/version/components.json
index 18f925b25..c4c0f0e5a 100644
--- a/version/components.json
+++ b/version/components.json
@@ -134,6 +134,7 @@
             "v1.28.7": "8aa005bdf6af43e47fc818b26f4cb9f361aae8ec4390519e8d4033be65fbef2b",
             "v1.28.8": "c11946cbfd962e1197062534514226cfd70230349e6343ff3ecebfca5476ee64",
             "v1.28.9": "a4d8acf0a74cb1d07d96a1a34148f54c6420874221af16d8ec902d9bffc7ef89",
+            "v1.28.10": "1a344d34755c5f005120308f09a730e7564c8f857de6606b6bc5f18a69606e5a",
             "v1.29.0": "629d4630657caace9c819fd3797f4a70c397fbd41a2a7e464a0507dad675d52c",
             "v1.29.1": "d4d81d9020b550c896376fb9e0586a9f15a332175890d061619b52b3e9bc6cbd",
             "v1.29.2": "2d4e4fa8685bcbfb661cb41050cd4756f50a7aa147f68492d51a99f9cdfd69ac",
@@ -275,6 +276,7 @@
             "v1.28.7": "f556e49494737f97a15bf15bb4b27d45f8747b477302cdfd22dd61816bc02203",
             "v1.28.8": "e0f47adc69ef84e2f6c42cc341b8a790904a929ad10ed1c23c2e822ec804e247",
             "v1.28.9": "cd6aefad8144a9771fd470529ff14be2675df7b561f7c56dee3fed4f81332dc4",
+            "v1.28.10": "f6809d72ed1bf6fde460e48e5c714c3bc92f680e328defa9bd592a796347b644",
             "v1.29.0": "bbddee2d46d2e1643ae3623698b45b13aa2e858616d61c642f2f49e5bb14c980",
             "v1.29.1": "3bff8c50c104c45e416cce9991706c6ac46365f0defbcd54f8cf4ace0fa68dcf",
             "v1.29.2": "e05720feb9d2d67eff25b0156a5c22e2de37be2ffab4e1f4d31e8c526fafd0e1",
@@ -418,6 +420,7 @@
             "v1.28.7": "120b1495babc4364f7e16a9d0f8b8e6b6f78316d047e4f6de77b5569b05813c7",
             "v1.28.8": "049b412a5861255cd3922f612acb79ab51135e166c5d80acf12fba9179eebf0c",
             "v1.28.9": "f3af46cff11c675a80d91ebb38ebc4e85a9f813ce93e56ee131e7fea1491b786",
+            "v1.28.10": "a361e744aaeef4539f0636ecd1827c85207a5f2b0c2b0a98dbbce1498061f509",
             "v1.29.0": "e1c38137db8d8777eed8813646b59bf4d22d19b9011ab11dc28e2e34f6b80a05",
             "v1.29.1": "1b1975c58d38be1a99a8bcba4564ac489afd223b0abe9f2ab08bbde89d2412a3",
             "v1.29.2": "f71a85039b71fe08f1c063a93d61a1c952dc8f9a8c6be9b13fbdac8f0d9ff960",
@@ -559,6 +562,7 @@
             "v1.28.7": "e2c98b39b0b0745ef3e30febaeb8eaaf31ec721012405bd0dcf25e84026c221e",
             "v1.28.8": "90d61f40b7bb061b0fc6d08b8b9ddae51f90863c899b098e19eaa89dc855f2c0",
             "v1.28.9": "312471ad255acfcdeea2c5849b171467af4518e96d69d727a3197ff334e9299d",
+            "v1.28.10": "feae161e374ee0155b5263cda339f30f16b525631535a003be7aa437661e1580",
             "v1.29.0": "0e0e4544c2a0a3475529154b7534d0d58683466efa04a2bb2e763b476db0bb16",
             "v1.29.1": "e46417ab1ceae995f0e00d4177959a36ed34b807829422bc9dda70b263fe5c5d",
             "v1.29.2": "9b4aa572d4cd51a41b1067161d961423d0d12b120fb636ea887a12a975d4b19a",
@@ -702,6 +706,7 @@
             "v1.28.7": "aff42d3167685e4d8e86fda0ad9c6ce6ec6c047bc24d608041d54717a18192ba",
             "v1.28.8": "e02aad5c0bac52c970700b814645b62c4f18b634144398ac344875dbaf1072f8",
             "v1.28.9": "b4693d0b22f509250694b10c7727c42b427d570af04f2065fe23a55d6c0051f1",
+            "v1.28.10": "389c17a9700a4b01ebb055e39b8bc0886330497440dde004b5ed90f2a3a028db",
             "v1.29.0": "0e03ab096163f61ab610b33f37f55709d3af8e16e4dcc1eb682882ef80f96fd5",
             "v1.29.1": "69ab3a931e826bf7ac14d38ba7ca637d66a6fcb1ca0e3333a2cafdf15482af9f",
             "v1.29.2": "7816d067740f47f949be826ac76943167b7b3a38c4f0c18b902fffa8779a5afa",
@@ -843,6 +848,7 @@
             "v1.28.7": "13d547495bdea49b223fe06bffb6d2bef96436634847f759107655aa80fc990e",
             "v1.28.8": "93d60dd36093b4c719f1f1bafcf59437c17cb2209341c7c94771e7dd9acdab33",
             "v1.28.9": "e0341d3973213f8099e7fcbbf6d1d506967bc2b7a4faac3fb3b4340f226e9b2f",
+            "v1.28.10": "e659d23d442c2706debe5b96742326c0a1e1d7b5c695a9fe7dfe8ea7402caee8",
             "v1.29.0": "8f7a4bd6bae900a4ddab12bd1399aa652c0d59ea508f39b910e111d248893ff7",
             "v1.29.1": "96d6dc7b2bdcd344ce58d17631c452225de5bbf59b83fd3c89c33c6298fb5d8b",
             "v1.29.2": "3507ecb4224cf05ae2151a98d4932253624e7762159936d5347b19fe037655ca",

From 0d906b17a9816c6c81dda3ee4cc829e08d03d49c Mon Sep 17 00:00:00 2001
From: ks-ci-bot <47586280+ks-ci-bot@users.noreply.github.com>
Date: Fri, 17 May 2024 00:58:25 +0000
Subject: [PATCH 25/31] Add new kubernetes version

---
 docs/kubernetes-versions.md |  3 +++
 version/components.json     | 30 ++++++++++++++++++++++++------
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/docs/kubernetes-versions.md b/docs/kubernetes-versions.md
index 7db3a39ba..32736227f 100644
--- a/docs/kubernetes-versions.md
+++ b/docs/kubernetes-versions.md
@@ -124,6 +124,7 @@
 | v1.27.11 | :white_check_mark: |
 | v1.27.12 | :white_check_mark: |
 | v1.27.13 | :white_check_mark: |
+| v1.27.14 | :white_check_mark: |
 | v1.28.0 | :white_check_mark: |
 | v1.28.1 | :white_check_mark: |
 | v1.28.2 | :white_check_mark: |
@@ -140,4 +141,6 @@
 | v1.29.2 | :white_check_mark: |
 | v1.29.3 | :white_check_mark: |
 | v1.29.4 | :white_check_mark: |
+| v1.29.5 | :white_check_mark: |
 | v1.30.0 | :white_check_mark: |
+| v1.30.1 | :white_check_mark: |
diff --git a/version/components.json b/version/components.json
index c4c0f0e5a..47e3bf10b 100644
--- a/version/components.json
+++ b/version/components.json
@@ -124,6 +124,7 @@
             "v1.27.11": "31bf446a712fb08190838c35d1f4c93b0f975708c59634a5dc3d8915a241c83e",
             "v1.27.12": "06ee36cc80cfdfc01c937d750783d3ca6169a3da76382c7af3dd172d9f6bfa4e",
             "v1.27.13": "b88c30b7067f095b7fa02c5560cc50d6e69a5a9fecc606ef477dc7efc86453b9",
+            "v1.27.14": "1ce264643e521494e111b1c9ee59694a54d1f2464bbac3a7a531324ffeae0182",
             "v1.28.0": "12ea68bfef0377ccedc1a7c98a05ea76907decbcf1e1ec858a60a7b9b73211bb",
             "v1.28.1": "6134dbc92dcb83c3bae1a8030f7bb391419b5d13ea94badd3a79b7ece75b2736",
             "v1.28.2": "6a4808230661c69431143db2e200ea2d021c7f1b1085e6353583075471310d00",
@@ -140,7 +141,9 @@
             "v1.29.2": "2d4e4fa8685bcbfb661cb41050cd4756f50a7aa147f68492d51a99f9cdfd69ac",
             "v1.29.3": "6abaa1208bf40b6d1f49e518bd68c8ae4a1be0c5b7d3e45d87979999ab070d8b",
             "v1.29.4": "ea20ab064f716ab7f69a36d72df340257b31c9721ea86e1cf9d70b35999ddeea",
-            "v1.30.0": "29f4232c50e6524abba3443ff3b9948d386964d79eb8dfefb409e1f8a8434c14"
+            "v1.29.5": "e424dcdbe661314b6ca1fcc94726eb554bc3f4392b060b9626f9df8d7d44d42c",
+            "v1.30.0": "29f4232c50e6524abba3443ff3b9948d386964d79eb8dfefb409e1f8a8434c14",
+            "v1.30.1": "651faa3bbbfb368ed00460e4d11732614310b690b767c51810a7b638cc0961a2"
         },
         "arm64": {
             "v1.19.0": "db1c432646e6e6484989b6f7191f3610996ac593409f12574290bfc008ea11f5",
@@ -266,6 +269,7 @@
             "v1.27.11": "b8452d6c3f1331beb3d5fa42466a9bc96638a76c40980dba9822300f230c0858",
             "v1.27.12": "e74d47c14b5a251cff961dcce92cd632abcfd0fba4a07e78f0a5a5b2796e4b84",
             "v1.27.13": "f334ba0612fada50e98a7ea56b686b35c22f0e3243ec2210f2a6a87e841a139f",
+            "v1.27.14": "cb840eb83404047cdafec0b15054023c90a47491b56d2dceba1050040f37cc7e",
             "v1.28.0": "b9b473d2d9136559b19eb465006af77df45c09862cd7ce6673a33aae517ff5ab",
             "v1.28.1": "7d2f68917470a5d66bd2a7d62897f59cb4afaeffb2f26c028afa119acd8c3fc8",
             "v1.28.2": "010789a94cf512d918ec4a3ef8ec734dea0061d89a8293059ef9101ca1bf6bff",
@@ -282,7 +286,9 @@
             "v1.29.2": "e05720feb9d2d67eff25b0156a5c22e2de37be2ffab4e1f4d31e8c526fafd0e1",
             "v1.29.3": "ce2e4c230f954e59ae77e34c4ff2ae08cad3970505ae1e21b6337e6d83b21682",
             "v1.29.4": "438287a91e08cbefecab79be8ac893a935c3dbf6e87bea895fb99f2bc38cf06e",
-            "v1.30.0": "c36afd28921303e6db8e58274de16c60a80a1e75030fc3c4e9c4ed6249b6b696"
+            "v1.29.5": "d4db8c514f2764edc039462c218dbcd316577f76f21b209b76e9a4b1f08e3100",
+            "v1.30.0": "c36afd28921303e6db8e58274de16c60a80a1e75030fc3c4e9c4ed6249b6b696",
+            "v1.30.1": "bda423cb4b9d056f99a2ef116bdf227fadbc1c3309fa3d76da571427a7f41478"
         }
     },
     "kubelet": {
@@ -410,6 +416,7 @@
             "v1.27.11": "2ce92a5d8985b93bd8ffc4f5519cd79bf2f844590aa38228a3d809c5bf5986e0",
             "v1.27.12": "aae861a21913c274228ccdad1609b370e5198c9f4b39b8924b20a7ffe7f148e0",
             "v1.27.13": "ed68df2a77f3057ab47f57eacb6e9310e91731e4f43c58a3c3b5c857d78d0080",
+            "v1.27.14": "f28defa43f80f82ce909940c1b57b71cba1fcf0de6fc4723e798ef5c72376c28",
             "v1.28.0": "bfb6b977100963f2879a33e5fbaa59a5276ba829a957a6819c936e9c1465f981",
             "v1.28.1": "2bc22332f44f8fcd3fce57879fd873f977949ebd261571fbae31fbb2713a5dd3",
             "v1.28.2": "17edb866636f14eceaad58c56eab12af7ab3be3c78400aff9680635d927f1185",
@@ -426,7 +433,9 @@
             "v1.29.2": "f71a85039b71fe08f1c063a93d61a1c952dc8f9a8c6be9b13fbdac8f0d9ff960",
             "v1.29.3": "d8b55a2f8a87c8cd2cbf867d76d1d7f98b7198a740db19bad6ed7b8b813de771",
             "v1.29.4": "58571f0ed62543a9bbac541e52c15d8385083113a463e23aec1341d0b5043939",
-            "v1.30.0": "32a32ec3d7e7f8b2648c9dd503ce9ef63b4af1d1677f5b5aed7846fb02d66f18"
+            "v1.29.5": "261dc3f3c384d138835fe91a02071c642af94abb0cca56ebc04719240440944c",
+            "v1.30.0": "32a32ec3d7e7f8b2648c9dd503ce9ef63b4af1d1677f5b5aed7846fb02d66f18",
+            "v1.30.1": "87bd6e5de9c0769c605da5fedb77a35c8b764e3bda1632447883c935dcf219d3"
         },
         "arm64": {
             "v1.19.0": "d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf",
@@ -552,6 +561,7 @@
             "v1.27.11": "e81987a864fb47afe14f65fa4e93760bc19c424335e0f0540c6c725b727ce22a",
             "v1.27.12": "0d7d2d25c8b909d6cec7c1c2a5bfe51428ec33eaa5e8b209c718b77983e9dcba",
             "v1.27.13": "d7bfb14d0b0fc2c41074baf02617cf98589fd029fb3539ea017825e36371f19c",
+            "v1.27.14": "31c97a723021ccc90a47a15ad1de1ffdf58ba109aea922eb359ad2fcb8e8ce4b",
             "v1.28.0": "05dd12e35783cab4960e885ec0e7d0e461989b94297e7bea9018ccbd15c4dce9",
             "v1.28.1": "9b7fa64b2785da4a38768377961e227f8da629c56a5df43ca1b665dd07b56f3c",
             "v1.28.2": "32269e9ec38c561d028b65c3048ea6a100e1292cbe9e505565222455c8096577",
@@ -568,7 +578,9 @@
             "v1.29.2": "9b4aa572d4cd51a41b1067161d961423d0d12b120fb636ea887a12a975d4b19a",
             "v1.29.3": "891dce19ed0eae34050c2eca0454204892e97bfe1a926f988cd044a987a9c7c9",
             "v1.29.4": "dc4bb6ea6cd35b024d63cc20d1c1800a9c695bd6f70411c57358d7c407513b00",
-            "v1.30.0": "fa887647422d34f3c7cc5b30fefcf97084d2c3277eff237c5808685ba8e4b15a"
+            "v1.29.5": "0d4328a3c67e4f0dbf270fa49343f3eab9316adde1a1bd2a857fa56876a9aff1",
+            "v1.30.0": "fa887647422d34f3c7cc5b30fefcf97084d2c3277eff237c5808685ba8e4b15a",
+            "v1.30.1": "c45049b829af876588ec1a30def3884ce77c2c175cd77485d49c78d2064a38fb"
         }
     },
     "kubectl": {
@@ -696,6 +708,7 @@
             "v1.27.11": "7ae327978a1edb43700070c86f5fd77215792c6b58a7ea70192647e0da848e29",
             "v1.27.12": "d639eda39be2dce42fbec21e038942ab5734541715e3ea5fb29c9ad76686bd7f",
             "v1.27.13": "e991f163197cbd85bbff22f656a74d48b69db5addfa43cc04cca0cf5328f57f1",
+            "v1.27.14": "1d2431c68bb6dfa9de3cd40fd66d97a9ac73593c489f9467249eea43e9c16a1e",
             "v1.28.0": "4717660fd1466ec72d59000bb1d9f5cdc91fac31d491043ca62b34398e0799ce",
             "v1.28.1": "e7a7d6f9d06fab38b4128785aa80f65c54f6675a0d2abef655259ddd852274e1",
             "v1.28.2": "c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5",
@@ -712,7 +725,9 @@
             "v1.29.2": "7816d067740f47f949be826ac76943167b7b3a38c4f0c18b902fffa8779a5afa",
             "v1.29.3": "89c0435cec75278f84b62b848b8c0d3e15897d6947b6c59a49ddccd93d7312bf",
             "v1.29.4": "10e343861c3cb0010161e703307ba907add2aeeeaffc6444779ad915f9889c88",
-            "v1.30.0": "7c3807c0f5c1b30110a2ff1e55da1d112a6d0096201f1beb81b269f582b5d1c5"
+            "v1.29.5": "603c8681fc0d8609c851f9cc58bcf55eeb97e2934896e858d0232aa8d1138366",
+            "v1.30.0": "7c3807c0f5c1b30110a2ff1e55da1d112a6d0096201f1beb81b269f582b5d1c5",
+            "v1.30.1": "5b86f0b06e1a5ba6f8f00e2b01e8ed39407729c4990aeda961f83a586f975e8a"
         },
         "arm64": {
             "v1.19.0": "d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f",
@@ -838,6 +853,7 @@
             "v1.27.11": "d30e1aa873e78eb376ddee3c785aa78c44eddc56ce2ef901dac1ce0c2c4f50b0",
             "v1.27.12": "bfc6cb71041ebc0f048402988eccc107cfff2b866c864231c9ada05ab328e5bf",
             "v1.27.13": "4838ad8f3902c928f6139d69eba962b1cc2471511c828885650d728d970594d1",
+            "v1.27.14": "29f3a1f520d929df38873c68dec73519c1e5e521140e01cf9d7701f7b5ffe4f3",
             "v1.28.0": "f5484bd9cac66b183c653abed30226b561f537d15346c605cc81d98095f1717c",
             "v1.28.1": "46954a604b784a8b0dc16754cfc3fa26aabca9fd4ffd109cd028bfba99d492f6",
             "v1.28.2": "ea6d89b677a8d9df331a82139bb90d9968131530b94eab26cee561531eff4c53",
@@ -854,7 +870,9 @@
             "v1.29.2": "3507ecb4224cf05ae2151a98d4932253624e7762159936d5347b19fe037655ca",
             "v1.29.3": "191a96b27e3c6ae28b330da4c9bfefc9592762670727df4fcf124c9f1d5a466a",
             "v1.29.4": "61537408eedcad064d7334384aed508a8aa1ea786311b87b505456a2e0535d36",
-            "v1.30.0": "669af0cf520757298ea60a8b6eb6b719ba443a9c7d35f36d3fb2fd7513e8c7d2"
+            "v1.29.5": "9ee9168def12ac6a6c0c6430e0f73175e756ed262db6040f8aa2121ad2c1f62e",
+            "v1.30.0": "669af0cf520757298ea60a8b6eb6b719ba443a9c7d35f36d3fb2fd7513e8c7d2",
+            "v1.30.1": "d90446719b815e3abfe7b2c46ddf8b3fda17599f03ab370d6e47b1580c0e869e"
         }
     },
     "etcd": {

From fddc79eb3dbfedd85d1ce551bd905debfe0a7d78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E8=AF=9A?= <cheng.zhang@magic-shield.com>
Date: Mon, 27 May 2024 16:22:51 +0800
Subject: [PATCH 26/31] add check support k8s version

---
 cmd/kk/pkg/bootstrap/confirm/tasks.go | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/cmd/kk/pkg/bootstrap/confirm/tasks.go b/cmd/kk/pkg/bootstrap/confirm/tasks.go
index 74227cb12..180f5bb29 100644
--- a/cmd/kk/pkg/bootstrap/confirm/tasks.go
+++ b/cmd/kk/pkg/bootstrap/confirm/tasks.go
@@ -21,8 +21,12 @@ import (
 	"fmt"
 	"os"
 	"regexp"
+	"slices"
 	"strings"
 
+	versionK8S "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
+
+	kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
@@ -106,6 +110,26 @@ func (i *InstallationConfirm) Execute(runtime connector.Runtime) error {
 	fmt.Println("https://github.com/kubesphere/kubekey#requirements-and-recommendations")
 	fmt.Println("")
 
+	// check k8s version is supported
+	k8sVersion := i.KubeConf.Cluster.Kubernetes.Version
+	if k8sVersion != kubekeyapiv1alpha2.DefaultKubeVersion {
+		suppportVersions := versionK8S.SupportedK8sVersionList()
+		if !slices.ContainsFunc(suppportVersions, func(s string) bool {
+			return strings.Contains(s, k8sVersion)
+		}) {
+			fmt.Printf("The Kubernetes version :%s isn't support.\n", k8sVersion)
+			fmt.Println("Use kk version --show-supported-k8s, show support k8s versions")
+			fmt.Println("")
+			stopFlag = true
+		} else {
+			fmt.Println("Install k8s with specify version: ", k8sVersion)
+			fmt.Println("")
+		}
+	} else {
+		fmt.Println("Install k8s with default version: ", kubekeyapiv1alpha2.DefaultKubeVersion)
+		fmt.Println("")
+	}
+
 	if i.KubeConf.Cluster.Kubernetes.IsAtLeastV124() && i.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
 		fmt.Println("[Notice]")
 		fmt.Println("For Kubernetes v1.24 and later, dockershim has been deprecated.")

From f4026ef85aebca78a0adb2a51cdbd57c22da2ac7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E8=AF=9A?= <cheng.zhang@magic-shield.com>
Date: Tue, 28 May 2024 12:10:40 +0800
Subject: [PATCH 27/31] use k8s.io/utils/strings/slices to replace builtin
 slices

---
 cmd/kk/pkg/bootstrap/confirm/tasks.go | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/cmd/kk/pkg/bootstrap/confirm/tasks.go b/cmd/kk/pkg/bootstrap/confirm/tasks.go
index 180f5bb29..0e4da5508 100644
--- a/cmd/kk/pkg/bootstrap/confirm/tasks.go
+++ b/cmd/kk/pkg/bootstrap/confirm/tasks.go
@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"os"
 	"regexp"
-	"slices"
 	"strings"
 
 	versionK8S "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
@@ -36,6 +35,7 @@ import (
 	"github.com/modood/table"
 	"github.com/pkg/errors"
 	versionutil "k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/utils/strings/slices"
 )
 
 // PreCheckResults defines the items to be checked.
@@ -114,11 +114,12 @@ func (i *InstallationConfirm) Execute(runtime connector.Runtime) error {
 	k8sVersion := i.KubeConf.Cluster.Kubernetes.Version
 	if k8sVersion != kubekeyapiv1alpha2.DefaultKubeVersion {
 		suppportVersions := versionK8S.SupportedK8sVersionList()
-		if !slices.ContainsFunc(suppportVersions, func(s string) bool {
-			return strings.Contains(s, k8sVersion)
-		}) {
-			fmt.Printf("The Kubernetes version :%s isn't support.\n", k8sVersion)
-			fmt.Println("Use kk version --show-supported-k8s, show support k8s versions")
+		if !strings.HasPrefix(k8sVersion, "v") {
+			k8sVersion = "v" + k8sVersion
+		}
+		if !slices.Contains(suppportVersions, k8sVersion) {
+			fmt.Printf("The Kubernetes version: %s isn't supported.\n", k8sVersion)
+			fmt.Println("Use kk version --show-supported-k8s,show supported k8s versions")
 			fmt.Println("")
 			stopFlag = true
 		} else {

From 9452a28695505cd0ae646269d3c722303fe3da21 Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Sun, 2 Jun 2024 09:08:06 +0800
Subject: [PATCH 28/31] style: resolve indent problem

---
 cmd/kk/pkg/pipelines/create_cluster.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/kk/pkg/pipelines/create_cluster.go b/cmd/kk/pkg/pipelines/create_cluster.go
index ea22eeebd..a083d290b 100644
--- a/cmd/kk/pkg/pipelines/create_cluster.go
+++ b/cmd/kk/pkg/pipelines/create_cluster.go
@@ -93,7 +93,7 @@ func NewCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&kubernetes.SaveKubeConfigModule{},
 		&plugins.DeployPluginsModule{},
 		&customscripts.CustomScriptsModule{Phase: "PostClusterInstall", Scripts: runtime.Cluster.System.PostClusterInstall},
-    &addons.AddonsModule{Skip: runtime.Arg.SkipInstallAddons},
+		&addons.AddonsModule{Skip: runtime.Arg.SkipInstallAddons},
 		&storage.DeployLocalVolumeModule{Skip: skipLocalStorage},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
 		&kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
@@ -168,7 +168,7 @@ func NewK3sCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()},
 		&k3s.SaveKubeConfigModule{},
 		&customscripts.CustomScriptsModule{Phase: "PostClusterInstall", Scripts: runtime.Cluster.System.PostClusterInstall},
-    &addons.AddonsModule{Skip: runtime.Arg.SkipInstallAddons},
+		&addons.AddonsModule{Skip: runtime.Arg.SkipInstallAddons},
 		&storage.DeployLocalVolumeModule{Skip: skipLocalStorage},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
 		&kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
@@ -243,7 +243,7 @@ func NewK8eCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()},
 		&k8e.SaveKubeConfigModule{},
 		&customscripts.CustomScriptsModule{Phase: "PostClusterInstall", Scripts: runtime.Cluster.System.PostClusterInstall},
-    &addons.AddonsModule{Skip: runtime.Arg.SkipInstallAddons},
+		&addons.AddonsModule{Skip: runtime.Arg.SkipInstallAddons},
 		&storage.DeployLocalVolumeModule{Skip: skipLocalStorage},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},
 		&kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled},

From 974ee5ef8a249ef6585dd35c2281eee900d57fd4 Mon Sep 17 00:00:00 2001
From: ks-ci-bot <47586280+ks-ci-bot@users.noreply.github.com>
Date: Thu, 13 Jun 2024 01:13:04 +0000
Subject: [PATCH 29/31] Add new kubernetes version

---
 docs/kubernetes-versions.md |  4 ++++
 version/components.json     | 36 ++++++++++++++++++++++++++++++------
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/docs/kubernetes-versions.md b/docs/kubernetes-versions.md
index 32736227f..fe36a8ac9 100644
--- a/docs/kubernetes-versions.md
+++ b/docs/kubernetes-versions.md
@@ -125,6 +125,7 @@
 | v1.27.12 | :white_check_mark: |
 | v1.27.13 | :white_check_mark: |
 | v1.27.14 | :white_check_mark: |
+| v1.27.15 | :white_check_mark: |
 | v1.28.0 | :white_check_mark: |
 | v1.28.1 | :white_check_mark: |
 | v1.28.2 | :white_check_mark: |
@@ -136,11 +137,14 @@
 | v1.28.8 | :white_check_mark: |
 | v1.28.9 | :white_check_mark: |
 | v1.28.10 | :white_check_mark: |
+| v1.28.11 | :white_check_mark: |
 | v1.29.0 | :white_check_mark: |
 | v1.29.1 | :white_check_mark: |
 | v1.29.2 | :white_check_mark: |
 | v1.29.3 | :white_check_mark: |
 | v1.29.4 | :white_check_mark: |
 | v1.29.5 | :white_check_mark: |
+| v1.29.6 | :white_check_mark: |
 | v1.30.0 | :white_check_mark: |
 | v1.30.1 | :white_check_mark: |
+| v1.30.2 | :white_check_mark: |
diff --git a/version/components.json b/version/components.json
index 47e3bf10b..01ce11386 100644
--- a/version/components.json
+++ b/version/components.json
@@ -125,6 +125,7 @@
             "v1.27.12": "06ee36cc80cfdfc01c937d750783d3ca6169a3da76382c7af3dd172d9f6bfa4e",
             "v1.27.13": "b88c30b7067f095b7fa02c5560cc50d6e69a5a9fecc606ef477dc7efc86453b9",
             "v1.27.14": "1ce264643e521494e111b1c9ee59694a54d1f2464bbac3a7a531324ffeae0182",
+            "v1.27.15": "e6f8313e213a8a75deda81edf445d0d785493f6a0a0caadf31e88d173b607943",
             "v1.28.0": "12ea68bfef0377ccedc1a7c98a05ea76907decbcf1e1ec858a60a7b9b73211bb",
             "v1.28.1": "6134dbc92dcb83c3bae1a8030f7bb391419b5d13ea94badd3a79b7ece75b2736",
             "v1.28.2": "6a4808230661c69431143db2e200ea2d021c7f1b1085e6353583075471310d00",
@@ -136,14 +137,17 @@
             "v1.28.8": "c11946cbfd962e1197062534514226cfd70230349e6343ff3ecebfca5476ee64",
             "v1.28.9": "a4d8acf0a74cb1d07d96a1a34148f54c6420874221af16d8ec902d9bffc7ef89",
             "v1.28.10": "1a344d34755c5f005120308f09a730e7564c8f857de6606b6bc5f18a69606e5a",
+            "v1.28.11": "1f2c7c69736698aa13a59c6705ac26b7b6752d9651330605369357c1ac99c7c6",
             "v1.29.0": "629d4630657caace9c819fd3797f4a70c397fbd41a2a7e464a0507dad675d52c",
             "v1.29.1": "d4d81d9020b550c896376fb9e0586a9f15a332175890d061619b52b3e9bc6cbd",
             "v1.29.2": "2d4e4fa8685bcbfb661cb41050cd4756f50a7aa147f68492d51a99f9cdfd69ac",
             "v1.29.3": "6abaa1208bf40b6d1f49e518bd68c8ae4a1be0c5b7d3e45d87979999ab070d8b",
             "v1.29.4": "ea20ab064f716ab7f69a36d72df340257b31c9721ea86e1cf9d70b35999ddeea",
             "v1.29.5": "e424dcdbe661314b6ca1fcc94726eb554bc3f4392b060b9626f9df8d7d44d42c",
+            "v1.29.6": "8f1e04079e614dd549e36be8114ee7022517d646ea715b5778e7c6ab353eb354",
             "v1.30.0": "29f4232c50e6524abba3443ff3b9948d386964d79eb8dfefb409e1f8a8434c14",
-            "v1.30.1": "651faa3bbbfb368ed00460e4d11732614310b690b767c51810a7b638cc0961a2"
+            "v1.30.1": "651faa3bbbfb368ed00460e4d11732614310b690b767c51810a7b638cc0961a2",
+            "v1.30.2": "672b0cae2accce5eac10a1fe4ea6b166e5b518c79ccf71a2fbe7b53c2ca74062"
         },
         "arm64": {
             "v1.19.0": "db1c432646e6e6484989b6f7191f3610996ac593409f12574290bfc008ea11f5",
@@ -270,6 +274,7 @@
             "v1.27.12": "e74d47c14b5a251cff961dcce92cd632abcfd0fba4a07e78f0a5a5b2796e4b84",
             "v1.27.13": "f334ba0612fada50e98a7ea56b686b35c22f0e3243ec2210f2a6a87e841a139f",
             "v1.27.14": "cb840eb83404047cdafec0b15054023c90a47491b56d2dceba1050040f37cc7e",
+            "v1.27.15": "577630b65c54ba9b629c9c42e35ba0939ce03dcb8f38fb79583ec3c9db9141a7",
             "v1.28.0": "b9b473d2d9136559b19eb465006af77df45c09862cd7ce6673a33aae517ff5ab",
             "v1.28.1": "7d2f68917470a5d66bd2a7d62897f59cb4afaeffb2f26c028afa119acd8c3fc8",
             "v1.28.2": "010789a94cf512d918ec4a3ef8ec734dea0061d89a8293059ef9101ca1bf6bff",
@@ -281,14 +286,17 @@
             "v1.28.8": "e0f47adc69ef84e2f6c42cc341b8a790904a929ad10ed1c23c2e822ec804e247",
             "v1.28.9": "cd6aefad8144a9771fd470529ff14be2675df7b561f7c56dee3fed4f81332dc4",
             "v1.28.10": "f6809d72ed1bf6fde460e48e5c714c3bc92f680e328defa9bd592a796347b644",
+            "v1.28.11": "15a021fdecf08989d6b64af873d89c61750d3a0564bee58c248eafe5cc4df433",
             "v1.29.0": "bbddee2d46d2e1643ae3623698b45b13aa2e858616d61c642f2f49e5bb14c980",
             "v1.29.1": "3bff8c50c104c45e416cce9991706c6ac46365f0defbcd54f8cf4ace0fa68dcf",
             "v1.29.2": "e05720feb9d2d67eff25b0156a5c22e2de37be2ffab4e1f4d31e8c526fafd0e1",
             "v1.29.3": "ce2e4c230f954e59ae77e34c4ff2ae08cad3970505ae1e21b6337e6d83b21682",
             "v1.29.4": "438287a91e08cbefecab79be8ac893a935c3dbf6e87bea895fb99f2bc38cf06e",
             "v1.29.5": "d4db8c514f2764edc039462c218dbcd316577f76f21b209b76e9a4b1f08e3100",
+            "v1.29.6": "3ba6879ef491cdd8433647020d345d86c0ea8e77f726375bc4b5495888bbf778",
             "v1.30.0": "c36afd28921303e6db8e58274de16c60a80a1e75030fc3c4e9c4ed6249b6b696",
-            "v1.30.1": "bda423cb4b9d056f99a2ef116bdf227fadbc1c3309fa3d76da571427a7f41478"
+            "v1.30.1": "bda423cb4b9d056f99a2ef116bdf227fadbc1c3309fa3d76da571427a7f41478",
+            "v1.30.2": "7268762b7afd44bf07619985dd52c376b63e47d73b8f9a3b08cc49624a8fbd55"
         }
     },
     "kubelet": {
@@ -417,6 +425,7 @@
             "v1.27.12": "aae861a21913c274228ccdad1609b370e5198c9f4b39b8924b20a7ffe7f148e0",
             "v1.27.13": "ed68df2a77f3057ab47f57eacb6e9310e91731e4f43c58a3c3b5c857d78d0080",
             "v1.27.14": "f28defa43f80f82ce909940c1b57b71cba1fcf0de6fc4723e798ef5c72376c28",
+            "v1.27.15": "c0ab414cdd116e9644c8c154ae4a6565c75c2605e2c7b6a5a04b56037ef2ba88",
             "v1.28.0": "bfb6b977100963f2879a33e5fbaa59a5276ba829a957a6819c936e9c1465f981",
             "v1.28.1": "2bc22332f44f8fcd3fce57879fd873f977949ebd261571fbae31fbb2713a5dd3",
             "v1.28.2": "17edb866636f14eceaad58c56eab12af7ab3be3c78400aff9680635d927f1185",
@@ -428,14 +437,17 @@
             "v1.28.8": "049b412a5861255cd3922f612acb79ab51135e166c5d80acf12fba9179eebf0c",
             "v1.28.9": "f3af46cff11c675a80d91ebb38ebc4e85a9f813ce93e56ee131e7fea1491b786",
             "v1.28.10": "a361e744aaeef4539f0636ecd1827c85207a5f2b0c2b0a98dbbce1498061f509",
+            "v1.28.11": "230f0634ea42a54a6c96771f12eecd6cadfe0b76ab41c3bc39aa7cbbe4dfb12e",
             "v1.29.0": "e1c38137db8d8777eed8813646b59bf4d22d19b9011ab11dc28e2e34f6b80a05",
             "v1.29.1": "1b1975c58d38be1a99a8bcba4564ac489afd223b0abe9f2ab08bbde89d2412a3",
             "v1.29.2": "f71a85039b71fe08f1c063a93d61a1c952dc8f9a8c6be9b13fbdac8f0d9ff960",
             "v1.29.3": "d8b55a2f8a87c8cd2cbf867d76d1d7f98b7198a740db19bad6ed7b8b813de771",
             "v1.29.4": "58571f0ed62543a9bbac541e52c15d8385083113a463e23aec1341d0b5043939",
             "v1.29.5": "261dc3f3c384d138835fe91a02071c642af94abb0cca56ebc04719240440944c",
+            "v1.29.6": "a946789d4fef64e6f5905dbd7dca01d4c3abd302d0da7958fdaa924fe2729c0b",
             "v1.30.0": "32a32ec3d7e7f8b2648c9dd503ce9ef63b4af1d1677f5b5aed7846fb02d66f18",
-            "v1.30.1": "87bd6e5de9c0769c605da5fedb77a35c8b764e3bda1632447883c935dcf219d3"
+            "v1.30.1": "87bd6e5de9c0769c605da5fedb77a35c8b764e3bda1632447883c935dcf219d3",
+            "v1.30.2": "6923abe67ef069afca61c71c585023840426e802b198298055af3a82e11a4e52"
         },
         "arm64": {
             "v1.19.0": "d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf",
@@ -562,6 +574,7 @@
             "v1.27.12": "0d7d2d25c8b909d6cec7c1c2a5bfe51428ec33eaa5e8b209c718b77983e9dcba",
             "v1.27.13": "d7bfb14d0b0fc2c41074baf02617cf98589fd029fb3539ea017825e36371f19c",
             "v1.27.14": "31c97a723021ccc90a47a15ad1de1ffdf58ba109aea922eb359ad2fcb8e8ce4b",
+            "v1.27.15": "ac8abb249dd78a22d92e28306ce9823f7e3ce23cd4ba8dbb0f3bd34f48b6c12b",
             "v1.28.0": "05dd12e35783cab4960e885ec0e7d0e461989b94297e7bea9018ccbd15c4dce9",
             "v1.28.1": "9b7fa64b2785da4a38768377961e227f8da629c56a5df43ca1b665dd07b56f3c",
             "v1.28.2": "32269e9ec38c561d028b65c3048ea6a100e1292cbe9e505565222455c8096577",
@@ -573,14 +586,17 @@
             "v1.28.8": "90d61f40b7bb061b0fc6d08b8b9ddae51f90863c899b098e19eaa89dc855f2c0",
             "v1.28.9": "312471ad255acfcdeea2c5849b171467af4518e96d69d727a3197ff334e9299d",
             "v1.28.10": "feae161e374ee0155b5263cda339f30f16b525631535a003be7aa437661e1580",
+            "v1.28.11": "0e01c1393f8746965994431b70a20c32b8547dfb1dcf2770cb692990fc65ba5f",
             "v1.29.0": "0e0e4544c2a0a3475529154b7534d0d58683466efa04a2bb2e763b476db0bb16",
             "v1.29.1": "e46417ab1ceae995f0e00d4177959a36ed34b807829422bc9dda70b263fe5c5d",
             "v1.29.2": "9b4aa572d4cd51a41b1067161d961423d0d12b120fb636ea887a12a975d4b19a",
             "v1.29.3": "891dce19ed0eae34050c2eca0454204892e97bfe1a926f988cd044a987a9c7c9",
             "v1.29.4": "dc4bb6ea6cd35b024d63cc20d1c1800a9c695bd6f70411c57358d7c407513b00",
             "v1.29.5": "0d4328a3c67e4f0dbf270fa49343f3eab9316adde1a1bd2a857fa56876a9aff1",
+            "v1.29.6": "0f0fa9429d0bcf04f271dcf4f666582dd4a4b15d6f116a45f17b5fcda90c2d2c",
             "v1.30.0": "fa887647422d34f3c7cc5b30fefcf97084d2c3277eff237c5808685ba8e4b15a",
-            "v1.30.1": "c45049b829af876588ec1a30def3884ce77c2c175cd77485d49c78d2064a38fb"
+            "v1.30.1": "c45049b829af876588ec1a30def3884ce77c2c175cd77485d49c78d2064a38fb",
+            "v1.30.2": "72ceb082311b42032827a936f80cd2437b8eee03053d05dbe36ba48585febfb8"
         }
     },
     "kubectl": {
@@ -709,6 +725,7 @@
             "v1.27.12": "d639eda39be2dce42fbec21e038942ab5734541715e3ea5fb29c9ad76686bd7f",
             "v1.27.13": "e991f163197cbd85bbff22f656a74d48b69db5addfa43cc04cca0cf5328f57f1",
             "v1.27.14": "1d2431c68bb6dfa9de3cd40fd66d97a9ac73593c489f9467249eea43e9c16a1e",
+            "v1.27.15": "c12cf17ec30fb162f8f5fb168e67d4e5b7d6826e08c7648a5f1c6b4e9ba23f39",
             "v1.28.0": "4717660fd1466ec72d59000bb1d9f5cdc91fac31d491043ca62b34398e0799ce",
             "v1.28.1": "e7a7d6f9d06fab38b4128785aa80f65c54f6675a0d2abef655259ddd852274e1",
             "v1.28.2": "c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5",
@@ -720,14 +737,17 @@
             "v1.28.8": "e02aad5c0bac52c970700b814645b62c4f18b634144398ac344875dbaf1072f8",
             "v1.28.9": "b4693d0b22f509250694b10c7727c42b427d570af04f2065fe23a55d6c0051f1",
             "v1.28.10": "389c17a9700a4b01ebb055e39b8bc0886330497440dde004b5ed90f2a3a028db",
+            "v1.28.11": "1dba63e1a5c9520fc516c6e817924d927b9b83b8e08254c8fe2a2edb65da7a9c",
             "v1.29.0": "0e03ab096163f61ab610b33f37f55709d3af8e16e4dcc1eb682882ef80f96fd5",
             "v1.29.1": "69ab3a931e826bf7ac14d38ba7ca637d66a6fcb1ca0e3333a2cafdf15482af9f",
             "v1.29.2": "7816d067740f47f949be826ac76943167b7b3a38c4f0c18b902fffa8779a5afa",
             "v1.29.3": "89c0435cec75278f84b62b848b8c0d3e15897d6947b6c59a49ddccd93d7312bf",
             "v1.29.4": "10e343861c3cb0010161e703307ba907add2aeeeaffc6444779ad915f9889c88",
             "v1.29.5": "603c8681fc0d8609c851f9cc58bcf55eeb97e2934896e858d0232aa8d1138366",
+            "v1.29.6": "339553c919874ebe3b719e9e1fcd68b55bc8875f9b5a005cf4c028738d54d309",
             "v1.30.0": "7c3807c0f5c1b30110a2ff1e55da1d112a6d0096201f1beb81b269f582b5d1c5",
-            "v1.30.1": "5b86f0b06e1a5ba6f8f00e2b01e8ed39407729c4990aeda961f83a586f975e8a"
+            "v1.30.1": "5b86f0b06e1a5ba6f8f00e2b01e8ed39407729c4990aeda961f83a586f975e8a",
+            "v1.30.2": "c6e9c45ce3f82c90663e3c30db3b27c167e8b19d83ed4048b61c1013f6a7c66e"
         },
         "arm64": {
             "v1.19.0": "d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f",
@@ -854,6 +874,7 @@
             "v1.27.12": "bfc6cb71041ebc0f048402988eccc107cfff2b866c864231c9ada05ab328e5bf",
             "v1.27.13": "4838ad8f3902c928f6139d69eba962b1cc2471511c828885650d728d970594d1",
             "v1.27.14": "29f3a1f520d929df38873c68dec73519c1e5e521140e01cf9d7701f7b5ffe4f3",
+            "v1.27.15": "6bd6bd3092c0a8e536e8e5a56b22aaab87ab1e54a4556ce4d1bff2493180fb8c",
             "v1.28.0": "f5484bd9cac66b183c653abed30226b561f537d15346c605cc81d98095f1717c",
             "v1.28.1": "46954a604b784a8b0dc16754cfc3fa26aabca9fd4ffd109cd028bfba99d492f6",
             "v1.28.2": "ea6d89b677a8d9df331a82139bb90d9968131530b94eab26cee561531eff4c53",
@@ -865,14 +886,17 @@
             "v1.28.8": "93d60dd36093b4c719f1f1bafcf59437c17cb2209341c7c94771e7dd9acdab33",
             "v1.28.9": "e0341d3973213f8099e7fcbbf6d1d506967bc2b7a4faac3fb3b4340f226e9b2f",
             "v1.28.10": "e659d23d442c2706debe5b96742326c0a1e1d7b5c695a9fe7dfe8ea7402caee8",
+            "v1.28.11": "7984a98d52365d190b6f56caa962339a7228b6f432e58ba5f1b1e60dbedac275",
             "v1.29.0": "8f7a4bd6bae900a4ddab12bd1399aa652c0d59ea508f39b910e111d248893ff7",
             "v1.29.1": "96d6dc7b2bdcd344ce58d17631c452225de5bbf59b83fd3c89c33c6298fb5d8b",
             "v1.29.2": "3507ecb4224cf05ae2151a98d4932253624e7762159936d5347b19fe037655ca",
             "v1.29.3": "191a96b27e3c6ae28b330da4c9bfefc9592762670727df4fcf124c9f1d5a466a",
             "v1.29.4": "61537408eedcad064d7334384aed508a8aa1ea786311b87b505456a2e0535d36",
             "v1.29.5": "9ee9168def12ac6a6c0c6430e0f73175e756ed262db6040f8aa2121ad2c1f62e",
+            "v1.29.6": "21816488cf3af4cf2b956ee58f7afc5b4964c29488f63756f5ddcf09b0df5be9",
             "v1.30.0": "669af0cf520757298ea60a8b6eb6b719ba443a9c7d35f36d3fb2fd7513e8c7d2",
-            "v1.30.1": "d90446719b815e3abfe7b2c46ddf8b3fda17599f03ab370d6e47b1580c0e869e"
+            "v1.30.1": "d90446719b815e3abfe7b2c46ddf8b3fda17599f03ab370d6e47b1580c0e869e",
+            "v1.30.2": "56becf07105fbacd2b70f87f3f696cfbed226cb48d6d89ed7f65ba4acae3f2f8"
         }
     },
     "etcd": {

From c3c241daebdfd8107523c1816d3344e0df4ed3a1 Mon Sep 17 00:00:00 2001
From: William Wang <williamw0825@gmail.com>
Date: Fri, 8 Mar 2024 12:49:25 +0800
Subject: [PATCH 30/31] feat: add nodeEtcHosts property

---
 cmd/kk/apis/kubekey/v1alpha2/dns_types.go        |  1 +
 cmd/kk/pkg/bootstrap/os/templates/init_script.go | 12 ++++++++++++
 2 files changed, 13 insertions(+)

diff --git a/cmd/kk/apis/kubekey/v1alpha2/dns_types.go b/cmd/kk/apis/kubekey/v1alpha2/dns_types.go
index aeaf0081d..dfed3f39f 100644
--- a/cmd/kk/apis/kubekey/v1alpha2/dns_types.go
+++ b/cmd/kk/apis/kubekey/v1alpha2/dns_types.go
@@ -18,6 +18,7 @@ package v1alpha2
 
 type DNS struct {
 	DNSEtcHosts  string       `yaml:"dnsEtcHosts" json:"dnsEtcHosts"`
+	NodeEtcHosts string       `yaml:"nodeEtcHosts" json:"nodeEtcHosts,omitempty"`
 	CoreDNS      CoreDNS      `yaml:"coredns" json:"coredns"`
 	NodeLocalDNS NodeLocalDNS `yaml:"nodelocaldns" json:"nodelocaldns"`
 }
diff --git a/cmd/kk/pkg/bootstrap/os/templates/init_script.go b/cmd/kk/pkg/bootstrap/os/templates/init_script.go
index 327fad661..d06dd6eb1 100644
--- a/cmd/kk/pkg/bootstrap/os/templates/init_script.go
+++ b/cmd/kk/pkg/bootstrap/os/templates/init_script.go
@@ -18,6 +18,7 @@ package templates
 
 import (
 	"fmt"
+	"strings"
 	"text/template"
 
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry"
@@ -272,6 +273,17 @@ func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) [
 
 	}
 
+	nodeEtcHosts := kubeConf.Cluster.DNS.NodeEtcHosts
+	if len(nodeEtcHosts) > 0 {
+		lines := strings.Split(strings.TrimSpace(nodeEtcHosts), "\n")
+		for i := range lines {
+			line := strings.TrimSpace(lines[i])
+			if line != "" {
+				hostsList = append(hostsList, line)
+			}
+		}
+	}
+
 	hostsList = append(hostsList, lbHost)
 	return hostsList
 }

From 07e2c6a69d668874511da11edceb9f3146c7b190 Mon Sep 17 00:00:00 2001
From: pixiake <guofeng@yunify.com>
Date: Tue, 9 Jul 2024 21:54:52 +0800
Subject: [PATCH 31/31] refine tasks logic

Signed-off-by: pixiake <guofeng@yunify.com>
---
 cmd/kk/pkg/bootstrap/os/templates/init_script.go | 8 ++++----
 cmd/kk/pkg/bootstrap/precheck/tasks.go           | 6 ++++--
 cmd/kk/pkg/kubernetes/kubernetes_status.go       | 5 +++++
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/cmd/kk/pkg/bootstrap/os/templates/init_script.go b/cmd/kk/pkg/bootstrap/os/templates/init_script.go
index d06dd6eb1..d9042eff2 100644
--- a/cmd/kk/pkg/bootstrap/os/templates/init_script.go
+++ b/cmd/kk/pkg/bootstrap/os/templates/init_script.go
@@ -82,8 +82,8 @@ echo 'net.ipv4.tcp_keepalive_intvl = 30' >> /etc/sysctl.conf
 echo 'net.ipv4.tcp_keepalive_probes = 10' >> /etc/sysctl.conf
 echo 'net.ipv4.udp_rmem_min = 131072' >> /etc/sysctl.conf
 echo 'net.ipv4.udp_wmem_min = 131072' >> /etc/sysctl.conf
-echo 'net.ipv4.conf.all.rp_filter = 1' >> /etc/sysctl.conf
-echo 'net.ipv4.conf.default.rp_filter = 1' >> /etc/sysctl.conf
+echo 'net.ipv4.conf.all.rp_filter = 0' >> /etc/sysctl.conf
+echo 'net.ipv4.conf.default.rp_filter = 0' >> /etc/sysctl.conf
 echo 'net.ipv4.conf.all.arp_accept = 1' >> /etc/sysctl.conf
 echo 'net.ipv4.conf.default.arp_accept = 1' >> /etc/sysctl.conf
 echo 'net.ipv4.conf.all.arp_ignore = 1' >> /etc/sysctl.conf
@@ -108,8 +108,8 @@ echo 'net.ipv6.conf.all.forwarding=1' >> /etc/sysctl.conf
 #See https://help.aliyun.com/document_detail/118806.html#uicontrol-e50-ddj-w0y
 sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1|2)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?net.ipv4.tcp_tw_reuse ?= ?(0|1)@net.ipv4.tcp_tw_reuse = 0@g" /etc/sysctl.conf
-sed -r -i "s@#{0,}?net.ipv4.conf.all.rp_filter ?= ?(0|1|2)@net.ipv4.conf.all.rp_filter = 1@g" /etc/sysctl.conf
-sed -r -i "s@#{0,}?net.ipv4.conf.default.rp_filter ?= ?(0|1|2)@net.ipv4.conf.default.rp_filter = 1@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.conf.all.rp_filter ?= ?(0|1|2)@net.ipv4.conf.all.rp_filter = 0@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.conf.default.rp_filter ?= ?(0|1|2)@net.ipv4.conf.default.rp_filter = 0@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?net.ipv4.ip_forward ?= ?(0|1)@net.ipv4.ip_forward = 1@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-arptables ?= ?(0|1)@net.bridge.bridge-nf-call-arptables = 1@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-ip6tables ?= ?(0|1)@net.bridge.bridge-nf-call-ip6tables = 1@g" /etc/sysctl.conf
diff --git a/cmd/kk/pkg/bootstrap/precheck/tasks.go b/cmd/kk/pkg/bootstrap/precheck/tasks.go
index 5517228a9..868517eb2 100644
--- a/cmd/kk/pkg/bootstrap/precheck/tasks.go
+++ b/cmd/kk/pkg/bootstrap/precheck/tasks.go
@@ -335,8 +335,10 @@ func (g *GetKubernetesNodesStatus) Execute(runtime connector.Runtime) error {
 
 	featureGates, err := runtime.GetRunner().SudoCmd("cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep feature-gates", false)
 	if err != nil {
-		return err
+		g.PipelineCache.Set(common.ClusterFeatureGates, "")
+	} else {
+		g.PipelineCache.Set(common.ClusterFeatureGates, featureGates)
 	}
-	g.PipelineCache.Set(common.ClusterFeatureGates, featureGates)
+
 	return nil
 }
diff --git a/cmd/kk/pkg/kubernetes/kubernetes_status.go b/cmd/kk/pkg/kubernetes/kubernetes_status.go
index 582d5f596..3853f5bf4 100644
--- a/cmd/kk/pkg/kubernetes/kubernetes_status.go
+++ b/cmd/kk/pkg/kubernetes/kubernetes_status.go
@@ -55,6 +55,11 @@ func (k *KubernetesStatus) SearchVersion(runtime connector.Runtime) error {
 }
 
 func (k *KubernetesStatus) SearchJoinInfo(runtime connector.Runtime) error {
+	// check if the cluster status contain initial data
+	if k.BootstrapToken != "" && k.CertificateKey != "" {
+		return nil
+	}
+
 	checkKubeadmConfig, err := runtime.GetRunner().SudoCmd("cat /etc/kubernetes/kubeadm-config.yaml", false)
 	if err != nil {
 		return err