Skip to content

Commit

Permalink
Merge pull request #353 from AllenShen/hotfix/renderset_override
Browse files Browse the repository at this point in the history 
Hotfix/renderset override
  • Loading branch information
@landylee007
landylee007 authored Sep 29, 2021
2 parents c593f56 + 354aa94 commit dc4d33a
Show file tree
Hide file tree
Showing 4 changed files with 173 additions and 117 deletions.
16 changes: 16 additions & 0 deletions pkg/microservice/aslan/core/common/repository/models/template/product.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,22 @@ func (r *RenderKV) RemoveDupServices() {
r.Services = result
}

func (rc *RenderChart) DiffOverrideYaml(other *RenderChart) bool {
if rc.OverrideYaml == nil {
if other.OverrideYaml != nil {
return true
}
return false
}
if other.OverrideYaml == nil {
return true
}
if rc.OverrideYaml.YamlSource != other.OverrideYaml.YamlSource || rc.OverrideYaml.YamlContent != other.OverrideYaml.YamlContent {
return true
}
return false
}

func (rc *RenderChart) GetOverrideYaml() string {
if rc.OverrideYaml == nil {
return ""
Expand Down
2 changes: 1 addition & 1 deletion pkg/microservice/aslan/core/environment/service/environment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1119,7 +1119,7 @@ func UpdateHelmProduct(productName, envName, updateType, username, requestID str
func checkOverrideValuesChange(source *template.RenderChart, args *commonservice.RenderChartArg) bool {
tmpRenderCharts := &template.RenderChart{}
args.FillRenderChartModel(tmpRenderCharts, "")
if source.OverrideValues != tmpRenderCharts.OverrideValues || source.GetOverrideYaml() != tmpRenderCharts.GetOverrideYaml() {
if source.OverrideValues != tmpRenderCharts.OverrideValues || source.DiffOverrideYaml(tmpRenderCharts) {
return true
}
return false
Expand Down
252 changes: 141 additions & 111 deletions pkg/microservice/warpdrive/core/service/taskplugin/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package taskplugin

import (
"context"
"encoding/json"
"fmt"
"path/filepath"
"regexp"
Expand Down Expand Up @@ -47,7 +46,6 @@ import (
"github.com/koderover/zadig/pkg/tool/kube/getter"
"github.com/koderover/zadig/pkg/tool/kube/multicluster"
"github.com/koderover/zadig/pkg/tool/kube/updater"
"github.com/koderover/zadig/pkg/tool/log"
s3tool "github.com/koderover/zadig/pkg/tool/s3"
"github.com/koderover/zadig/pkg/util"
"github.com/koderover/zadig/pkg/util/fs"
Expand Down Expand Up @@ -321,14 +319,16 @@ func (p *DeployTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, _ *
}
} else if p.Task.ServiceType == setting.HelmDeployType {
var (
productInfo *types.Product
chartInfoMap = make(map[string]*types.RenderChart)
renderChart *types.RenderChart
isExist = false
replaceValuesYaml string
yamlValuesByte []byte
renderInfo *types.RenderSet
helmClient helmclient.Client
productInfo *types.Product
renderChart *types.RenderChart
replacedValuesYaml string
mergedValuesYaml string
replacedMergedValuesYaml string
servicePath string
chartPath string
replaceValuesMap map[string]interface{}
renderInfo *types.RenderSet
helmClient helmclient.Client
)

deployments, _ := getter.ListDeployments(p.Task.Namespace, nil, p.kubeClient)
Expand Down Expand Up @@ -361,6 +361,9 @@ func (p *DeployTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, _ *
}
}

p.Log.Infof("start helm deploy, productName %s serviceName %s containerName %s namespace %s", p.Task.ProductName,
p.Task.ServiceName, p.Task.ContainerName, p.Task.Namespace)

productInfo, err = p.getProductInfo(ctx, &EnvArgs{EnvName: p.Task.EnvName, ProductName: p.Task.ProductName})
if err != nil {
err = errors.WithMessagef(
Expand Down Expand Up @@ -392,116 +395,146 @@ func (p *DeployTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, _ *
}

if targetContainer == nil {
err = fmt.Errorf("failed to find container %s from service %s", p.Task.ContainerName, p.Task.ServiceName)
err = errors.Errorf("failed to find target container %s from service %s", p.Task.ContainerName, p.Task.ServiceName)
return
}

for _, chartInfo := range renderInfo.ChartInfos {
chartInfoMap[chartInfo.ServiceName] = chartInfo
if chartInfo.ServiceName == p.Task.ServiceName {
renderChart = chartInfo
break
}
}

if renderChart, isExist = chartInfoMap[p.Task.ServiceName]; isExist {
yamlValuesByte, err = yaml.YAMLToJSON([]byte(renderChart.ValuesYaml))
if err != nil {
err = errors.WithMessagef(
err,
"failed to YAMLToJSON %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}
if renderChart == nil {
err = errors.Errorf("failed to update container image in %s/%s，not find",
p.Task.Namespace, p.Task.ServiceName)
return
}

var currentValuesYamlMap map[string]interface{}
if err = json.Unmarshal(yamlValuesByte, &currentValuesYamlMap); err != nil {
err = errors.WithMessagef(
err,
"failed to Unmarshal values.yaml %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}
serviceValuesYaml := renderChart.ValuesYaml

replaceValuesYaml, err = p.replaceImage(targetContainer.ImagePath, currentValuesYamlMap, p.Task.Image)
if err != nil {
log.Error("failed to replace image: %s", p.Task.Image)
}
// prepare image replace info
getValidMatchData := getValidMatchData(targetContainer.ImagePath)
replaceValuesMap, err = assignImageData(p.Task.Image, getValidMatchData)
if err != nil {
err = errors.WithMessagef(
err,
"failed to pase image uri %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}

if replaceValuesYaml != "" {
helmClient, err = helmtool.NewClientFromRestConf(p.restConfig, p.Task.Namespace)
if err != nil {
err = errors.WithMessagef(
err,
"failed to create helm client %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}
// replace image into service's values.yaml
replacedValuesYaml, err = replaceImage(serviceValuesYaml, replaceValuesMap)
if err != nil {
err = errors.WithMessagef(
err,
"failed to replace image uri %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}
if replacedValuesYaml == "" {
err = errors.Errorf("failed to set new image uri into service's values.yaml %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}

path, err := p.downloadService(pipelineTask.ProductName, p.Task.ServiceName, pipelineTask.StorageURI)
if err != nil {
err = errors.WithMessagef(
err,
"failed to download service %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}
chartPath, err := fs.RelativeToCurrentPath(path)
if err != nil {
err = errors.WithMessagef(
err,
"failed to get relative path %s",
path,
)
return
}
// merge override values and kvs into service's yaml
mergedValuesYaml, err = helmtool.MergeOverrideValues(serviceValuesYaml, renderChart.GetOverrideYaml(), renderChart.OverrideValues)
if err != nil {
err = errors.WithMessagef(
err,
"failed to merge override values %s",
renderChart.OverrideValues,
)
return
}

mergedValuesYaml, err := helmtool.MergeOverrideValues(replaceValuesYaml, renderChart.GetOverrideYaml(), renderChart.OverrideValues)
if err != nil {
err = errors.WithMessagef(
err,
"failed to merge override values %s",
renderChart.OverrideValues,
)
return
}
// replace image into final merged values.yaml
replacedMergedValuesYaml, err = replaceImage(mergedValuesYaml, replaceValuesMap)
if err != nil {
err = errors.WithMessagef(
err,
"failed to replace image uri into helm values %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}
if replacedMergedValuesYaml == "" {
err = errors.Errorf("failed to set image uri into mreged values.yaml in %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}

chartSpec := helmclient.ChartSpec{
ReleaseName: util.GeneHelmReleaseName(p.Task.Namespace, p.Task.ServiceName),
ChartName: chartPath,
Namespace: p.Task.Namespace,
ReuseValues: true,
Version: renderChart.ChartVersion,
ValuesYaml: mergedValuesYaml,
SkipCRDs: false,
UpgradeCRDs: true,
Timeout: time.Second * DeployTimeout,
}
p.Log.Infof("final replaced merged values: \n%s", replacedMergedValuesYaml)

if _, err = helmClient.InstallOrUpgradeChart(context.TODO(), &chartSpec); err != nil {
err = errors.WithMessagef(
err,
"failed to Install helm chart %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}
helmClient, err = helmtool.NewClientFromRestConf(p.restConfig, p.Task.Namespace)
if err != nil {
err = errors.WithMessagef(
err,
"failed to create helm client %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}

//替换环境变量中的chartInfos
for _, chartInfo := range renderInfo.ChartInfos {
if chartInfo.ServiceName == p.Task.ServiceName {
chartInfo.ValuesYaml = replaceValuesYaml
break
}
}
_ = p.updateRenderSet(ctx, &types.RenderSet{
Name: renderInfo.Name,
Revision: renderInfo.Revision,
ChartInfos: renderInfo.ChartInfos,
})
servicePath, err = p.downloadService(pipelineTask.ProductName, p.Task.ServiceName, pipelineTask.StorageURI)
if err != nil {
err = errors.WithMessagef(
err,
"failed to download service %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}
chartPath, err = fs.RelativeToCurrentPath(servicePath)
if err != nil {
err = errors.WithMessagef(
err,
"failed to get relative path %s",
servicePath,
)
return
}

}
chartSpec := helmclient.ChartSpec{
ReleaseName: util.GeneHelmReleaseName(p.Task.Namespace, p.Task.ServiceName),
ChartName: chartPath,
Namespace: p.Task.Namespace,
ReuseValues: true,
Version: renderChart.ChartVersion,
ValuesYaml: replacedMergedValuesYaml,
SkipCRDs: false,
UpgradeCRDs: true,
Timeout: time.Second * DeployTimeout,
}

if _, err = helmClient.InstallOrUpgradeChart(context.TODO(), &chartSpec); err != nil {
err = errors.WithMessagef(
err,
"failed to Install helm chart %s/%s",
p.Task.Namespace, p.Task.ServiceName)
return
}
err = errors.WithMessagef(
err,
"failed to update container image in %s/%s，not find",
p.Task.Namespace, p.Task.ServiceName)

//替换环境变量中的chartInfos
for _, chartInfo := range renderInfo.ChartInfos {
if chartInfo.ServiceName == p.Task.ServiceName {
chartInfo.ValuesYaml = replacedValuesYaml
break
}
}

// TODO too dangerous to override entire renderset!
err = p.updateRenderSet(ctx, &types.RenderSet{
Name: renderInfo.Name,
Revision: renderInfo.Revision,
ChartInfos: renderInfo.ChartInfos,
})
if err != nil {
err = errors.WithMessagef(
err,
"failed to update renderset info %s/%s, renderset %s",
p.Task.Namespace, p.Task.ServiceName, renderInfo.Name)
}
}
}

Expand Down Expand Up @@ -590,16 +623,13 @@ func getValidMatchData(spec *types.ImagePathSpec) map[string]string {
}

// replace image defines in yaml by new version
func (p *DeployTaskPlugin) replaceImage(imagePathSpec *types.ImagePathSpec, valuesMap map[string]interface{}, imageUri string) (string, error) {
if imagePathSpec == nil {
return "", errors.New("image path parse info is nil")
}
getValidMatchData := getValidMatchData(imagePathSpec)
replaceValuesMap, err := assignImageData(imageUri, getValidMatchData)
func replaceImage(sourceYaml string, imageValuesMap map[string]interface{}) (string, error) {
valuesMap := make(map[string]interface{})
err := yaml.Unmarshal([]byte(sourceYaml), &valuesMap)
if err != nil {
return "", err
}
replaceMap := util.ReplaceMapValue(valuesMap, replaceValuesMap)
replaceMap := util.ReplaceMapValue(valuesMap, imageValuesMap)
replacedValuesYaml, err := util.JSONToYaml(replaceMap)
if err != nil {
return "", err
Expand Down Expand Up @@ -650,7 +680,7 @@ func assignImageData(imageUrl string, matchData map[string]string) (map[string]i
}
}

return nil, fmt.Errorf("match data illegal, expect length: 1-3, actual length: %d", len(matchData))
return nil, errors.Errorf("match data illegal, expect length: 1-3, actual length: %d", len(matchData))
}

// parse image url to map: repo=>xxx/xx/xx image=>xx tag=>xxx
Expand Down
20 changes: 15 additions & 5 deletions pkg/microservice/warpdrive/core/service/types/environment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,26 @@ type RenderKV struct {
Services []string `bson:"services" json:"services"`
}

type GitRepoConfig struct {
CodehostID int `bson:"codehost_id,omitempty"`
Owner string `bson:"owner,omitempty"`
Repo string `bson:"repo,omitempty"`
Branch string `bson:"branch,omitempty"`
}

type OverrideYaml struct {
YamlContent string `bson:"yaml_content,omitempty" json:"yaml_content,omitempty"`
YamlSource string `bson:"yaml_source,omitempty" json:"yaml_source,omitempty"`
YamlContent string `bson:"yaml_content,omitempty" json:"yaml_content,omitempty"`
GitRepoConfig *GitRepoConfig `bson:"git_repo_config,omitempty" json:"gitRepoConfig,omitempty"`
ValuesPaths []string `bson:"values_paths,omitempty" json:"values_paths,omitempty"`
}

type RenderChart struct {
ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"`
ChartVersion string `bson:"chart_version,omitempty" json:"chart_version,omitempty"`
ValuesYaml string `bson:"values_yaml,omitempty" json:"values_yaml,omitempty"`
ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"`
ChartVersion string `bson:"chart_version,omitempty" json:"chart_version,omitempty"`
ValuesYaml string `bson:"values_yaml,omitempty" json:"values_yaml,omitempty"`
OverrideYaml *OverrideYaml `bson:"override_yaml,omitempty" json:"override_yaml,omitempty"`
OverrideValues string `bson:"override_values,omitempty" json:"override_values,omitempty"`
OverrideValues string `bson:"override_values,omitempty" json:"override_values,omitempty"`
}

type ProductAuth struct {
Expand Down

0 comments on commit dc4d33a

Please sign in to comment.