From 3c743fb135cd625b562b82c62228614a2898a93d Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Fri, 19 Apr 2024 11:13:24 +0200 Subject: [PATCH] Fixup hard problems on bound volumes Recreated the method to bind local content to configMaps with subPath. That simplify a few how we can bound files and not only directory content. --- generator/configMap.go | 22 ++++++++- generator/deployment.go | 107 ++++++++++++++++++++++------------------ generator/generator.go | 85 ++++++++++++------------------- 3 files changed, 110 insertions(+), 104 deletions(-) diff --git a/generator/configMap.go b/generator/configMap.go index ac02590..73ff639 100644 --- a/generator/configMap.go +++ b/generator/configMap.go @@ -127,10 +127,10 @@ func NewConfigMap(service types.ServiceConfig, appName string) *ConfigMap { return cm } -// NewConfigMapFromFiles creates a new ConfigMap from a compose service. This path is the path to the +// NewConfigMapFromDirectory creates a new ConfigMap from a compose service. This path is the path to the // file or directory. If the path is a directory, all files in the directory are added to the ConfigMap. // Each subdirectory are ignored. Note that the Generate() function will create the subdirectories ConfigMaps. -func NewConfigMapFromFiles(service types.ServiceConfig, appName string, path string) *ConfigMap { +func NewConfigMapFromDirectory(service types.ServiceConfig, appName string, path string) *ConfigMap { normalized := path normalized = strings.TrimLeft(normalized, ".") normalized = strings.TrimLeft(normalized, "/") @@ -178,6 +178,7 @@ func (c *ConfigMap) AppendDir(path string) { if err != nil { log.Fatalf("Path %s does not exist\n", path) } + log.Printf("Appending files from %s to configmap\n", path) // recursively read all files in the path and add them to the configmap if stat.IsDir() { files, err := os.ReadDir(path) @@ -207,6 +208,23 @@ func (c *ConfigMap) AppendDir(path string) { } } +func (c *ConfigMap) AppendFile(path string) { + // read all files in the path and add them to the configmap + stat, err := os.Stat(path) + if err != nil { + log.Fatalf("Path %s does not exist\n", path) + } + // recursively read all files in the path and add them to the configmap + if !stat.IsDir() { + // add the file to the configmap + content, err := os.ReadFile(path) + if err != nil { + log.Fatal(err) + } + c.AddData(filepath.Base(path), string(content)) + } +} + // Filename returns the filename of the configmap. If the configmap is used for files, the filename contains the path. func (c *ConfigMap) Filename() string { switch c.usage { diff --git a/generator/deployment.go b/generator/deployment.go index 07760f8..ff70224 100644 --- a/generator/deployment.go +++ b/generator/deployment.go @@ -20,14 +20,24 @@ import ( var _ Yaml = (*Deployment)(nil) +type mountPathConfig struct { + mountPath string + subPath string +} + +type ConfigMapMount struct { + configMap *ConfigMap + mountPath []mountPathConfig +} + // Deployment is a kubernetes Deployment. type Deployment struct { *appsv1.Deployment `yaml:",inline"` - chart *HelmChart `yaml:"-"` - configMaps map[string]bool `yaml:"-"` - service *types.ServiceConfig `yaml:"-"` - defaultTag string `yaml:"-"` - isMainApp bool `yaml:"-"` + chart *HelmChart `yaml:"-"` + configMaps map[string]*ConfigMapMount `yaml:"-"` + service *types.ServiceConfig `yaml:"-"` + defaultTag string `yaml:"-"` + isMainApp bool `yaml:"-"` } // NewDeployment creates a new Deployment from a compose service. The appName is the name of the application taken from the project name. @@ -74,7 +84,7 @@ func NewDeployment(service types.ServiceConfig, chart *HelmChart) *Deployment { }, }, }, - configMaps: map[string]bool{}, + configMaps: map[string]*ConfigMapMount{}, } // add containers @@ -182,6 +192,7 @@ func (d *Deployment) AddVolumes(service types.ServiceConfig, appName string) { isSamePod = v != "" } + container, index := utils.GetContainerByName(service.Name, d.Spec.Template.Spec.Containers) for _, volume := range service.Volumes { // not declared as a bind volume, skip if _, ok := tobind[volume.Source]; !isSamePod && volume.Type == "bind" && !ok { @@ -195,7 +206,6 @@ func (d *Deployment) AddVolumes(service types.ServiceConfig, appName string) { continue } - container, index := utils.GetContainerByName(service.Name, d.Spec.Template.Spec.Containers) if container == nil { utils.Warn("Container not found for volume", volume.Source) continue @@ -231,61 +241,62 @@ func (d *Deployment) AddVolumes(service types.ServiceConfig, appName string) { }) case "bind": // Add volume to container - cm := NewConfigMapFromFiles(service, appName, volume.Source) - d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: utils.PathToName(volume.Source), - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cm.Name, - }, - }, - }, - }) - // add the mount path to the container - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: utils.PathToName(volume.Source), - MountPath: volume.Target, - }) - - d.configMaps[utils.PathToName(volume.Source)] = true - // add all subdirectories to the list of directories stat, err := os.Stat(volume.Source) if err != nil { log.Fatal(err) } + if stat.IsDir() { - files, err := os.ReadDir(volume.Source) + pathnme := utils.PathToName(volume.Source) + if _, ok := d.configMaps[pathnme]; !ok { + d.configMaps[pathnme] = &ConfigMapMount{ + mountPath: []mountPathConfig{}, + } + } + + // TODO: make it recursive to add all files in the directory and subdirectories + _, err := os.ReadDir(volume.Source) if err != nil { log.Fatal(err) } - for _, file := range files { - if file.IsDir() { - cm := NewConfigMapFromFiles(service, appName, filepath.Join(volume.Source, file.Name())) - name := utils.PathToName(volume.Source) + "-" + file.Name() - d.configMaps[name] = true - d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: utils.PathToName(volume.Source) + "-" + file.Name(), - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cm.Name, - }, - }, - }, - }) - // add the mount path to the container - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: name, - MountPath: filepath.Join(volume.Target, file.Name()), - }) + cm := NewConfigMapFromDirectory(service, appName, volume.Source) + d.configMaps[pathnme] = &ConfigMapMount{ + configMap: cm, + mountPath: append(d.configMaps[pathnme].mountPath, mountPathConfig{ + mountPath: volume.Target, + }), + } + } else { + dirname := filepath.Dir(volume.Source) + pathnme := utils.PathToName(dirname) + var cm *ConfigMap + if v, ok := d.configMaps[pathnme]; !ok { + cm = NewConfigMap(*d.service, appName) + cm.usage = FileMapUsageFiles + cm.path = dirname + cm.Name = utils.TplName(service.Name, appName) + "-" + pathnme + d.configMaps[pathnme] = &ConfigMapMount{ + configMap: cm, + mountPath: []mountPathConfig{}, } + } else { + cm = v.configMap + } + + cm.AppendFile(volume.Source) + d.configMaps[pathnme] = &ConfigMapMount{ + configMap: cm, + mountPath: append(d.configMaps[pathnme].mountPath, mountPathConfig{ + mountPath: volume.Target, + subPath: filepath.Base(volume.Source), + }), } } } - d.Spec.Template.Spec.Containers[index] = *container } + + d.Spec.Template.Spec.Containers[index] = *container } func (d *Deployment) BindFrom(service types.ServiceConfig, binded *Deployment) { diff --git a/generator/generator.go b/generator/generator.go index 6e8ba95..5d18d52 100644 --- a/generator/generator.go +++ b/generator/generator.go @@ -6,8 +6,6 @@ import ( "bytes" "fmt" "log" - "os" - "path/filepath" "regexp" "strconv" "strings" @@ -385,66 +383,45 @@ func buildVolumes(service types.ServiceConfig, chart *HelmChart, deployments map Content: y, Servicename: service.Name, // TODO, use name } + } + } - case "bind": - // ensure the path is in labels - bindPath := map[string]string{} - if _, ok := service.Labels[LABEL_CM_FILES]; ok { - files := []string{} - if err := yaml.Unmarshal([]byte(service.Labels[LABEL_CM_FILES]), &files); err != nil { - return err - } - for _, f := range files { - bindPath[f] = f - } - } - if _, ok := bindPath[v.Source]; !ok { - continue - } - - cm := NewConfigMapFromFiles(service, appName, v.Source) - var err error + // add the bound configMaps files to the deployment containers + for _, d := range deployments { + container, index := utils.GetContainerByName(service.Name, d.Spec.Template.Spec.Containers) + for volumeName, config := range d.configMaps { var y []byte - if y, err = cm.Yaml(); err != nil { + var err error + if y, err = config.configMap.Yaml(); err != nil { log.Fatal(err) } - chart.Templates[cm.Filename()] = &ChartTemplate{ + // add the configmap to the chart + d.chart.Templates[config.configMap.Filename()] = &ChartTemplate{ Content: y, - Servicename: service.Name, - } - - // continue with subdirectories - stat, err := os.Stat(v.Source) - if err != nil { - return err - } - if stat.IsDir() { - files, err := filepath.Glob(filepath.Join(v.Source, "*")) - if err != nil { - return err - } - for _, f := range files { - if f == v.Source { - continue - } - if stat, err := os.Stat(f); err != nil || !stat.IsDir() { - continue - } - cm := NewConfigMapFromFiles(service, appName, f) - var err error - var y []byte - if y, err = cm.Yaml(); err != nil { - log.Fatal(err) - } - log.Printf("Adding configmap %s %s", cm.Filename(), f) - chart.Templates[cm.Filename()] = &ChartTemplate{ - Content: y, - Servicename: service.Name, - } - } + Servicename: d.service.Name, + } + // add the moint path to the container + for _, m := range config.mountPath { + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: utils.PathToName(volumeName), + MountPath: m.mountPath, + SubPath: m.subPath, + }) } + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: utils.PathToName(volumeName), + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.configMap.Name, + }, + }, + }, + }) } + + d.Spec.Template.Spec.Containers[index] = *container } return nil }