katenary/internal/generator/generator.go

package generator

import (
	"bytes"
	"fmt"
	"regexp"
	"strings"

	"katenary.io/internal/generator/labels"
	"katenary.io/internal/generator/labels/labelstructs"
	"katenary.io/internal/logger"
	"katenary.io/internal/utils"

	"github.com/compose-spec/compose-go/v2/types"
	corev1 "k8s.io/api/core/v1"
	"sigs.k8s.io/yaml"
)

// Generate a chart from a compose project.
// This does not write files to disk, it only creates the HelmChart object.
//
// The Generate function will create the HelmChart object this way:
//
//   - Detect the service port name or leave the port number if not found.
//   - Create a deployment for each service that are not ingore.
//   - Create a service and ingresses for each service that has ports and/or declared ingresses.
//   - Create a PVC or Configmap volumes for each volume.
//   - Create init containers for each service which has dependencies to other services.
//   - Create a chart dependencies.
//   - Create a configmap and secrets from the environment variables.
//   - Merge the same-pod services.
func Generate(project *types.Project) (*HelmChart, error) {
	var (
		appName     = project.Name
		deployments = make(map[string]*Deployment, len(project.Services))
		services    = make(map[string]*Service)
		podToMerge  = make(map[string]*types.ServiceConfig)
	)
	chart := NewChart(appName)

	// Add the compose files hash to the chart annotations
	hash, err := utils.HashComposefiles(project.ComposeFiles)
	if err != nil {
		return nil, err
	}
	Annotations[labels.LabelName("compose-hash")] = hash
	chart.composeHash = &hash

	// drop all services with the "ignore" label
	dropIngoredServices(project)

	fixContainerNames(project)

	// rename all services name to remove dashes
	if err := fixResourceNames(project); err != nil {
		return nil, err
	}

	// find the "main-app" label, and set chart.AppVersion to the tag if exists
	mainCount := 0
	for _, service := range project.Services {
		if serviceIsMain(service) {
			mainCount++
			if mainCount > 1 {
				return nil, fmt.Errorf("found more than one main app")
			}
			chart.setChartVersion(service)
		}
	}
	if mainCount == 0 {
		chart.AppVersion = "0.1.0"
	}

	// first pass, create all deployments whatewer they are.
	for _, service := range project.Services {
		err := chart.generateDeployment(service, deployments, services, podToMerge, appName)
		if err != nil {
			return nil, err
		}
	}

	// now we have all deployments, we can create PVC if needed (it's separated from
	// the above loop because we need all deployments to not duplicate PVC for "same-pod" services)
	// bind static volumes
	for _, service := range project.Services {
		addStaticVolumes(deployments, service)
	}
	for _, service := range project.Services {
		err := buildVolumes(service, chart, deployments)
		if err != nil {
			return nil, err
		}
	}

	// if we have built exchange volumes, we need to moint them in each deployment
	for _, d := range deployments {
		d.MountExchangeVolumes()
	}

	// drop all "same-pod" deployments because the containers and volumes are already
	// in the target deployment
	drops := []string{}
	for _, service := range podToMerge {
		if samepod, ok := service.Labels[labels.LabelSamePod]; ok && samepod != "" {
			// move this deployment volumes to the target deployment
			if target, ok := deployments[samepod]; ok {
				target.AddContainer(*service)
				target.BindFrom(*service, deployments[service.Name])
				target.SetEnvFrom(*service, appName, true)
				// copy all init containers
				initContainers := deployments[service.Name].Spec.Template.Spec.InitContainers
				target.Spec.Template.Spec.InitContainers = append(target.Spec.Template.Spec.InitContainers, initContainers...)
				drops = append(drops, service.Name)
			} else {
				err := fmt.Errorf("service %s is declared as %s, but %s is not defined", service.Name, labels.LabelSamePod, samepod)
				logger.Failure(err.Error())
				return nil, err
			}
		}
	}

	// create init containers for all DependsOn
	for _, s := range project.Services {
		for _, d := range s.GetDependencies() {
			if dep, ok := deployments[d]; ok {
				err := deployments[s.Name].DependsOn(dep, d)
				if err != nil {
					logger.Info(fmt.Sprintf("error creating init container for service %[1]s: %[2]s", s.Name, err))
				}
			} else {
				err := fmt.Errorf("service %[1]s depends on %[2]s, but %[2]s is not defined", s.Name, d)
				logger.Failure(err.Error())
				return nil, err
			}
		}
	}

	// warn users if dependent service has no ports
	for _, s := range project.Services {
		for _, d := range s.GetDependencies() {
			if dep, ok := deployments[d]; ok {
				if len(dep.service.Ports) == 0 {
					logger.Warnf("Service %s is used in depends_on but has no ports declared. No Kubernetes Service will be created for it. Add katenary.v3/ports label if you need to create a Service.", d)
				}
			}
		}
	}

	// set ServiceAccountName for deployments that need it
	for _, d := range deployments {
		d.SetServiceAccountName()
	}

	for _, name := range drops {
		delete(deployments, name)
	}
	// it's now time to get "value-from", before makeing the secrets and configmaps!
	for _, s := range project.Services {
		chart.setEnvironmentValuesFrom(s, deployments)
	}

	// generate RBAC resources for services that need K8s API access (non-legacy depends_on)
	if err := chart.generateRBAC(deployments); err != nil {
		logger.Fatalf("error generating RBAC: %s", err)
	}

	// generate configmaps with environment variables
	if err := chart.generateConfigMapsAndSecrets(project); err != nil {
		logger.Fatalf("error generating configmaps and secrets: %s", err)
	}

	// if the env-from label is set, we need to add the env vars from the configmap
	// to the environment of the service
	for _, s := range project.Services {
		chart.setSharedConf(s, deployments)
	}

	// remove all "boundEnv" from the values
	for _, d := range deployments {
		if len(d.boundEnvVar) == 0 {
			continue
		}
		for _, e := range d.boundEnvVar {
			delete(chart.Values[d.service.Name].(*Value).Environment, e)
		}
	}

	// generate yaml files
	for _, d := range deployments {
		y, err := d.Yaml()
		if err != nil {
			return nil, err
		}
		chart.Templates[d.Filename()] = &ChartTemplate{
			Content:     y,
			Servicename: d.service.Name,
		}
	}

	// generate all services
	for _, s := range services {
		// add the service ports to the target service if it's a "same-pod" service
		if samePod, ok := podToMerge[s.service.Name]; ok {
			// get the target service
			target := services[samePod.Name]
			// merge the services
			s.Spec.Ports = append(s.Spec.Ports, target.Spec.Ports...)
		}
		y, _ := s.Yaml()
		chart.Templates[s.Filename()] = &ChartTemplate{
			Content:     y,
			Servicename: s.service.Name,
		}
	}

	// drop all "same-pod" services
	for _, s := range podToMerge {
		// get the target service
		target := services[s.Name]
		if target != nil {
			delete(chart.Templates, target.Filename())
		}
	}

	// compute all needed resplacements in YAML templates
	for n, v := range chart.Templates {
		v.Content = removeReplaceString(v.Content)
		v.Content = computeNIndent(v.Content)
		chart.Templates[n].Content = v.Content
	}

	// generate helper
	chart.Helper = Helper(appName)

	return chart, nil
}

// dropIngoredServices removes all services with the "ignore" label set to true (or yes).
func dropIngoredServices(project *types.Project) {
	for name, service := range project.Services {
		if isIgnored(service) {
			delete(project.Services, name)
		}
	}
}

// fixResourceNames renames all services and related resources to remove dashes.
func fixResourceNames(project *types.Project) error {
	// rename all services name to remove dashes
	for i, service := range project.Services {
		if service.Name != utils.AsResourceName(service.Name) {
			fixed := utils.AsResourceName(service.Name)
			for j, s := range project.Services {
				// for the same-pod services, we need to keep the original name
				if samepod, ok := s.Labels[labels.LabelSamePod]; ok && samepod == service.Name {
					s.Labels[labels.LabelSamePod] = fixed
					project.Services[j] = s
				}

				// also, the value-from label should be updated
				if valuefrom, ok := s.Labels[labels.LabelValuesFrom]; ok {
					vf, err := labelstructs.GetValueFrom(valuefrom)
					if err != nil {
						return err
					}
					for varname, bind := range *vf {
						bind := strings.ReplaceAll(bind, service.Name, fixed)
						(*vf)[varname] = bind
					}
					output, err := yaml.Marshal(vf)
					if err != nil {
						return err
					}
					s.Labels[labels.LabelValuesFrom] = string(output)
				}
			}
			service.Name = fixed
		}
		// rename depends_on
		for _, d := range service.GetDependencies() {
			depname := utils.AsResourceName(d)
			dep := service.DependsOn[d]
			delete(service.DependsOn, d)
			service.DependsOn[depname] = dep
		}
		project.Services[i] = service
	}
	return nil
}

// serviceIsMain returns true if the service is the main app.
func serviceIsMain(service types.ServiceConfig) bool {
	if main, ok := service.Labels[labels.LabelMainApp]; ok {
		return main == "true" || main == "yes" || main == "1"
	}
	return false
}

func addStaticVolumes(deployments map[string]*Deployment, service types.ServiceConfig) {
	// add the bound configMaps files to the deployment containers
	var d *Deployment
	var ok bool
	if d, ok = deployments[service.Name]; !ok {
		logger.Warnf("service %s not found in deployments", service.Name)
		return
	}

	container, index := utils.GetContainerByName(service.ContainerName, d.Spec.Template.Spec.Containers)
	if container == nil { // may append for the same-pod services
		return
	}
	for volumeName, config := range d.configMaps {
		var y []byte
		var err error
		if y, err = config.configMap.Yaml(); err != nil {
			logger.Fatal(err)
		}

		// add the configmap to the chart
		d.chart.Templates[config.configMap.Filename()] = &ChartTemplate{
			Content:     y,
			Servicename: d.service.Name,
		}
		// add the moint path to the container
		for _, m := range config.mountPath {
			// volumeName can be empty, in this case we generate a name
			if volumeName == "" {
				volumeName = utils.PathToName(m.subPath)
			}
			container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
				Name:      utils.PathToName(volumeName),
				MountPath: m.mountPath,
				SubPath:   m.subPath,
			})
		}

		d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, corev1.Volume{
			Name: utils.PathToName(volumeName),
			VolumeSource: corev1.VolumeSource{
				ConfigMap: &corev1.ConfigMapVolumeSource{
					LocalObjectReference: corev1.LocalObjectReference{
						Name: config.configMap.Name,
					},
				},
			},
		})
	}

	d.Spec.Template.Spec.Containers[index] = *container
}

// computeNIndentm replace all __indent__ labels with the number of spaces before the label.
func computeNIndent(b []byte) []byte {
	lines := bytes.Split(b, []byte("\n"))
	for i, line := range lines {
		if !bytes.Contains(line, []byte("__indent__")) {
			continue
		}
		startSpaces := ""
		spaces := regexp.MustCompile(`^\s+`).FindAllString(string(line), -1)
		if len(spaces) > 0 {
			startSpaces = spaces[0]
		}
		line = []byte(startSpaces + strings.TrimLeft(string(line), " "))
		line = bytes.ReplaceAll(line, []byte("__indent__"), fmt.Appendf(nil, "%d", len(startSpaces)))
		lines[i] = line
	}
	return bytes.Join(lines, []byte("\n"))
}

// removeReplaceString replace all __replace_ labels with the value of the
// capture group and remove all new lines and repeated spaces.
//
// we created:
//
//	__replace_bar: '{{ include "foo.labels" .
//	   }}'
//
// note the new line and spaces...
//
// we now want to replace it with {{ include "foo.labels" . }}, without the label name.
func removeReplaceString(b []byte) []byte {
	// replace all matches with the value of the capture group
	// and remove all new lines and repeated spaces
	b = replaceLabelRegexp.ReplaceAllFunc(b, func(b []byte) []byte {
		inc := replaceLabelRegexp.FindSubmatch(b)[1]
		inc = bytes.ReplaceAll(inc, []byte("\n"), []byte(""))
		inc = bytes.ReplaceAll(inc, []byte("\r"), []byte(""))
		inc = regexp.MustCompile(`\s+`).ReplaceAll(inc, []byte(" "))
		return inc
	})
	return b
}

// buildVolumes creates the volumes for the service.
func buildVolumes(service types.ServiceConfig, chart *HelmChart, deployments map[string]*Deployment) error {
	appName := chart.Name
	for _, v := range service.Volumes {
		// Do not add volumes if the pod is injected in a deployments
		// via "same-pod" and the volume in destination deployment exists
		if samePodVolume(service, v, deployments) {
			continue
		}
		switch v.Type {
		case "volume":
			v.Source = utils.AsResourceName(v.Source)
			pvc := NewVolumeClaim(service, v.Source, appName)

			// if the service is integrated in another deployment, we need to add the volume
			// to the target deployment
			if override, ok := service.Labels[labels.LabelSamePod]; ok {
				pvc.nameOverride = override
				pvc.Spec.StorageClassName = utils.StrPtr(`{{ .Values.` + override + `.persistence.` + v.Source + `.storageClass }}`)
				chart.Values[override].(*Value).AddPersistence(v.Source)
			}
			y, _ := pvc.Yaml()
			chart.Templates[pvc.Filename()] = &ChartTemplate{
				Content:     y,
				Servicename: service.Name,
			}
		}
	}

	return nil
}

// samePodVolume returns true if the volume is already in the target deployment.
func samePodVolume(service types.ServiceConfig, v types.ServiceVolumeConfig, deployments map[string]*Deployment) bool {
	// if the service has volumes, and it has "same-pod" label
	// - get the target deployment
	// - check if it has the same volume
	// if not, return false

	if v.Source == "" {
		return false
	}

	if len(service.Volumes) == 0 {
		return false
	}

	targetDeployment := ""
	if targetName, ok := service.Labels[labels.LabelSamePod]; !ok {
		return false
	} else {
		targetDeployment = targetName
	}

	// get the target deployment
	target := findDeployment(targetDeployment, deployments)
	if target == nil {
		return false
	}

	// check if it has the same volume
	for _, tv := range target.Spec.Template.Spec.Volumes {
		if tv.Name == v.Source {
			logger.Warnf("found same pod volume %s in deployment %s and %s", tv.Name, service.Name, targetDeployment)
			return true
		}
	}
	return false
}

// generateRBAC creates RBAC resources (ServiceAccount, Role, RoleBinding) for services that need K8s API access.
// A service needs RBAC if it has non-legacy depends_on relationships.
func (chart *HelmChart) generateRBAC(deployments map[string]*Deployment) error {
	serviceMap := make(map[string]bool)

	for _, d := range deployments {
		if !d.needsServiceAccount {
			continue
		}

		sa := NewServiceAccount(*d.service, chart.Name)
		role := NewRestrictedRole(*d.service, chart.Name)
		rb := NewRestrictedRoleBinding(*d.service, chart.Name)

		var buf bytes.Buffer

		saYaml, err := yaml.Marshal(sa.ServiceAccount)
		if err != nil {
			return fmt.Errorf("error marshaling ServiceAccount for %s: %w", d.service.Name, err)
		}
		buf.Write(saYaml)
		buf.WriteString("---\n")

		roleYaml, err := yaml.Marshal(role.Role)
		if err != nil {
			return fmt.Errorf("error marshaling Role for %s: %w", d.service.Name, err)
		}
		buf.Write(roleYaml)
		buf.WriteString("---\n")

		rbYaml, err := yaml.Marshal(rb.RoleBinding)
		if err != nil {
			return fmt.Errorf("error marshaling RoleBinding for %s: %w", d.service.Name, err)
		}
		buf.Write(rbYaml)

		filename := d.service.Name + "/depends-on.rbac.yaml"
		chart.Templates[filename] = &ChartTemplate{
			Content:     buf.Bytes(),
			Servicename: d.service.Name,
		}

		serviceMap[d.service.Name] = true
	}

	for svcName := range serviceMap {
		logger.Log(logger.IconPackage, "Creating RBAC", svcName)
	}

	return nil
}

func fixContainerNames(project *types.Project) {
	// fix container names to be unique
	for i, service := range project.Services {
		if service.ContainerName == "" {
			service.ContainerName = utils.FixedResourceName(service.Name)
		} else {
			service.ContainerName = utils.FixedResourceName(service.ContainerName)
		}
		project.Services[i] = service
	}
}