feat(refacto): move everything in internal package
This allows to install katenary with `go install` and to clean up the project folder.
This commit is contained in:
433
internal/generator/generator.go
Normal file
433
internal/generator/generator.go
Normal file
@@ -0,0 +1,433 @@
|
||||
package generator
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/katenary/katenary/internal/generator/labels"
|
||||
"github.com/katenary/katenary/internal/generator/labels/labelstructs"
|
||||
"github.com/katenary/katenary/internal/utils"
|
||||
|
||||
"github.com/compose-spec/compose-go/types"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"sigs.k8s.io/yaml"
|
||||
)
|
||||
|
||||
// Generate a chart from a compose project.
|
||||
// This does not write files to disk, it only creates the HelmChart object.
|
||||
//
|
||||
// The Generate function will create the HelmChart object this way:
|
||||
//
|
||||
// - Detect the service port name or leave the port number if not found.
|
||||
// - Create a deployment for each service that are not ingnore.
|
||||
// - Create a service and ingresses for each service that has ports and/or declared ingresses.
|
||||
// - Create a PVC or Configmap volumes for each volume.
|
||||
// - Create init containers for each service which has dependencies to other services.
|
||||
// - Create a chart dependencies.
|
||||
// - Create a configmap and secrets from the environment variables.
|
||||
// - Merge the same-pod services.
|
||||
func Generate(project *types.Project) (*HelmChart, error) {
|
||||
var (
|
||||
appName = project.Name
|
||||
deployments = make(map[string]*Deployment, len(project.Services))
|
||||
services = make(map[string]*Service)
|
||||
podToMerge = make(map[string]*types.ServiceConfig)
|
||||
)
|
||||
chart := NewChart(appName)
|
||||
|
||||
// Add the compose files hash to the chart annotations
|
||||
hash, err := utils.HashComposefiles(project.ComposeFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
Annotations[labels.LabelName("compose-hash")] = hash
|
||||
chart.composeHash = &hash
|
||||
|
||||
// drop all services with the "ignore" label
|
||||
dropIngoredServices(project)
|
||||
|
||||
fixContainerNames(project)
|
||||
|
||||
// rename all services name to remove dashes
|
||||
if err := fixResourceNames(project); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// find the "main-app" label, and set chart.AppVersion to the tag if exists
|
||||
mainCount := 0
|
||||
for _, service := range project.Services {
|
||||
if serviceIsMain(service) {
|
||||
mainCount++
|
||||
if mainCount > 1 {
|
||||
return nil, fmt.Errorf("found more than one main app")
|
||||
}
|
||||
chart.setChartVersion(service)
|
||||
}
|
||||
}
|
||||
if mainCount == 0 {
|
||||
chart.AppVersion = "0.1.0"
|
||||
}
|
||||
|
||||
// first pass, create all deployments whatewer they are.
|
||||
for _, service := range project.Services {
|
||||
err := chart.generateDeployment(service, deployments, services, podToMerge, appName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// now we have all deployments, we can create PVC if needed (it's separated from
|
||||
// the above loop because we need all deployments to not duplicate PVC for "same-pod" services)
|
||||
// bind static volumes
|
||||
for _, service := range project.Services {
|
||||
addStaticVolumes(deployments, service)
|
||||
}
|
||||
for _, service := range project.Services {
|
||||
err := buildVolumes(service, chart, deployments)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// if we have built exchange volumes, we need to moint them in each deployment
|
||||
for _, d := range deployments {
|
||||
d.MountExchangeVolumes()
|
||||
}
|
||||
|
||||
// drop all "same-pod" deployments because the containers and volumes are already
|
||||
// in the target deployment
|
||||
for _, service := range podToMerge {
|
||||
if samepod, ok := service.Labels[labels.LabelSamePod]; ok && samepod != "" {
|
||||
// move this deployment volumes to the target deployment
|
||||
if target, ok := deployments[samepod]; ok {
|
||||
target.AddContainer(*service)
|
||||
target.BindFrom(*service, deployments[service.Name])
|
||||
target.SetEnvFrom(*service, appName, true)
|
||||
// copy all init containers
|
||||
initContainers := deployments[service.Name].Spec.Template.Spec.InitContainers
|
||||
target.Spec.Template.Spec.InitContainers = append(target.Spec.Template.Spec.InitContainers, initContainers...)
|
||||
delete(deployments, service.Name)
|
||||
} else {
|
||||
log.Printf("service %[1]s is declared as %[2]s, but %[2]s is not defined", service.Name, labels.LabelSamePod)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// create init containers for all DependsOn
|
||||
for _, s := range project.Services {
|
||||
for _, d := range s.GetDependencies() {
|
||||
if dep, ok := deployments[d]; ok {
|
||||
err := deployments[s.Name].DependsOn(dep, d)
|
||||
if err != nil {
|
||||
log.Printf("error creating init container for service %[1]s: %[2]s", s.Name, err)
|
||||
}
|
||||
} else {
|
||||
log.Printf("service %[1]s depends on %[2]s, but %[2]s is not defined", s.Name, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
// it's now time to get "value-from", before makeing the secrets and configmaps!
|
||||
for _, s := range project.Services {
|
||||
chart.setEnvironmentValuesFrom(s, deployments)
|
||||
}
|
||||
|
||||
// generate configmaps with environment variables
|
||||
if err := chart.generateConfigMapsAndSecrets(project); err != nil {
|
||||
log.Fatalf("error generating configmaps and secrets: %s", err)
|
||||
}
|
||||
|
||||
// if the env-from label is set, we need to add the env vars from the configmap
|
||||
// to the environment of the service
|
||||
for _, s := range project.Services {
|
||||
chart.setSharedConf(s, deployments)
|
||||
}
|
||||
|
||||
// remove all "boundEnv" from the values
|
||||
for _, d := range deployments {
|
||||
if len(d.boundEnvVar) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, e := range d.boundEnvVar {
|
||||
delete(chart.Values[d.service.Name].(*Value).Environment, e)
|
||||
}
|
||||
}
|
||||
|
||||
// generate yaml files
|
||||
for _, d := range deployments {
|
||||
y, err := d.Yaml()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chart.Templates[d.Filename()] = &ChartTemplate{
|
||||
Content: y,
|
||||
Servicename: d.service.Name,
|
||||
}
|
||||
}
|
||||
|
||||
// generate all services
|
||||
for _, s := range services {
|
||||
// add the service ports to the target service if it's a "same-pod" service
|
||||
if samePod, ok := podToMerge[s.service.Name]; ok {
|
||||
// get the target service
|
||||
target := services[samePod.Name]
|
||||
// merge the services
|
||||
s.Spec.Ports = append(s.Spec.Ports, target.Spec.Ports...)
|
||||
}
|
||||
y, _ := s.Yaml()
|
||||
chart.Templates[s.Filename()] = &ChartTemplate{
|
||||
Content: y,
|
||||
Servicename: s.service.Name,
|
||||
}
|
||||
}
|
||||
|
||||
// drop all "same-pod" services
|
||||
for _, s := range podToMerge {
|
||||
// get the target service
|
||||
target := services[s.Name]
|
||||
if target != nil {
|
||||
delete(chart.Templates, target.Filename())
|
||||
}
|
||||
}
|
||||
|
||||
// compute all needed resplacements in YAML templates
|
||||
for n, v := range chart.Templates {
|
||||
v.Content = removeReplaceString(v.Content)
|
||||
v.Content = computeNIndent(v.Content)
|
||||
chart.Templates[n].Content = v.Content
|
||||
}
|
||||
|
||||
// generate helper
|
||||
chart.Helper = Helper(appName)
|
||||
|
||||
return chart, nil
|
||||
}
|
||||
|
||||
// dropIngoredServices removes all services with the "ignore" label set to true (or yes).
|
||||
func dropIngoredServices(project *types.Project) {
|
||||
for i, service := range project.Services {
|
||||
if isIgnored(service) {
|
||||
project.Services = append(project.Services[:i], project.Services[i+1:]...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fixResourceNames renames all services and related resources to remove dashes.
|
||||
func fixResourceNames(project *types.Project) error {
|
||||
// rename all services name to remove dashes
|
||||
for i, service := range project.Services {
|
||||
if service.Name != utils.AsResourceName(service.Name) {
|
||||
fixed := utils.AsResourceName(service.Name)
|
||||
for j, s := range project.Services {
|
||||
// for the same-pod services, we need to keep the original name
|
||||
if samepod, ok := s.Labels[labels.LabelSamePod]; ok && samepod == service.Name {
|
||||
s.Labels[labels.LabelSamePod] = fixed
|
||||
project.Services[j] = s
|
||||
}
|
||||
// also, the value-from label should be updated
|
||||
if valuefrom, ok := s.Labels[labels.LabelValuesFrom]; ok {
|
||||
vf, err := labelstructs.GetValueFrom(valuefrom)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for varname, bind := range *vf {
|
||||
log.Printf("service %s, varname %s, bind %s", service.Name, varname, bind)
|
||||
bind := strings.ReplaceAll(bind, service.Name, fixed)
|
||||
(*vf)[varname] = bind
|
||||
}
|
||||
output, err := yaml.Marshal(vf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.Labels[labels.LabelValuesFrom] = string(output)
|
||||
}
|
||||
}
|
||||
service.Name = fixed
|
||||
project.Services[i] = service
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// serviceIsMain returns true if the service is the main app.
|
||||
func serviceIsMain(service types.ServiceConfig) bool {
|
||||
if main, ok := service.Labels[labels.LabelMainApp]; ok {
|
||||
return main == "true" || main == "yes" || main == "1"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func addStaticVolumes(deployments map[string]*Deployment, service types.ServiceConfig) {
|
||||
// add the bound configMaps files to the deployment containers
|
||||
var d *Deployment
|
||||
var ok bool
|
||||
if d, ok = deployments[service.Name]; !ok {
|
||||
log.Printf("service %s not found in deployments", service.Name)
|
||||
return
|
||||
}
|
||||
|
||||
container, index := utils.GetContainerByName(service.ContainerName, d.Spec.Template.Spec.Containers)
|
||||
if container == nil { // may append for the same-pod services
|
||||
return
|
||||
}
|
||||
for volumeName, config := range d.configMaps {
|
||||
var y []byte
|
||||
var err error
|
||||
if y, err = config.configMap.Yaml(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
// add the configmap to the chart
|
||||
d.chart.Templates[config.configMap.Filename()] = &ChartTemplate{
|
||||
Content: y,
|
||||
Servicename: d.service.Name,
|
||||
}
|
||||
// add the moint path to the container
|
||||
for _, m := range config.mountPath {
|
||||
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
|
||||
Name: utils.PathToName(volumeName),
|
||||
MountPath: m.mountPath,
|
||||
SubPath: m.subPath,
|
||||
})
|
||||
}
|
||||
|
||||
d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, corev1.Volume{
|
||||
Name: utils.PathToName(volumeName),
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
ConfigMap: &corev1.ConfigMapVolumeSource{
|
||||
LocalObjectReference: corev1.LocalObjectReference{
|
||||
Name: config.configMap.Name,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
d.Spec.Template.Spec.Containers[index] = *container
|
||||
}
|
||||
|
||||
// computeNIndentm replace all __indent__ labels with the number of spaces before the label.
|
||||
func computeNIndent(b []byte) []byte {
|
||||
lines := bytes.Split(b, []byte("\n"))
|
||||
for i, line := range lines {
|
||||
if !bytes.Contains(line, []byte("__indent__")) {
|
||||
continue
|
||||
}
|
||||
startSpaces := ""
|
||||
spaces := regexp.MustCompile(`^\s+`).FindAllString(string(line), -1)
|
||||
if len(spaces) > 0 {
|
||||
startSpaces = spaces[0]
|
||||
}
|
||||
line = []byte(startSpaces + strings.TrimLeft(string(line), " "))
|
||||
line = bytes.ReplaceAll(line, []byte("__indent__"), fmt.Appendf(nil, "%d", len(startSpaces)))
|
||||
lines[i] = line
|
||||
}
|
||||
return bytes.Join(lines, []byte("\n"))
|
||||
}
|
||||
|
||||
// removeReplaceString replace all __replace_ labels with the value of the
|
||||
// capture group and remove all new lines and repeated spaces.
|
||||
//
|
||||
// we created:
|
||||
//
|
||||
// __replace_bar: '{{ include "foo.labels" .
|
||||
// }}'
|
||||
//
|
||||
// note the new line and spaces...
|
||||
//
|
||||
// we now want to replace it with {{ include "foo.labels" . }}, without the label name.
|
||||
func removeReplaceString(b []byte) []byte {
|
||||
// replace all matches with the value of the capture group
|
||||
// and remove all new lines and repeated spaces
|
||||
b = replaceLabelRegexp.ReplaceAllFunc(b, func(b []byte) []byte {
|
||||
inc := replaceLabelRegexp.FindSubmatch(b)[1]
|
||||
inc = bytes.ReplaceAll(inc, []byte("\n"), []byte(""))
|
||||
inc = bytes.ReplaceAll(inc, []byte("\r"), []byte(""))
|
||||
inc = regexp.MustCompile(`\s+`).ReplaceAll(inc, []byte(" "))
|
||||
return inc
|
||||
})
|
||||
return b
|
||||
}
|
||||
|
||||
// buildVolumes creates the volumes for the service.
|
||||
func buildVolumes(service types.ServiceConfig, chart *HelmChart, deployments map[string]*Deployment) error {
|
||||
appName := chart.Name
|
||||
for _, v := range service.Volumes {
|
||||
// Do not add volumes if the pod is injected in a deployments
|
||||
// via "same-pod" and the volume in destination deployment exists
|
||||
if samePodVolume(service, v, deployments) {
|
||||
continue
|
||||
}
|
||||
switch v.Type {
|
||||
case "volume":
|
||||
v.Source = utils.AsResourceName(v.Source)
|
||||
pvc := NewVolumeClaim(service, v.Source, appName)
|
||||
|
||||
// if the service is integrated in another deployment, we need to add the volume
|
||||
// to the target deployment
|
||||
if override, ok := service.Labels[labels.LabelSamePod]; ok {
|
||||
pvc.nameOverride = override
|
||||
pvc.Spec.StorageClassName = utils.StrPtr(`{{ .Values.` + override + `.persistence.` + v.Source + `.storageClass }}`)
|
||||
chart.Values[override].(*Value).AddPersistence(v.Source)
|
||||
}
|
||||
y, _ := pvc.Yaml()
|
||||
chart.Templates[pvc.Filename()] = &ChartTemplate{
|
||||
Content: y,
|
||||
Servicename: service.Name,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// samePodVolume returns true if the volume is already in the target deployment.
|
||||
func samePodVolume(service types.ServiceConfig, v types.ServiceVolumeConfig, deployments map[string]*Deployment) bool {
|
||||
// if the service has volumes, and it has "same-pod" label
|
||||
// - get the target deployment
|
||||
// - check if it has the same volume
|
||||
// if not, return false
|
||||
|
||||
if v.Source == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
if len(service.Volumes) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
targetDeployment := ""
|
||||
if targetName, ok := service.Labels[labels.LabelSamePod]; !ok {
|
||||
return false
|
||||
} else {
|
||||
targetDeployment = targetName
|
||||
}
|
||||
|
||||
// get the target deployment
|
||||
target := findDeployment(targetDeployment, deployments)
|
||||
if target == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// check if it has the same volume
|
||||
for _, tv := range target.Spec.Template.Spec.Volumes {
|
||||
if tv.Name == v.Source {
|
||||
log.Printf("found same pod volume %s in deployment %s and %s", tv.Name, service.Name, targetDeployment)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fixContainerNames(project *types.Project) {
|
||||
// fix container names to be unique
|
||||
for i, service := range project.Services {
|
||||
if service.ContainerName == "" {
|
||||
service.ContainerName = utils.FixedResourceName(service.Name)
|
||||
} else {
|
||||
service.ContainerName = utils.FixedResourceName(service.ContainerName)
|
||||
}
|
||||
project.Services[i] = service
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user