Big fixes

- Add icon for external service creation log
- Rebase the concurrency
- Add more labels to manage ports
- Change the "ingress" label behavior
- Set more "conditional tests" in generated files
- Many others fixes

README.md

@@ -54,8 +54,8 @@ What can be interpreted by Katenary:
 - if `ports` and/or `expose` section, katenary will create Services and bind the port to the corresponding container port
 - `depends_on` will add init containers to wait for the depending service (using the first port)
 - `env_file` list will create a configMap object per environemnt file (⚠ todo: the "to-service" label doesn't work with configMap for now)
-- some labels can help to bind values:
+- some labels can help to bind values, for example:
-    - `katenary.io/expose-ingress: true` will expose the first port or expose to an ingress
+    - `katenary.io/ingress: 80` will expose the port 80 in a ingress
     - `katenary.io/to-service: VARNAME` will convert the value to a variable `{{ .Release.Name }}-VARNAME` - it's usefull when you want to pass the name of a service as a variable (think about the service name for mysql to pass to a container that wants to connect to this)
 
 Exemple of a possible `docker-compose.yaml` file:
@@ -77,6 +77,8 @@ services:
         labels:
             # explain to katenary that "DB_HOST" value is variable (using release name)
             katenary.io/to-servie: DB_HOST
+            # expose the port 80 as an ingress
+            katenary.io/ingress: 80
     database:
         image: mariabd:10
         env_file:
@@ -92,5 +94,6 @@ services:
 # Labels
 
 - `katenary.io/to-service` binds the given (coma separated) variables names  to {{ .Release.Name }}-value
-- `katenary.io/expose-ingress`: create an ingress and bind it to the service
+- `katenary.io/ingress`: create an ingress and bind it to the given port
 - `katenary.io/as-secret`: force the creation of a secret for the given coma separated list of "env_file"
+- `katenary.io/service-ports` is a coma separated list of ports if you want to avoid the "ports" section in your docker-compose for any reason

generator/main.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"errors"
 )
@@ -17,6 +18,15 @@ var servicesMap = make(map[string]int)
 var serviceWaiters = make(map[string][]chan int)
 var locker = &sync.Mutex{}
 
+const (
+	ICON_PACKAGE = "📦"
+	ICON_SERVICE = "🔌"
+	ICON_SECRET  = "🔏"
+	ICON_CONF    = "📝"
+	ICON_STORE   = "⚡"
+	ICON_INGRESS = "🌐"
+)
+
 // Values is kept in memory to create a values.yaml file.
 var Values = make(map[string]map[string]interface{})
 var VolumeValues = make(map[string]map[string]map[string]interface{})
@@ -34,20 +44,43 @@ echo "Done"
 `
 
 // Create a Deployment for a given compose.Service. It returns a list of objects: a Deployment and a possible Service (kubernetes represnetation as maps).
-func CreateReplicaObject(name string, s compose.Service) (ret []interface{}) {
+func CreateReplicaObject(name string, s compose.Service) chan interface{} {
+
+	// fetch label to specific exposed port, and add them in "ports" section
+	if portlabel, ok := s.Labels[helm.K+"/service-ports"]; ok {
+		services := strings.Split(portlabel, ",")
+		for _, serviceport := range services {
+			portexists := false
+			for _, found := range s.Ports {
+				if found == serviceport {
+					portexists = true
+				}
+			}
+			if !portexists {
+				s.Ports = append(s.Ports, serviceport)
+			}
+		}
+	}
+
+	ret := make(chan interface{}, len(s.Ports)+len(s.Expose)+1)
+	go parseService(name, s, ret)
+	return ret
+}
+
+// This function will try to yied deployment and services based on a service from the compose file structure.
+func parseService(name string, s compose.Service, ret chan interface{}) {
+	Magenta(ICON_PACKAGE+" Generating deployment for ", name)
 
-	Magenta("Generating deployment for ", name)
 	o := helm.NewDeployment(name)
-	ret = append(ret, o)
-
 	container := helm.NewContainer(name, s.Image, s.Environment, s.Labels)
 
+	// prepare secrets
 	secretsFiles := make([]string, 0)
-
 	if v, ok := s.Labels[helm.K+"/as-secret"]; ok {
 		secretsFiles = strings.Split(v, ",")
 	}
 
+	// manage environment files (env_file in compose)
 	for _, envfile := range s.EnvFiles {
 		f := strings.ReplaceAll(envfile, "_", "-")
 		f = strings.ReplaceAll(f, ".env", "")
@@ -61,10 +94,10 @@ func CreateReplicaObject(name string, s compose.Service) (ret []interface{}) {
 		}
 		var store helm.InlineConfig
 		if !isSecret {
-			Bluef("Generating configMap %s\n", cf)
+			Bluef(ICON_CONF+" Generating configMap %s\n", cf)
 			store = helm.NewConfigMap(cf)
 		} else {
-			Bluef("Generating secret %s\n", cf)
+			Bluef(ICON_SECRET+" Generating secret %s\n", cf)
 			store = helm.NewSecret(cf)
 		}
 		if err := store.AddEnvFile(envfile); err != nil {
@@ -77,19 +110,16 @@ func CreateReplicaObject(name string, s compose.Service) (ret []interface{}) {
 			},
 		})
 
-		ret = append(ret, store)
+		ret <- store
-		if isSecret {
-			Greenf("Done secret %s\n", cf)
-		} else {
-			Greenf("Done configMap %s\n", cf)
-		}
 	}
 
+	// check the image, and make it "variable" in values.yaml
 	container.Image = "{{ .Values." + name + ".image }}"
 	Values[name] = map[string]interface{}{
 		"image": s.Image,
 	}
 
+	// manage ports
 	exists := make(map[int]string)
 	for _, port := range s.Ports {
 		_p := strings.Split(port, ":")
@@ -110,6 +140,8 @@ func CreateReplicaObject(name string, s compose.Service) (ret []interface{}) {
 		})
 		exists[portNumber] = name
 	}
+
+	// manage the "expose" section to be a NodePort in Kubernetes
 	for _, port := range s.Expose {
 		if _, exist := exists[port]; exist {
 			continue
@@ -120,6 +152,7 @@ func CreateReplicaObject(name string, s compose.Service) (ret []interface{}) {
 		})
 	}
 
+	// Prepare volumes
 	volumes := make([]map[string]interface{}, 0)
 	mountPoints := make([]interface{}, 0)
 	for _, volume := range s.Volumes {
@@ -127,12 +160,14 @@ func CreateReplicaObject(name string, s compose.Service) (ret []interface{}) {
 		volname := parts[0]
 		volepath := parts[1]
 		if strings.HasPrefix(volname, ".") || strings.HasPrefix(volname, "/") {
+			// local volume cannt be mounted
+			// TODO: propose a way to make configMap for some files or directory
 			Redf("You cannot, at this time, have local volume in %s service", name)
-			os.Exit(1)
+			continue
+			//os.Exit(1)
 		}
 
 		pvc := helm.NewPVC(name, volname)
-		ret = append(ret, pvc)
 		volumes = append(volumes, map[string]interface{}{
 			"name": volname,
 			"persistentVolumeClaim": map[string]string{
@@ -144,7 +179,7 @@ func CreateReplicaObject(name string, s compose.Service) (ret []interface{}) {
 			"mountPath": volepath,
 		})
 
-		Yellow("Generate volume values for ", volname)
+		Yellow(ICON_STORE+" Generate volume values for ", volname, " in deployment ", name)
 		locker.Lock()
 		if _, ok := VolumeValues[name]; !ok {
 			VolumeValues[name] = make(map[string]map[string]interface{})
@@ -154,71 +189,101 @@ func CreateReplicaObject(name string, s compose.Service) (ret []interface{}) {
 			"capacity": "1Gi",
 		}
 		locker.Unlock()
+		ret <- pvc
 	}
 	container.VolumeMounts = mountPoints
 
 	o.Spec.Template.Spec.Volumes = volumes
 	o.Spec.Template.Spec.Containers = []*helm.Container{container}
 
+	// Add some labels
 	o.Spec.Selector = map[string]interface{}{
 		"matchLabels": buildSelector(name, s),
 	}
-
 	o.Spec.Template.Metadata.Labels = buildSelector(name, s)
 
-	wait := &sync.WaitGroup{}
+	// Now, for "depends_on" section, it's a bit tricky...
+	// We need to detect "others" services, but we probably not have parsed them yet, so
+	// we will wait for them for a while.
 	initContainers := make([]*helm.Container, 0)
 	for _, dp := range s.DependsOn {
-		//if len(s.Ports) == 0 && len(s.Expose) == 0 {
-		//	Redf("No port exposed for %s that is in dependency", name)
-		//	os.Exit(1)
-		//}
 		c := helm.NewContainer("check-"+dp, "busybox", nil, s.Labels)
 		command := strings.ReplaceAll(strings.TrimSpace(dependScript), "__service__", dp)
 
-		wait.Add(1)
+		foundPort := -1
-		go func(dp string) {
+		if defaultPort, err := getPort(dp); err != nil {
-			defer wait.Done()
+			// BUG: Sometimes the chan remains opened
-			p := -1
+			foundPort := <-waitPort(dp)
-			if defaultPort, err := getPort(dp); err != nil {
+			if foundPort == -1 {
-				p = <-waitPort(dp)
+				log.Fatalf(
-			} else {
+					"ERROR, the %s service is waiting for %s port number, "+
-				p = defaultPort
+						"but it is never discovered. You must declare at least one port in "+
+						"the \"ports\" section of the service in the docker-compose file",
+					name,
+					dp,
+				)
 			}
-			command = strings.ReplaceAll(command, "__port__", strconv.Itoa(p))
+		} else {
+			foundPort = defaultPort
+		}
+		command = strings.ReplaceAll(command, "__port__", strconv.Itoa(foundPort))
 
-			c.Command = []string{
+		c.Command = []string{
-				"sh",
+			"sh",
-				"-c",
+			"-c",
-				command,
+			command,
-			}
+		}
-			initContainers = append(initContainers, c)
+		initContainers = append(initContainers, c)
-		}(dp)
 	}
-	wait.Wait()
 	o.Spec.Template.Spec.InitContainers = initContainers
 
+	// Then, create services for "ports" and "expose" section
 	if len(s.Ports) > 0 || len(s.Expose) > 0 {
-		ks := createService(name, s)
+		for _, s := range createService(name, s) {
-		ret = append(ret, ks...)
+			ret <- s
+		}
 	}
 
+	// Special case, it there is no "ports", so there is no associated services...
+	// But... some other deployment can wait for it, so we alert that this deployment hasn't got any
+	// associated service.
+	if len(s.Ports) == 0 {
+		locker.Lock()
+		// alert any current or **futur** waiters that this service is not exposed
+		go func() {
+			for {
+				select {
+				case <-time.Tick(1 * time.Millisecond):
+					for _, c := range serviceWaiters[name] {
+						c <- -1
+						close(c)
+					}
+				}
+			}
+		}()
+		locker.Unlock()
+	}
+
+	// add the volumes in Values
 	if len(VolumeValues[name]) > 0 {
+		locker.Lock()
 		Values[name]["persistence"] = VolumeValues[name]
+		locker.Unlock()
 	}
 
-	Green("Done deployment ", name)
+	// the deployment is ready, give it
+	ret <- o
 
-	return
+	// and then, we can say that it's the end
+	ret <- nil
 }
 
 // Create a service (k8s).
 func createService(name string, s compose.Service) []interface{} {
 
 	ret := make([]interface{}, 0)
-	Magenta("Generating service for ", name)
+	Magenta(ICON_SERVICE+" Generating service for ", name)
 	ks := helm.NewService(name)
-	defaultPort := 0
 
 	for i, p := range s.Ports {
 		port := strings.Split(p, ":")
@@ -226,27 +291,27 @@ func createService(name string, s compose.Service) []interface{} {
 		target := src
 		if len(port) > 1 {
 			target, _ = strconv.Atoi(port[1])
-			log.Println(target)
 		}
 		ks.Spec.Ports = append(ks.Spec.Ports, helm.NewServicePort(target, target))
 		if i == 0 {
-			defaultPort = target
 			detected(name, target)
 		}
 	}
 	ks.Spec.Selector = buildSelector(name, s)
 
 	ret = append(ret, ks)
-	if v, ok := s.Labels[helm.K+"/expose-ingress"]; ok && v == "true" {
+	if v, ok := s.Labels[helm.K+"/ingress"]; ok {
-		Cyanf("Create an ingress for %d port on %s service\n", defaultPort, name)
+		port, err := strconv.Atoi(v)
-		ing := createIngress(name, defaultPort, s)
+		if err != nil {
+			log.Fatalf("The given port \"%v\" as ingress port in %s service is not an integer\n", v, name)
+		}
+		Cyanf(ICON_INGRESS+" Create an ingress for port %d on %s service\n", port, name)
+		ing := createIngress(name, port, s)
 		ret = append(ret, ing)
-		Green("Done ingress ", name)
 	}
-	Green("Done service ", name)
 
 	if len(s.Expose) > 0 {
-		Magenta("Generating service for ", name+"-external")
+		Magenta(ICON_SERVICE+" Generating service for ", name+"-external")
 		ks := helm.NewService(name + "-external")
 		ks.Spec.Type = "NodePort"
 		for _, p := range s.Expose {
@@ -264,7 +329,7 @@ func createIngress(name string, port int, s compose.Service) *helm.Ingress {
 	ingress := helm.NewIngress(name)
 	Values[name]["ingress"] = map[string]interface{}{
 		"class":   "nginx",
-		"host":    "chart.example.tld",
+		"host":    name + "." + helm.Appname + ".tld",
 		"enabled": false,
 	}
 	ingress.Spec.Rules = []helm.IngressRule{
@@ -291,7 +356,8 @@ func createIngress(name string, port int, s compose.Service) *helm.Ingress {
 	return ingress
 }
 
-// This function is called when a possible service is detected, it append the port in a map to make others to be able to get the service name. It also try to send the data to any "waiter" for this service.
+// This function is called when a possible service is detected, it append the port in a map to make others
+// to be able to get the service name. It also try to send the data to any "waiter" for this service.
 func detected(name string, port int) {
 	locker.Lock()
 	servicesMap[name] = port
@@ -300,6 +366,7 @@ func detected(name string, port int) {
 		for _, c := range cx {
 			if v, ok := servicesMap[name]; ok {
 				c <- v
+				close(c)
 			}
 		}
 	}()
@@ -321,6 +388,7 @@ func waitPort(name string) chan int {
 	go func() {
 		if v, ok := servicesMap[name]; ok {
 			c <- v
+			close(c)
 		}
 	}()
 	locker.Unlock()

main.go

@@ -10,7 +10,6 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"sync"
 
 	"gopkg.in/yaml.v3"
 )
@@ -22,7 +21,6 @@ var Version = "master"
 var ChartsDir = "chart"
 
 func main() {
-
 	flag.StringVar(&ChartsDir, "chart-dir", ChartsDir, "set the chart directory")
 	flag.StringVar(&ComposeFile, "compose", ComposeFile, "set the compose file to parse")
 	flag.StringVar(&AppName, "appname", AppName, "sive the helm chart app name")
@@ -61,28 +59,31 @@ func main() {
 	helm.Version = Version
 	p := compose.NewParser(ComposeFile)
 	p.Parse(AppName)
-	wait := sync.WaitGroup{}
 
-	files := make(map[string][]interface{})
+	files := make(map[string]chan interface{})
 
+	//wait := sync.WaitGroup{}
 	for name, s := range p.Data.Services {
-		wait.Add(1)
+		//wait.Add(1)
 		// it's mandatory to build in goroutines because some dependencies can
 		// wait for a port number discovery.
 		// So the entire services are built in parallel.
-		go func(name string, s compose.Service) {
+		//go func(name string, s compose.Service) {
-			o := generator.CreateReplicaObject(name, s)
+		//	defer wait.Done()
-			files[name] = o
+		o := generator.CreateReplicaObject(name, s)
-			wait.Done()
+		files[name] = o
-		}(name, s)
+		//}(name, s)
 	}
-	wait.Wait()
+	//wait.Wait()
 
 	// to generate notes, we need to keep an Ingresses list
 	ingresses := make(map[string]*helm.Ingress)
 
 	for n, f := range files {
-		for _, c := range f {
+		for c := range f {
+			if c == nil {
+				break
+			}
 			kind := c.(helm.Kinded).Get()
 			kind = strings.ToLower(kind)
 			c.(helm.Signable).BuildSHA(ComposeFile)
@@ -121,6 +122,7 @@ func main() {
 					fp.WriteString(line + "\n")
 				}
 				fp.Close()
+
 			case *helm.Service:
 				suffix := ""
 				if c.Spec.Type == "NodePort" {
@@ -129,18 +131,36 @@ func main() {
 				fname := filepath.Join(templatesDir, n+suffix+"."+kind+".yaml")
 				fp, _ := os.Create(fname)
 				enc := yaml.NewEncoder(fp)
+				enc.SetIndent(2)
 				enc.Encode(c)
 				fp.Close()
 
 			case *helm.Ingress:
+				buffer := bytes.NewBuffer(nil)
 				fname := filepath.Join(templatesDir, n+"."+kind+".yaml")
-				fp, _ := os.Create(fname)
 				ingresses[n] = c // keep it to generate notes
-				enc := yaml.NewEncoder(fp)
+				enc := yaml.NewEncoder(buffer)
 				enc.SetIndent(2)
-				fp.WriteString("{{- if .Values." + n + ".ingress.enabled -}}\n")
+				buffer.WriteString("{{- if .Values." + n + ".ingress.enabled -}}\n")
 				enc.Encode(c)
-				fp.WriteString("{{- end -}}")
+				buffer.WriteString("{{- end -}}")
+
+				fp, _ := os.Create(fname)
+				content := string(buffer.Bytes())
+				lines := strings.Split(content, "\n")
+				for _, l := range lines {
+					if strings.Contains(l, "ingressClassName") {
+						p := strings.Split(l, ":")
+						condition := p[1]
+						condition = strings.ReplaceAll(condition, "'", "")
+						condition = strings.ReplaceAll(condition, "{{", "")
+						condition = strings.ReplaceAll(condition, "}}", "")
+						condition = strings.TrimSpace(condition)
+						condition = "{{- if " + condition + " }}"
+						l = "  " + condition + "\n" + l + "\n  {{- end }}"
+					}
+					fp.WriteString(l + "\n")
+				}
 				fp.Close()
 
 			default: