Use compose-go + improvements (#9)

Use compose-go https://github.com/compose-spec/compose-go  to make Katenary parsing compose file the official way.
Add labels:
- `volume-from` (with `same-pod`) to avoid volume repetition
- `ignore` to ignore a service
- `mapenv` (replaces the `env-to-service`) to map environment to helm variable (as a template string)
- `secret-vars` declares variables as secret values

More:
- Now, environment (as secret vars) are set in values.yaml
- Ingress has got annotations in values.yaml
- Probes (liveness probe) are improved
- fixed code to optimize
- many others fixes about path, bad volume check, refactorisation, tests...
This commit is contained in:
2022-05-08 09:55:25 +02:00
committed by GitHub
parent 165054ca53
commit 418a0a8029
41 changed files with 1696 additions and 1029 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -9,6 +9,8 @@ import (
"path/filepath"
"strings"
"testing"
"github.com/compose-spec/compose-go/cli"
)
const DOCKER_COMPOSE_YML = `version: '3'
@@ -48,7 +50,8 @@ services:
ANOTHER_ENV_VAR: another_value
DB_HOST: database
labels:
katenary.io/env-to-service: DB_HOST
katenary.io/mapenv: |
DB_HOST: {{ .Release.Name }}-database
database:
image: mysql:5.7
@@ -87,37 +90,85 @@ services:
- SOME_ENV_VAR=some_value
- ANOTHER_ENV_VAR=another_value
# use environment file
useenvfile:
image: nginx
env_file:
- config/env
volumes:
data:
driver: local
`
var defaultCliFiles = cli.DefaultFileNames
var TMP_DIR = ""
var TMPWORK_DIR = ""
func init() {
logger.NOLOG = true
logger.NOLOG = len(os.Getenv("NOLOG")) < 1
}
func setUp(t *testing.T) (string, *compose.Parser) {
p := compose.NewParser("", DOCKER_COMPOSE_YML)
p.Parse("testapp")
// cleanup "made" files
helm.ResetMadePVC()
cli.DefaultFileNames = defaultCliFiles
// create a temporary directory
tmp, err := os.MkdirTemp(os.TempDir(), "katenary-test")
t.Log("Generated ", tmp, "directory")
tmp, err := os.MkdirTemp(os.TempDir(), "katenary-test-")
if err != nil {
t.Fatal(err)
}
Generate(p, "test-0", "testapp", "1.2.3", DOCKER_COMPOSE_YML, tmp)
tmpwork, err := os.MkdirTemp(os.TempDir(), "katenary-test-work-")
if err != nil {
t.Fatal(err)
}
composefile := filepath.Join(tmpwork, "docker-compose.yaml")
p := compose.NewParser(composefile, DOCKER_COMPOSE_YML)
// create envfile for "useenvfile" service
err = os.Mkdir(filepath.Join(tmpwork, "config"), 0777)
if err != nil {
t.Fatal(err)
}
envfile := filepath.Join(tmpwork, "config", "env")
fp, err := os.Create(envfile)
if err != nil {
t.Fatal("MKFILE", err)
}
fp.WriteString("FILEENV1=some_value\n")
fp.WriteString("FILEENV2=another_value\n")
fp.Close()
TMP_DIR = tmp
TMPWORK_DIR = tmpwork
p.Parse("testapp")
Generate(p, "test-0", "testapp", "1.2.3", "4.5.6", DOCKER_COMPOSE_YML, tmp)
return tmp, p
}
func tearDown() {
if len(TMP_DIR) > 0 {
os.RemoveAll(TMP_DIR)
}
if len(TMPWORK_DIR) > 0 {
os.RemoveAll(TMPWORK_DIR)
}
}
// Check if the web2 service has got a command.
func TestCommand(t *testing.T) {
tmp, p := setUp(t)
defer os.RemoveAll(tmp)
defer tearDown()
for name := range p.Data.Services {
for _, service := range p.Data.Services {
name := service.Name
if name == "web2" {
// Ensure that the command is correctly set
// The command should be a string array
@@ -159,9 +210,10 @@ func TestCommand(t *testing.T) {
// Check if environment is correctly set.
func TestEnvs(t *testing.T) {
tmp, p := setUp(t)
defer os.RemoveAll(tmp)
defer tearDown()
for name := range p.Data.Services {
for _, service := range p.Data.Services {
name := service.Name
if name == "php" {
// the "DB_HOST" environment variable inside the template must be set to '{{ .Release.Name }}-database'
@@ -173,20 +225,20 @@ func TestEnvs(t *testing.T) {
lines, _ := ioutil.ReadAll(fp)
next := false
for _, line := range strings.Split(string(lines), "\n") {
if strings.Contains(line, "DB_HOST") {
if !next && strings.Contains(line, "name: DB_HOST") {
next = true
continue
}
if next {
} else if next && strings.Contains(line, "value:") {
matched = true
if !strings.Contains(line, helm.RELEASE_NAME+"-database") {
t.Error("DB_HOST variable should be set to " + helm.RELEASE_NAME + "-database")
if !strings.Contains(line, "{{ tpl .Values.php.environment.DB_HOST . }}") {
t.Error("DB_HOST variable should be set to {{ tpl .Values.php.environment.DB_HOST . }}", line, string(lines))
}
break
}
}
if !matched {
t.Error("DB_HOST variable not found in ", path)
t.Log(string(lines))
}
}
}
@@ -195,9 +247,10 @@ func TestEnvs(t *testing.T) {
// Check if the same pod is not deployed twice.
func TestSamePod(t *testing.T) {
tmp, p := setUp(t)
defer os.RemoveAll(tmp)
defer tearDown()
for name, service := range p.Data.Services {
for _, service := range p.Data.Services {
name := service.Name
path := filepath.Join(tmp, "templates", name+".deployment.yaml")
if _, found := service.Labels[helm.LABEL_SAMEPOD]; found {
@@ -220,9 +273,10 @@ func TestSamePod(t *testing.T) {
// Check if the ports are correctly set.
func TestPorts(t *testing.T) {
tmp, p := setUp(t)
defer os.RemoveAll(tmp)
defer tearDown()
for name, service := range p.Data.Services {
for _, service := range p.Data.Services {
name := service.Name
path := ""
// if the service has a port found in helm.LABEL_PORT or ports, so the service file should exist
@@ -238,7 +292,7 @@ func TestPorts(t *testing.T) {
t.Log("Checking ", name, " service file")
_, err := os.Stat(path)
if err != nil {
t.Fatal(err)
t.Error(err)
}
}
}
@@ -247,9 +301,10 @@ func TestPorts(t *testing.T) {
// Check if the volumes are correctly set.
func TestPVC(t *testing.T) {
tmp, p := setUp(t)
defer os.RemoveAll(tmp)
defer tearDown()
for name := range p.Data.Services {
for _, service := range p.Data.Services {
name := service.Name
path := filepath.Join(tmp, "templates", name+"-data.pvc.yaml")
// the "database" service should have a pvc file in templates (name-data.pvc.yaml)
@@ -258,6 +313,8 @@ func TestPVC(t *testing.T) {
t.Log("Checking ", name, " pvc file")
_, err := os.Stat(path)
if err != nil {
list, _ := filepath.Glob(tmp + "/templates/*")
t.Log(list)
t.Fatal(err)
}
}
@@ -267,9 +324,10 @@ func TestPVC(t *testing.T) {
//Check if web service has got a ingress.
func TestIngress(t *testing.T) {
tmp, p := setUp(t)
defer os.RemoveAll(tmp)
defer tearDown()
for name := range p.Data.Services {
for _, service := range p.Data.Services {
name := service.Name
path := filepath.Join(tmp, "templates", name+".ingress.yaml")
// the "web" service should have a ingress file in templates (name.ingress.yaml)
@@ -287,9 +345,10 @@ func TestIngress(t *testing.T) {
// Check unmapped volumes
func TestUnmappedVolumes(t *testing.T) {
tmp, p := setUp(t)
defer os.RemoveAll(tmp)
defer tearDown()
for name := range p.Data.Services {
for _, service := range p.Data.Services {
name := service.Name
if name == "novol" {
path := filepath.Join(tmp, "templates", name+".deployment.yaml")
fp, _ := os.Open(path)
@@ -307,10 +366,11 @@ func TestUnmappedVolumes(t *testing.T) {
// Check if service using equal sign for environment works
func TestEqualSignOnEnv(t *testing.T) {
tmp, p := setUp(t)
defer os.RemoveAll(tmp)
defer tearDown()
// if the name is eqenv, the service should habe environment
for name, _ := range p.Data.Services {
for _, service := range p.Data.Services {
name := service.Name
if name == "eqenv" {
path := filepath.Join(tmp, "templates", name+".deployment.yaml")
fp, _ := os.Open(path)
@@ -328,8 +388,9 @@ func TestEqualSignOnEnv(t *testing.T) {
match++
}
}
if match != 2 {
if match != 4 { // because the value points on .Values...
t.Error("eqenv service should have 2 environment variables")
t.Log(string(lines))
}
}
}

22
generator/utils.go Normal file
View File

@@ -0,0 +1,22 @@
package generator
import (
"katenary/compose"
"regexp"
"strings"
)
// replaceChars replaces some chars in a string.
const replaceChars = `[^a-zA-Z0-9._]`
// GetRelPath return the relative path from the root of the project.
func GetRelPath(path string) string {
return strings.Replace(path, compose.GetCurrentDir(), ".", 1)
}
// PathToName transform a path to a yaml name.
func PathToName(path string) string {
path = strings.TrimPrefix(GetRelPath(path), "./")
path = regexp.MustCompile(replaceChars).ReplaceAllString(path, "-")
return path
}

View File

@@ -4,18 +4,42 @@ import (
"katenary/compose"
"katenary/generator/writers"
"katenary/helm"
"log"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
"github.com/compose-spec/compose-go/types"
"gopkg.in/yaml.v3"
)
// HelmFile represents a helm file from helm package that has got some necessary methods
// to generate a helm file.
type HelmFile interface {
GetType() string
GetPathRessource() string
}
// HelmFileGenerator is a chanel of HelmFile.
type HelmFileGenerator chan HelmFile
var PrefixRE = regexp.MustCompile(`\{\{.*\}\}-?`)
func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFile, dirName string) {
func portExists(port int, ports []types.ServicePortConfig) bool {
for _, p := range ports {
if p.Target == uint32(port) {
log.Println("portExists:", port, p.Target)
return true
}
}
return false
}
// Generate get a parsed compose file, and generate the helm files.
func Generate(p *compose.Parser, katernayVersion, appName, appVersion, chartVersion, composeFile, dirName string) {
// make the appname global (yes... ugly but easy)
helm.Appname = appName
@@ -25,64 +49,108 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi
// try to create the directory
err := os.MkdirAll(templatesDir, 0755)
if err != nil {
panic(err)
log.Fatal(err)
}
files := make(map[string]chan interface{})
generators := make(map[string]HelmFileGenerator)
// list avoided services
avoids := make(map[string]bool)
for n, service := range p.Data.Services {
if _, ok := service.Labels[helm.LABEL_SAMEPOD]; ok {
avoids[n] = true
// remove skipped services from the parsed data
for i, service := range p.Data.Services {
if v, ok := service.Labels[helm.LABEL_IGNORE]; !ok || v != "true" {
continue
}
p.Data.Services = append(p.Data.Services[:i], p.Data.Services[i+1:]...)
i--
// find this service in others as "depends_on" and remove it
for _, service2 := range p.Data.Services {
delete(service2.DependsOn, service.Name)
}
}
for name, s := range p.Data.Services {
for i, service := range p.Data.Services {
n := service.Name
// Manage emptyDir volumes
if empty, ok := s.Labels[helm.LABEL_EMPTYDIRS]; ok {
//split empty list by coma
emptyDirs := strings.Split(empty, ",")
//append them in EmptyDirs
EmptyDirs = append(EmptyDirs, emptyDirs...)
}
// fetch corresponding service in "links"
linked := make(map[string]*compose.Service, 0)
// find service linked to this one
for n, service := range p.Data.Services {
if _, ok := service.Labels[helm.LABEL_SAMEPOD]; ok {
if service.Labels[helm.LABEL_SAMEPOD] == name {
linked[n] = service
// if the service port is declared in labels, add it to the service.
if ports, ok := service.Labels[helm.LABEL_PORT]; ok {
if service.Ports == nil {
service.Ports = make([]types.ServicePortConfig, 0)
}
for _, port := range strings.Split(ports, ",") {
target, err := strconv.Atoi(port)
if err != nil {
log.Fatal(err)
}
if portExists(target, service.Ports) {
continue
}
service.Ports = append(service.Ports, types.ServicePortConfig{
Target: uint32(target),
})
}
}
// find port and store it in servicesMap
for _, port := range service.Ports {
target := int(port.Target)
if target != 0 {
servicesMap[n] = target
break
}
}
if _, found := avoids[name]; found {
// manage emptyDir volumes
if empty, ok := service.Labels[helm.LABEL_EMPTYDIRS]; ok {
//split empty list by coma
emptyDirs := strings.Split(empty, ",")
for i, emptyDir := range emptyDirs {
emptyDirs[i] = strings.TrimSpace(emptyDir)
}
//append them in EmptyDirs
EmptyDirs = append(EmptyDirs, emptyDirs...)
}
p.Data.Services[i] = service
}
// for all services in linked map, and not in samePods map, generate the service
for _, s := range p.Data.Services {
name := s.Name
// do not make a deployment for services declared to be in the same pod than another
if _, ok := s.Labels[helm.LABEL_SAMEPOD]; ok {
continue
}
files[name] = CreateReplicaObject(name, s, linked)
// find services that is in the same pod
linked := make(map[string]types.ServiceConfig, 0)
for _, service := range p.Data.Services {
n := service.Name
if linkname, ok := service.Labels[helm.LABEL_SAMEPOD]; ok && linkname == name {
linked[n] = service
}
}
generators[name] = CreateReplicaObject(name, s, linked)
}
// to generate notes, we need to keep an Ingresses list
ingresses := make(map[string]*helm.Ingress)
for n, f := range files {
for c := range f {
if c == nil {
for n, generator := range generators { // generators is a map : name -> generator
for helmFile := range generator { // generator is a chan
if helmFile == nil { // generator finished
break
}
kind := c.(helm.Kinded).Get()
kind := helmFile.(helm.Kinded).Get()
kind = strings.ToLower(kind)
// Add a SHA inside the generated file, it's only
// to make it easy to check it the compose file corresponds to the
// generated helm chart
c.(helm.Signable).BuildSHA(composeFile)
helmFile.(helm.Signable).BuildSHA(composeFile)
// Some types need special fixes in yaml generation
switch c := c.(type) {
switch c := helmFile.(type) {
case *helm.Storage:
// For storage, we need to add a "condition" to activate it
writers.BuildStorage(c, n, templatesDir)
@@ -104,45 +172,60 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi
case *helm.ConfigMap, *helm.Secret:
// there could be several files, so let's force the filename
name := c.(helm.Named).Name()
name := c.(helm.Named).Name() + "-" + c.GetType()
suffix := c.GetPathRessource()
suffix = PathToName(suffix)
name += suffix
name = PrefixRE.ReplaceAllString(name, "")
writers.BuildConfigMap(c, kind, n, name, templatesDir)
default:
fname := filepath.Join(templatesDir, n+"."+kind+".yaml")
fp, _ := os.Create(fname)
fp, err := os.Create(fname)
if err != nil {
log.Fatal(err)
}
defer fp.Close()
enc := yaml.NewEncoder(fp)
enc.SetIndent(2)
enc.SetIndent(writers.IndentSize)
enc.Encode(c)
fp.Close()
}
}
}
// Create the values.yaml file
fp, _ := os.Create(filepath.Join(dirName, "values.yaml"))
enc := yaml.NewEncoder(fp)
enc.SetIndent(2)
valueFile, err := os.Create(filepath.Join(dirName, "values.yaml"))
if err != nil {
log.Fatal(err)
}
defer valueFile.Close()
enc := yaml.NewEncoder(valueFile)
enc.SetIndent(writers.IndentSize)
enc.Encode(Values)
fp.Close()
// Create tht Chart.yaml file
fp, _ = os.Create(filepath.Join(dirName, "Chart.yaml"))
fp.WriteString(`# Create on ` + time.Now().Format(time.RFC3339) + "\n")
fp.WriteString(`# Katenary command line: ` + strings.Join(os.Args, " ") + "\n")
enc = yaml.NewEncoder(fp)
chartFile, err := os.Create(filepath.Join(dirName, "Chart.yaml"))
if err != nil {
log.Fatal(err)
}
defer chartFile.Close()
chartFile.WriteString(`# Create on ` + time.Now().Format(time.RFC3339) + "\n")
chartFile.WriteString(`# Katenary command line: ` + strings.Join(os.Args, " ") + "\n")
enc = yaml.NewEncoder(chartFile)
enc.SetIndent(writers.IndentSize)
enc.Encode(map[string]interface{}{
"apiVersion": "v2",
"name": appName,
"description": "A helm chart for " + appName,
"type": "application",
"version": "0.1.0",
"version": chartVersion,
"appVersion": appVersion,
})
fp.Close()
// And finally, create a NOTE.txt file
fp, _ = os.Create(filepath.Join(templatesDir, "NOTES.txt"))
fp.WriteString(helm.GenerateNotesFile(ingresses))
fp.Close()
noteFile, err := os.Create(filepath.Join(templatesDir, "NOTES.txt"))
if err != nil {
log.Fatal(err)
}
defer noteFile.Close()
noteFile.WriteString(helm.GenerateNotesFile(ingresses))
}

View File

@@ -7,8 +7,9 @@ import (
"gopkg.in/yaml.v3"
)
// BuildConfigMap writes the configMap.
func BuildConfigMap(c interface{}, kind, servicename, name, templatesDir string) {
fname := filepath.Join(templatesDir, servicename+"."+name+"."+kind+".yaml")
fname := filepath.Join(templatesDir, name+"."+kind+".yaml")
fp, _ := os.Create(fname)
enc := yaml.NewEncoder(fp)
enc.SetIndent(IndentSize)

View File

@@ -10,6 +10,7 @@ import (
"gopkg.in/yaml.v3"
)
// BuildDeployment builds a deployment.
func BuildDeployment(deployment *helm.Deployment, name, templatesDir string) {
kind := "deployment"
fname := filepath.Join(templatesDir, name+"."+kind+".yaml")

View File

@@ -10,9 +10,20 @@ import (
"gopkg.in/yaml.v3"
)
const classAndVersionCondition = `{{- if and .Values.__name__.ingress.class (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}` + "\n"
const versionCondition = `{{- if semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion }}` + "\n"
const (
classAndVersionCondition = `{{- if and .Values.__name__.ingress.class (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}` + "\n"
versionCondition118 = `{{- if semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion }}` + "\n"
versionCondition119 = `{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}` + "\n"
apiVersion = `{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}`
)
// BuildIngress generates the ingress yaml file with conditions.
func BuildIngress(ingress *helm.Ingress, name, templatesDir string) {
// Set the backend for 1.18
for _, b := range ingress.Spec.Rules {
@@ -32,22 +43,49 @@ func BuildIngress(ingress *helm.Ingress, name, templatesDir string) {
enc.Encode(ingress)
buffer.WriteString("{{- end -}}")
fp, _ := os.Create(fname)
fp, err := os.Create(fname)
if err != nil {
panic(err)
}
defer fp.Close()
content := string(buffer.Bytes())
lines := strings.Split(content, "\n")
backendHit := false
for _, l := range lines {
// apiVersion is a pain...
if strings.Contains(l, "apiVersion:") {
l = apiVersion
}
// add annotations linked to the Values
if strings.Contains(l, "annotations:") {
n := CountSpaces(l) + IndentSize
l += "\n" + strings.Repeat(" ", n) + "{{- range $k, $v := .Values.__name__.ingress.annotations }}\n"
l += strings.Repeat(" ", n) + "{{ $k }}: {{ $v }}\n"
l += strings.Repeat(" ", n) + "{{- end }}"
l = strings.ReplaceAll(l, "__name__", name)
}
// pathTyype is ony for 1.19+
if strings.Contains(l, "pathType:") {
n := CountSpaces(l)
l = strings.Repeat(" ", n) + versionCondition118 +
l + "\n" +
strings.Repeat(" ", n) + "{{- end }}"
}
if strings.Contains(l, "ingressClassName") {
// should be set only if the version of Kubernetes is 1.18-0 or higher
cond := strings.ReplaceAll(classAndVersionCondition, "__name__", name)
l = ` ` + cond + l + "\n" + ` {{- end }}`
}
// manage the backend format following the Kubernetes 1.18-0 version or higher
// manage the backend format following the Kubernetes 1.19-0 version or higher
if strings.Contains(l, "service:") {
n := CountSpaces(l)
l = strings.Repeat(" ", n) + versionCondition + l
l = strings.Repeat(" ", n) + versionCondition119 + l
}
if strings.Contains(l, "serviceName:") || strings.Contains(l, "servicePort:") {
n := CountSpaces(l)
@@ -58,9 +96,6 @@ func BuildIngress(ingress *helm.Ingress, name, templatesDir string) {
}
backendHit = true
}
fp.WriteString(l + "\n")
}
fp.Close()
}

View File

@@ -8,6 +8,7 @@ import (
"gopkg.in/yaml.v3"
)
// BuildService writes the service (external or not).
func BuildService(service *helm.Service, name, templatesDir string) {
kind := "service"
suffix := ""

View File

@@ -2,23 +2,31 @@ package writers
import (
"katenary/helm"
"log"
"os"
"path/filepath"
"gopkg.in/yaml.v3"
)
// BuildStorage writes the persistentVolumeClaim.
func BuildStorage(storage *helm.Storage, name, templatesDir string) {
kind := "pvc"
name = storage.Metadata.Labels[helm.K+"/component"]
pvcname := storage.Metadata.Labels[helm.K+"/pvc-name"]
fname := filepath.Join(templatesDir, name+"-"+pvcname+"."+kind+".yaml")
fp, _ := os.Create(fname)
fp, err := os.Create(fname)
if err != nil {
log.Fatal(err)
}
defer fp.Close()
volname := storage.K8sBase.Metadata.Labels[helm.K+"/pvc-name"]
fp.WriteString("{{ if .Values." + name + ".persistence." + volname + ".enabled }}\n")
enc := yaml.NewEncoder(fp)
enc.SetIndent(IndentSize)
enc.Encode(storage)
if err := enc.Encode(storage); err != nil {
log.Fatal(err)
}
fp.WriteString("{{- end -}}")
}

View File

@@ -1,6 +1,6 @@
package writers
// IndentSize set the indentation size for yaml output.
// IndentSize set the indentation size for yaml output. Could ba changed by command line argument.
var IndentSize = 2
// CountSpaces returns the number of spaces from the begining of the line.