From 5e8cf47ab6f12eb5a3482381b21994bfb65e9890 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Tue, 5 Apr 2022 08:05:33 +0200 Subject: [PATCH] WIP try to fix race condition and tests --- compose/parser.go | 30 +++++++++---------- generator/main.go | 51 ++++++++++++++++++++++---------- generator/main_test.go | 67 ++++++++++++++++++++++++++++++++++-------- generator/writer.go | 14 +++++++++ 4 files changed, 118 insertions(+), 44 deletions(-) diff --git a/compose/parser.go b/compose/parser.go index 427176a..ca95fd7 100644 --- a/compose/parser.go +++ b/compose/parser.go @@ -1,6 +1,7 @@ package compose import ( + "io/ioutil" "log" "os" "path/filepath" @@ -19,27 +20,24 @@ type Parser struct { temporary *string } -var Appname = "" +var ( + Appname = "" + CURRENT_DIR, _ = os.Getwd() +) // NewParser create a Parser and parse the file given in filename. If filename is empty, we try to parse the content[0] argument that should be a valid YAML content. func NewParser(filename string, content ...string) *Parser { p := &Parser{} - if len(content) > 0 { - //write it in a temporary file - tmp, err := os.MkdirTemp(os.TempDir(), "katenary-") + if len(content) > 0 { // mainly for the tests... + dir := filepath.Dir(filename) + err := os.MkdirAll(dir, 0755) if err != nil { log.Fatal(err) } - tmpfile, err := os.Create(filepath.Join(tmp, "tmp.yml")) - if err != nil { - log.Fatal(err) - } - tmpfile.WriteString(content[0]) - tmpfile.Close() - filename = tmpfile.Name() - p.temporary = &tmp + p.temporary = &dir + ioutil.WriteFile(filename, []byte(content[0]), 0644) cli.DefaultFileNames = []string{filename} } // if filename is not in cli Default files, add it @@ -84,7 +82,9 @@ func (p *Parser) Parse(appname string) { Appname = proj.Name p.Data = proj - if p.temporary != nil { - defer os.RemoveAll(*p.temporary) - } + CURRENT_DIR = p.Data.WorkingDir +} + +func GetCurrentDir() string { + return CURRENT_DIR } diff --git a/generator/main.go b/generator/main.go index 1afb380..95c0285 100644 --- a/generator/main.go +++ b/generator/main.go @@ -3,6 +3,7 @@ package generator import ( "fmt" "io/ioutil" + "katenary/compose" "katenary/helm" "katenary/logger" "log" @@ -123,9 +124,7 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se // add the volumes in Values if len(VolumeValues[name]) > 0 { - locker.Lock() - Values[name]["persistence"] = VolumeValues[name] - locker.Unlock() + AddValues(name, map[string]interface{}{"persistence": VolumeValues[name]}) } // the deployment is ready, give it @@ -143,11 +142,7 @@ func prepareContainer(container *helm.Container, service types.ServiceConfig, se } container.Image = "{{ .Values." + servicename + ".image }}" container.Command = service.Command - locker.Lock() - Values[servicename] = map[string]interface{}{ - "image": service.Image, - } - locker.Unlock() + AddValues(servicename, map[string]interface{}{"image": service.Image}) prepareProbes(servicename, service, container) generateContainerPorts(service, servicename, container) } @@ -195,11 +190,14 @@ func generateServicesAndIngresses(name string, s types.ServiceConfig) []interfac // Create an ingress. func createIngress(name string, port int, s types.ServiceConfig) *helm.Ingress { ingress := helm.NewIngress(name) - Values[name]["ingress"] = map[string]interface{}{ + + ingressVal := map[string]interface{}{ "class": "nginx", "host": name + "." + helm.Appname + ".tld", "enabled": false, } + AddValues(name, map[string]interface{}{"ingress": ingressVal}) + ingress.Spec.Rules = []helm.IngressRule{ { Host: fmt.Sprintf("{{ .Values.%s.ingress.host }}", name), @@ -425,15 +423,10 @@ func prepareVolumes(deployment, name string, s types.ServiceConfig, container *h }) logger.Yellow(ICON_STORE+" Generate volume values", volname, "for container named", name, "in deployment", deployment) - locker.Lock() - if _, ok := VolumeValues[deployment]; !ok { - VolumeValues[deployment] = make(map[string]map[string]interface{}) - } - VolumeValues[deployment][volname] = map[string]interface{}{ + AddVolumeValues(deployment, volname, map[string]interface{}{ "enabled": false, "capacity": "1Gi", - } - locker.Unlock() + }) if _, ok := madePVC[deployment+volname]; !ok { madePVC[deployment+volname] = true @@ -521,6 +514,8 @@ func prepareEnvFromFiles(name string, s types.ServiceConfig, container *helm.Con logger.Bluef(ICON_SECRET+" Generating secret %s\n", cf) store = helm.NewSecret(cf) } + + envfile = filepath.Join(compose.GetCurrentDir(), envfile) if err := store.AddEnvFile(envfile); err != nil { logger.ActivateColors = true logger.Red(err.Error()) @@ -542,3 +537,27 @@ func prepareEnvFromFiles(name string, s types.ServiceConfig, container *helm.Con ret <- store } } + +func AddValues(servicename string, values map[string]interface{}) { + locker.Lock() + defer locker.Unlock() + + if _, ok := values[servicename]; !ok { + Values[servicename] = make(map[string]interface{}) + } + + for k, v := range values { + Values[servicename][k] = v + } + +} + +func AddVolumeValues(deployment string, volname string, values map[string]interface{}) { + locker.Lock() + defer locker.Unlock() + + if _, ok := VolumeValues[deployment]; !ok { + VolumeValues[deployment] = make(map[string]map[string]interface{}) + } + VolumeValues[deployment][volname] = values +} diff --git a/generator/main_test.go b/generator/main_test.go index 50caf52..080591a 100644 --- a/generator/main_test.go +++ b/generator/main_test.go @@ -89,37 +89,79 @@ services: - SOME_ENV_VAR=some_value - ANOTHER_ENV_VAR=another_value + # use environment file + useenvfile: + image: nginx + env_file: + - config/env + volumes: data: driver: local ` var defaultCliFiles = cli.DefaultFileNames +var TMP_DIR = "" +var TMPWORK_DIR = "" func init() { - logger.NOLOG = true + logger.NOLOG = len(os.Getenv("NOLOG")) < 1 } func setUp(t *testing.T) (string, *compose.Parser) { cli.DefaultFileNames = defaultCliFiles - p := compose.NewParser("", DOCKER_COMPOSE_YML) - p.Parse("testapp") // create a temporary directory - tmp, err := os.MkdirTemp(os.TempDir(), "katenary-test") + tmp, err := os.MkdirTemp(os.TempDir(), "katenary-test-") if err != nil { t.Fatal(err) } + tmpwork, err := os.MkdirTemp(os.TempDir(), "katenary-test-work-") + if err != nil { + t.Fatal(err) + } + + composefile := filepath.Join(tmpwork, "docker-compose.yaml") + p := compose.NewParser(composefile, DOCKER_COMPOSE_YML) + + // create envfile for "useenvfile" service + err = os.Mkdir(filepath.Join(tmpwork, "config"), 0777) + if err != nil { + t.Fatal(err) + } + envfile := filepath.Join(tmpwork, "config", "env") + fp, err := os.Create(envfile) + if err != nil { + t.Fatal("MKFILE", err) + } + fp.WriteString("FILEENV1=some_value\n") + fp.WriteString("FILEENV2=another_value\n") + fp.Close() + + TMP_DIR = tmp + TMPWORK_DIR = tmpwork + + p.Parse("testapp") + Generate(p, "test-0", "testapp", "1.2.3", DOCKER_COMPOSE_YML, tmp) return tmp, p } +func tearDown() { + if len(TMP_DIR) > 0 { + os.RemoveAll(TMP_DIR) + } + if len(TMPWORK_DIR) > 0 { + os.RemoveAll(TMPWORK_DIR) + } +} + // Check if the web2 service has got a command. func TestCommand(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -142,7 +184,6 @@ func TestCommand(t *testing.T) { commands = append(commands, line) } } - //log.Printf("%#v\n", commands) ok := 0 for _, command := range commands { if strings.Contains(command, "- /bin/sh") { @@ -165,7 +206,7 @@ func TestCommand(t *testing.T) { // Check if environment is correctly set. func TestEnvs(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -202,7 +243,7 @@ func TestEnvs(t *testing.T) { // Check if the same pod is not deployed twice. func TestSamePod(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -228,7 +269,7 @@ func TestSamePod(t *testing.T) { // Check if the ports are correctly set. func TestPorts(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -256,7 +297,7 @@ func TestPorts(t *testing.T) { // Check if the volumes are correctly set. func TestPVC(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -277,7 +318,7 @@ func TestPVC(t *testing.T) { //Check if web service has got a ingress. func TestIngress(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -298,7 +339,7 @@ func TestIngress(t *testing.T) { // Check unmapped volumes func TestUnmappedVolumes(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -319,7 +360,7 @@ func TestUnmappedVolumes(t *testing.T) { // Check if service using equal sign for environment works func TestEqualSignOnEnv(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() // if the name is eqenv, the service should habe environment for _, service := range p.Data.Services { diff --git a/generator/writer.go b/generator/writer.go index fd61e14..dc35b52 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -18,6 +18,16 @@ import ( var PrefixRE = regexp.MustCompile(`\{\{.*\}\}-?`) +func portExists(port int, ports []types.ServicePortConfig) bool { + for _, p := range ports { + if p.Target == uint32(port) { + log.Println("portExists:", port, p.Target) + return true + } + } + return false +} + func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFile, dirName string) { // make the appname global (yes... ugly but easy) @@ -47,6 +57,10 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi if err != nil { log.Fatal(err) } + if portExists(target, service.Ports) { + continue + } + log.Println("Add port to service:", n, target) service.Ports = append(service.Ports, types.ServicePortConfig{ Target: uint32(target), })