From 3c6b9493db39090371a8c9383183676562b67641 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Sun, 3 Apr 2022 16:08:00 +0200 Subject: [PATCH 01/47] WIP Try to use composer-go - there are problems to resolve with health check - compose/parser.go and compose/types.go should be recheck - need to recheck writers --- compose/parser.go | 212 ++++------------------------------------- compose/parser_test.go | 199 -------------------------------------- generator/main.go | 140 ++++++--------------------- generator/main_test.go | 24 +++-- generator/writer.go | 16 ++-- go.mod | 1 + go.sum | 157 ++++++++++++++++++++++++++++++ helm/deployment.go | 10 +- 8 files changed, 240 insertions(+), 519 deletions(-) delete mode 100644 compose/parser_test.go diff --git a/compose/parser.go b/compose/parser.go index 03d68d5..294fa1e 100644 --- a/compose/parser.go +++ b/compose/parser.go @@ -1,13 +1,12 @@ package compose import ( - "fmt" - "katenary/helm" "log" "os" "strings" - "github.com/google/shlex" + "github.com/compose-spec/compose-go/cli" + "github.com/compose-spec/compose-go/types" "gopkg.in/yaml.v3" ) @@ -17,7 +16,7 @@ const ( // Parser is a docker-compose parser. type Parser struct { - Data *Compose + Data *types.Project } var Appname = "" @@ -44,205 +43,34 @@ func NewParser(filename string, content ...string) *Parser { } } - p := &Parser{Data: c} + p := &Parser{} return p } func (p *Parser) Parse(appname string) { - Appname = appname - services := make(map[string][]string) - // get the service list, to be sure that everything is ok + // Reminder: + // - set Appname + // - loas services - // fix ugly types - for _, s := range p.Data.Services { - parseEnv(s) - parseCommand(s) - parseEnvFiles(s) - parseHealthCheck(s) + options, err := cli.NewProjectOptions(nil, + cli.WithDefaultConfigPath, + cli.WithNormalization(true), + cli.WithInterpolation(true), + cli.WithResolvedPaths(true), + ) + if err != nil { + log.Fatal(err) } - c := p.Data - for name, s := range c.Services { - if portlabel, ok := s.Labels[helm.LABEL_PORT]; ok { - services := strings.Split(portlabel, ",") - for _, serviceport := range services { - portexists := false - for _, found := range s.Ports { - if found == serviceport { - portexists = true - } - } - if !portexists { - s.Ports = append(s.Ports, serviceport) - } - } - } - if len(s.Ports) > 0 { - services[name] = s.Ports - } + proj, err := cli.ProjectFromOptions(options) + if err != nil { + log.Fatal(err) } - // check if dependencies are resolved - missing := []string{} - for name, s := range c.Services { - for _, dep := range s.DependsOn { - if _, ok := services[dep]; !ok { - missing = append(missing, fmt.Sprintf( - "The service \"%s\" hasn't got "+ - "declared port for dependency from \"%s\" - please "+ - "append a %s label or a \"ports\" section in the docker-compose file", - dep, name, helm.LABEL_PORT), - ) - } - } - } + Appname = proj.Name - if len(missing) > 0 { - log.Fatal(strings.Join(missing, "\n")) - } - - // check if all "image" properties are set - missing = []string{} - for name, s := range c.Services { - if s.Image == "" { - missing = append(missing, fmt.Sprintf( - "The service \"%s\" hasn't got "+ - "an image property - please "+ - "append an image property in the docker-compose file", - name, - )) - } - } - if len(missing) > 0 { - log.Fatal(strings.Join(missing, "\n")) - } - - // check the build element - for name, s := range c.Services { - if s.RawBuild == nil { - continue - } - - fmt.Println(ICON_EXCLAMATION + - " \x1b[33myou will need to build and push your image named \"" + s.Image + "\"" + - " for the \"" + name + "\" service \x1b[0m") - - } + p.Data = proj } - -// manage environment variables, if the type is map[string]string so we can use it, else we need to split "=" sign -// and apply this in env variable -func parseEnv(s *Service) { - env := make(map[string]string) - if s.RawEnvironment == nil { - return - } - switch s.RawEnvironment.(type) { - case map[string]string: - env = s.RawEnvironment.(map[string]string) - case map[string]interface{}: - for k, v := range s.RawEnvironment.(map[string]interface{}) { - // force to string - env[k] = fmt.Sprintf("%v", v) - } - case []interface{}: - for _, v := range s.RawEnvironment.([]interface{}) { - // Splot the value of the env variable with "=" - parts := strings.Split(v.(string), "=") - env[parts[0]] = parts[1] - } - case string: - parts := strings.Split(s.RawEnvironment.(string), "=") - env[parts[0]] = parts[1] - default: - log.Printf("%+v, %T", s.RawEnvironment, s.RawEnvironment) - log.Fatal("Environment type not supported") - } - s.Environment = env -} - -func parseCommand(s *Service) { - - if s.RawCommand == nil { - return - } - - // following the command type, it can be a "slice" or a simple sting, so we need to check it - switch v := s.RawCommand.(type) { - case string: - // use shlex to parse the command - command, err := shlex.Split(v) - if err != nil { - log.Fatal(err) - } - s.Command = command - case []string: - s.Command = v - case []interface{}: - for _, v := range v { - s.Command = append(s.Command, v.(string)) - } - default: - log.Printf("%+v %T", s.RawCommand, s.RawCommand) - log.Fatal("Command type not supported") - } -} - -func parseEnvFiles(s *Service) { - // Same than parseEnv, but for env files - if s.RawEnvFiles == nil { - return - } - envfiles := make([]string, 0) - switch v := s.RawEnvFiles.(type) { - case []string: - envfiles = v - case []interface{}: - for _, v := range v { - envfiles = append(envfiles, v.(string)) - } - case string: - envfiles = append(envfiles, v) - default: - log.Printf("%+v %T", s.RawEnvFiles, s.RawEnvFiles) - log.Fatal("EnvFile type not supported") - } - s.EnvFiles = envfiles -} - -func parseHealthCheck(s *Service) { - // HealthCheck command can be a string or slice of strings - if s.HealthCheck == nil { - return - } - if s.HealthCheck.RawTest == nil { - return - } - - switch v := s.HealthCheck.RawTest.(type) { - case string: - c, err := shlex.Split(v) - if err != nil { - log.Fatal(err) - } - s.HealthCheck = &HealthCheck{ - Test: c, - } - - case []string: - s.HealthCheck = &HealthCheck{ - Test: v, - } - - case []interface{}: - for _, v := range v { - s.HealthCheck.Test = append(s.HealthCheck.Test, v.(string)) - } - default: - log.Printf("%+v %T", s.HealthCheck.RawTest, s.HealthCheck.RawTest) - log.Fatal("HealthCheck type not supported") - } -} diff --git a/compose/parser_test.go b/compose/parser_test.go deleted file mode 100644 index 904b00a..0000000 --- a/compose/parser_test.go +++ /dev/null @@ -1,199 +0,0 @@ -package compose - -import ( - "katenary/logger" - "testing" -) - -const DOCKER_COMPOSE_YML1 = ` -version: "3" - -services: - # first service, very basic - web: - image: nginx - ports: - - "80:80" - environment: - FOO: bar - BAZ: qux - networks: - - frontend - - - database: - image: postgres - networks: - - frontend - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: mysecretpassword - POSTGRES_DB: mydb - labels: - katenary.io/ports: "5432" - - commander1: - image: foo - command: ["/bin/sh", "-c", "echo 'hello world'"] - - commander2: - image: foo - command: echo "hello world" - - hc1: - image: foo - healthcheck: - test: ["CMD-SHELL", "echo 'hello world1'"] - - hc2: - image: foo - healthcheck: - test: echo "hello world2" - - hc3: - image: foo - healthcheck: - test: ["CMD", "echo 'hello world3'"] - - -` - -func init() { - logger.NOLOG = true -} - -func TestParser(t *testing.T) { - p := NewParser("", DOCKER_COMPOSE_YML1) - p.Parse("test") - - // check if the "web" and "database" service is parsed correctly - // by checking if the "ports" and "environment" - for name, service := range p.Data.Services { - if name == "web" { - if len(service.Ports) != 1 { - t.Errorf("Expected 1 port, got %d", len(service.Ports)) - } - if service.Ports[0] != "80:80" { - t.Errorf("Expected port 80:80, got %s", service.Ports[0]) - } - if len(service.Environment) != 2 { - t.Errorf("Expected 2 environment variables, got %d", len(service.Environment)) - } - if service.Environment["FOO"] != "bar" { - t.Errorf("Expected FOO=bar, got %s", service.Environment["FOO"]) - } - if service.Environment["BAZ"] != "qux" { - t.Errorf("Expected BAZ=qux, got %s", service.Environment["BAZ"]) - } - } - // same for the "database" service - if name == "database" { - if len(service.Ports) != 1 { - t.Errorf("Expected 1 port, got %d", len(service.Ports)) - } - if service.Ports[0] != "5432" { - t.Errorf("Expected port 5432, got %s", service.Ports[0]) - } - if len(service.Environment) != 3 { - t.Errorf("Expected 3 environment variables, got %d", len(service.Environment)) - } - if service.Environment["POSTGRES_USER"] != "postgres" { - t.Errorf("Expected POSTGRES_USER=postgres, got %s", service.Environment["POSTGRES_USER"]) - } - if service.Environment["POSTGRES_PASSWORD"] != "mysecretpassword" { - t.Errorf("Expected POSTGRES_PASSWORD=mysecretpassword, got %s", service.Environment["POSTGRES_PASSWORD"]) - } - if service.Environment["POSTGRES_DB"] != "mydb" { - t.Errorf("Expected POSTGRES_DB=mydb, got %s", service.Environment["POSTGRES_DB"]) - } - // check labels - if len(service.Labels) != 1 { - t.Errorf("Expected 1 label, got %d", len(service.Labels)) - } - // is label katenary.io/ports correct? - if service.Labels["katenary.io/ports"] != "5432" { - t.Errorf("Expected katenary.io/ports=5432, got %s", service.Labels["katenary.io/ports"]) - } - } - } -} - -func TestParseCommand(t *testing.T) { - p := NewParser("", DOCKER_COMPOSE_YML1) - p.Parse("test") - - for name, s := range p.Data.Services { - if name == "commander1" { - t.Log(s.Command) - if len(s.Command) != 3 { - t.Errorf("Expected 3 command, got %d", len(s.Command)) - } - if s.Command[0] != "/bin/sh" { - t.Errorf("Expected /bin/sh, got %s", s.Command[0]) - } - if s.Command[1] != "-c" { - t.Errorf("Expected -c, got %s", s.Command[1]) - } - if s.Command[2] != "echo 'hello world'" { - t.Errorf("Expected echo 'hello world', got %s", s.Command[2]) - } - } - if name == "commander2" { - t.Log(s.Command) - if len(s.Command) != 2 { - t.Errorf("Expected 1 command, got %d", len(s.Command)) - } - if s.Command[0] != "echo" { - t.Errorf("Expected echo, got %s", s.Command[0]) - } - if s.Command[1] != "hello world" { - t.Errorf("Expected hello world, got %s", s.Command[1]) - } - } - } -} - -func TestHealthChecks(t *testing.T) { - p := NewParser("", DOCKER_COMPOSE_YML1) - p.Parse("test") - - for name, s := range p.Data.Services { - if name != "hc1" && name != "hc2" && name != "hc3" { - continue - } - - if name == "hc1" { - if len(s.HealthCheck.Test) != 2 { - t.Errorf("Expected 2 healthcheck tests, got %d", len(s.HealthCheck.Test)) - } - if s.HealthCheck.Test[0] != "CMD-SHELL" { - t.Errorf("Expected CMD-SHELL, got %s", s.HealthCheck.Test[0]) - } - if s.HealthCheck.Test[1] != "echo 'hello world1'" { - t.Errorf("Expected echo 'hello world1', got %s", s.HealthCheck.Test[1]) - } - } - if name == "hc2" { - if len(s.HealthCheck.Test) != 2 { - t.Errorf("Expected 2 healthcheck tests, got %d", len(s.HealthCheck.Test)) - } - if s.HealthCheck.Test[0] != "echo" { - t.Errorf("Expected echo, got %s", s.HealthCheck.Test[1]) - } - if s.HealthCheck.Test[1] != "hello world2" { - t.Errorf("Expected echo 'hello world2', got %s", s.HealthCheck.Test[1]) - } - } - if name == "hc3" { - if len(s.HealthCheck.Test) != 2 { - t.Errorf("Expected 2 healthcheck tests, got %d", len(s.HealthCheck.Test)) - } - if s.HealthCheck.Test[0] != "CMD" { - t.Errorf("Expected CMD, got %s", s.HealthCheck.Test[0]) - } - if s.HealthCheck.Test[1] != "echo 'hello world3'" { - t.Errorf("Expected echo 'hello world3', got %s", s.HealthCheck.Test[1]) - } - } - } -} diff --git a/generator/main.go b/generator/main.go index e23bd47..0de5bf0 100644 --- a/generator/main.go +++ b/generator/main.go @@ -3,11 +3,9 @@ package generator import ( "fmt" "io/ioutil" - "katenary/compose" "katenary/helm" "katenary/logger" "log" - "net/url" "os" "path/filepath" "strconv" @@ -17,7 +15,7 @@ import ( "errors" - "github.com/google/shlex" + "github.com/compose-spec/compose-go/types" ) var servicesMap = make(map[string]int) @@ -56,14 +54,14 @@ echo "Done" var madeDeployments = make(map[string]helm.Deployment, 0) // Create a Deployment for a given compose.Service. It returns a list of objects: a Deployment and a possible Service (kubernetes represnetation as maps). -func CreateReplicaObject(name string, s *compose.Service, linked map[string]*compose.Service) chan interface{} { +func CreateReplicaObject(name string, s types.ServiceConfig, linked map[string]types.ServiceConfig) chan interface{} { ret := make(chan interface{}, len(s.Ports)+len(s.Expose)+1) go parseService(name, s, linked, ret) return ret } // This function will try to yied deployment and services based on a service from the compose file structure. -func parseService(name string, s *compose.Service, linked map[string]*compose.Service, ret chan interface{}) { +func parseService(name string, s types.ServiceConfig, linked map[string]types.ServiceConfig, ret chan interface{}) { logger.Magenta(ICON_PACKAGE+" Generating deployment for ", name) o := helm.NewDeployment(name) @@ -167,7 +165,7 @@ func parseService(name string, s *compose.Service, linked map[string]*compose.Se } // prepareContainer assigns image, command, env, and labels to a container. -func prepareContainer(container *helm.Container, service *compose.Service, servicename string) { +func prepareContainer(container *helm.Container, service types.ServiceConfig, servicename string) { // if there is no image name, this should fail! if service.Image == "" { log.Fatal(ICON_PACKAGE+" No image name for service ", servicename) @@ -182,22 +180,16 @@ func prepareContainer(container *helm.Container, service *compose.Service, servi } // Create a service (k8s). -func generateServicesAndIngresses(name string, s *compose.Service) []interface{} { +func generateServicesAndIngresses(name string, s types.ServiceConfig) []interface{} { ret := make([]interface{}, 0) // can handle helm.Service or helm.Ingress logger.Magenta(ICON_SERVICE+" Generating service for ", name) ks := helm.NewService(name) for i, p := range s.Ports { - port := strings.Split(p, ":") - src, _ := strconv.Atoi(port[0]) - target := src - if len(port) > 1 { - target, _ = strconv.Atoi(port[1]) - } - ks.Spec.Ports = append(ks.Spec.Ports, helm.NewServicePort(target, target)) + ks.Spec.Ports = append(ks.Spec.Ports, helm.NewServicePort(int(p.Target), int(p.Target))) if i == 0 { - detected(name, target) + detected(name, int(p.Target)) } } ks.Spec.Selector = buildSelector(name, s) @@ -217,7 +209,9 @@ func generateServicesAndIngresses(name string, s *compose.Service) []interface{} logger.Magenta(ICON_SERVICE+" Generating service for ", name+"-external") ks := helm.NewService(name + "-external") ks.Spec.Type = "NodePort" - for _, p := range s.Expose { + for _, expose := range s.Expose { + + p, _ := strconv.Atoi(expose) ks.Spec.Ports = append(ks.Spec.Ports, helm.NewServicePort(p, p)) } ks.Spec.Selector = buildSelector(name, s) @@ -228,7 +222,7 @@ func generateServicesAndIngresses(name string, s *compose.Service) []interface{} } // Create an ingress. -func createIngress(name string, port int, s *compose.Service) *helm.Ingress { +func createIngress(name string, port int, s types.ServiceConfig) *helm.Ingress { ingress := helm.NewIngress(name) Values[name]["ingress"] = map[string]interface{}{ "class": "nginx", @@ -303,7 +297,7 @@ func waitPort(name string) chan int { } // Build the selector for the service. -func buildSelector(name string, s *compose.Service) map[string]string { +func buildSelector(name string, s types.ServiceConfig) map[string]string { return map[string]string{ "katenary.io/component": name, "katenary.io/release": RELEASE_NAME, @@ -344,31 +338,28 @@ func buildCMFromPath(path string) *helm.ConfigMap { } // generateContainerPorts add the container ports of a service. -func generateContainerPorts(s *compose.Service, name string, container *helm.Container) { +func generateContainerPorts(s types.ServiceConfig, name string, container *helm.Container) { exists := make(map[int]string) for _, port := range s.Ports { - _p := strings.Split(port, ":") - port = _p[0] - if len(_p) > 1 { - port = _p[1] - } - portNumber, _ := strconv.Atoi(port) portName := name for _, n := range exists { if name == n { - portName = fmt.Sprintf("%s-%d", name, portNumber) + portName = fmt.Sprintf("%s-%d", name, port.Target) } } container.Ports = append(container.Ports, &helm.ContainerPort{ Name: portName, - ContainerPort: portNumber, + ContainerPort: int(port.Target), }) - exists[portNumber] = name + exists[int(port.Target)] = name } // manage the "expose" section to be a NodePort in Kubernetes - for _, port := range s.Expose { + for _, expose := range s.Expose { + + port, _ := strconv.Atoi(expose) + if _, exist := exists[port]; exist { continue } @@ -380,7 +371,7 @@ func generateContainerPorts(s *compose.Service, name string, container *helm.Con } // prepareVolumes add the volumes of a service. -func prepareVolumes(deployment, name string, s *compose.Service, container *helm.Container, madePVC map[string]bool, ret chan interface{}) []map[string]interface{} { +func prepareVolumes(deployment, name string, s types.ServiceConfig, container *helm.Container, madePVC map[string]bool, ret chan interface{}) []map[string]interface{} { volumes := make([]map[string]interface{}, 0) mountPoints := make([]interface{}, 0) @@ -389,16 +380,10 @@ func prepareVolumes(deployment, name string, s *compose.Service, container *helm configMapsVolumes = strings.Split(v, ",") } - for _, volume := range s.Volumes { + for _, vol := range s.Volumes { - parts := strings.Split(volume, ":") - if len(parts) == 1 { - // this is a volume declaration for Docker only, avoid it - continue - } - - volname := parts[0] - volepath := parts[1] + volname := vol.Source + volepath := vol.Target isCM := false for _, cmVol := range configMapsVolumes { @@ -511,12 +496,12 @@ func prepareVolumes(deployment, name string, s *compose.Service, container *helm } // prepareInitContainers add the init containers of a service. -func prepareInitContainers(name string, s *compose.Service, container *helm.Container) []*helm.Container { +func prepareInitContainers(name string, s types.ServiceConfig, container *helm.Container) []*helm.Container { // We need to detect others services, but we probably not have parsed them yet, so // we will wait for them for a while. initContainers := make([]*helm.Container, 0) - for _, dp := range s.DependsOn { + for dp, _ := range s.DependsOn { c := helm.NewContainer("check-"+dp, "busybox", nil, s.Labels) command := strings.ReplaceAll(strings.TrimSpace(dependScript), "__service__", dp) @@ -549,79 +534,12 @@ func prepareInitContainers(name string, s *compose.Service, container *helm.Cont } // prepareProbes generate http/tcp/command probes for a service. -func prepareProbes(name string, s *compose.Service, container *helm.Container) { +func prepareProbes(name string, s types.ServiceConfig, container *helm.Container) { - // manage the healthcheck property, if any - if s.HealthCheck != nil { - if s.HealthCheck.Interval == "" { - s.HealthCheck.Interval = "10s" - } - interval, err := time.ParseDuration(s.HealthCheck.Interval) - - if err != nil { - log.Fatal(err) - } - if s.HealthCheck.StartPeriod == "" { - s.HealthCheck.StartPeriod = "0s" - } - - initialDelaySeconds, err := time.ParseDuration(s.HealthCheck.StartPeriod) - if err != nil { - log.Fatal(err) - } - - probe := helm.NewProbe(int(interval.Seconds()), int(initialDelaySeconds.Seconds()), 1, s.HealthCheck.Retries) - - healthCheckLabel := s.Labels[helm.LABEL_HEALTHCHECK] - - if healthCheckLabel != "" { - - path := "/" - port := 80 - - u, err := url.Parse(healthCheckLabel) - if err == nil { - path = u.Path - port, _ = strconv.Atoi(u.Port()) - } else { - path = "/" - port = 80 - } - - if strings.HasPrefix(healthCheckLabel, "http://") { - probe.HttpGet = &helm.HttpGet{ - Path: path, - Port: port, - } - } else if strings.HasPrefix(healthCheckLabel, "tcp://") { - if err != nil { - log.Fatal(err) - } - probe.TCP = &helm.TCP{ - Port: port, - } - } else { - c, _ := shlex.Split(healthCheckLabel) - probe.Exec = &helm.Exec{ - - Command: c, - } - } - } else if s.HealthCheck.Test[0] == "CMD" || s.HealthCheck.Test[0] == "CMD-SHELL" { - probe.Exec = &helm.Exec{ - Command: s.HealthCheck.Test[1:], - } - } else { - probe.Exec = &helm.Exec{ - Command: s.HealthCheck.Test, - } - } - container.LivenessProbe = probe - } } // prepareEnvFromFiles generate configMap or secrets from environment files. -func prepareEnvFromFiles(name string, s *compose.Service, container *helm.Container, ret chan interface{}) { +func prepareEnvFromFiles(name string, s types.ServiceConfig, container *helm.Container, ret chan interface{}) { // prepare secrets secretsFiles := make([]string, 0) @@ -630,7 +548,7 @@ func prepareEnvFromFiles(name string, s *compose.Service, container *helm.Contai } // manage environment files (env_file in compose) - for _, envfile := range s.EnvFiles { + for _, envfile := range s.EnvFile { f := strings.ReplaceAll(envfile, "_", "-") f = strings.ReplaceAll(f, ".env", "") f = strings.ReplaceAll(f, ".", "") diff --git a/generator/main_test.go b/generator/main_test.go index 583f0b0..dcd4bfc 100644 --- a/generator/main_test.go +++ b/generator/main_test.go @@ -117,7 +117,8 @@ func TestCommand(t *testing.T) { tmp, p := setUp(t) defer os.RemoveAll(tmp) - for name := range p.Data.Services { + for _, service := range p.Data.Services { + name := service.Name if name == "web2" { // Ensure that the command is correctly set // The command should be a string array @@ -161,7 +162,8 @@ func TestEnvs(t *testing.T) { tmp, p := setUp(t) defer os.RemoveAll(tmp) - for name := range p.Data.Services { + for _, service := range p.Data.Services { + name := service.Name if name == "php" { // the "DB_HOST" environment variable inside the template must be set to '{{ .Release.Name }}-database' @@ -197,7 +199,8 @@ func TestSamePod(t *testing.T) { tmp, p := setUp(t) defer os.RemoveAll(tmp) - for name, service := range p.Data.Services { + for _, service := range p.Data.Services { + name := service.Name path := filepath.Join(tmp, "templates", name+".deployment.yaml") if _, found := service.Labels[helm.LABEL_SAMEPOD]; found { @@ -222,7 +225,8 @@ func TestPorts(t *testing.T) { tmp, p := setUp(t) defer os.RemoveAll(tmp) - for name, service := range p.Data.Services { + for _, service := range p.Data.Services { + name := service.Name path := "" // if the service has a port found in helm.LABEL_PORT or ports, so the service file should exist @@ -249,7 +253,8 @@ func TestPVC(t *testing.T) { tmp, p := setUp(t) defer os.RemoveAll(tmp) - for name := range p.Data.Services { + for _, service := range p.Data.Services { + name := service.Name path := filepath.Join(tmp, "templates", name+"-data.pvc.yaml") // the "database" service should have a pvc file in templates (name-data.pvc.yaml) @@ -269,7 +274,8 @@ func TestIngress(t *testing.T) { tmp, p := setUp(t) defer os.RemoveAll(tmp) - for name := range p.Data.Services { + for _, service := range p.Data.Services { + name := service.Name path := filepath.Join(tmp, "templates", name+".ingress.yaml") // the "web" service should have a ingress file in templates (name.ingress.yaml) @@ -289,7 +295,8 @@ func TestUnmappedVolumes(t *testing.T) { tmp, p := setUp(t) defer os.RemoveAll(tmp) - for name := range p.Data.Services { + for _, service := range p.Data.Services { + name := service.Name if name == "novol" { path := filepath.Join(tmp, "templates", name+".deployment.yaml") fp, _ := os.Open(path) @@ -310,7 +317,8 @@ func TestEqualSignOnEnv(t *testing.T) { defer os.RemoveAll(tmp) // if the name is eqenv, the service should habe environment - for name, _ := range p.Data.Services { + for _, service := range p.Data.Services { + name := service.Name if name == "eqenv" { path := filepath.Join(tmp, "templates", name+".deployment.yaml") fp, _ := os.Open(path) diff --git a/generator/writer.go b/generator/writer.go index 01f5e14..c0e40e5 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/compose-spec/compose-go/types" "gopkg.in/yaml.v3" ) @@ -32,13 +33,15 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi // list avoided services avoids := make(map[string]bool) - for n, service := range p.Data.Services { + for _, service := range p.Data.Services { + n := service.Name if _, ok := service.Labels[helm.LABEL_SAMEPOD]; ok { avoids[n] = true } } - for name, s := range p.Data.Services { + for _, s := range p.Data.Services { + name := s.Name // Manage emptyDir volumes if empty, ok := s.Labels[helm.LABEL_EMPTYDIRS]; ok { @@ -49,11 +52,12 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi } // fetch corresponding service in "links" - linked := make(map[string]*compose.Service, 0) + linked := make(map[string]types.ServiceConfig, 0) // find service linked to this one - for n, service := range p.Data.Services { - if _, ok := service.Labels[helm.LABEL_SAMEPOD]; ok { - if service.Labels[helm.LABEL_SAMEPOD] == name { + for _, service := range p.Data.Services { + n := service.Name + for _, label := range service.Labels { + if label == helm.LABEL_SAMEPOD { linked[n] = service } } diff --git a/go.mod b/go.mod index edf4df7..f5e777e 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module katenary go 1.16 require ( + github.com/compose-spec/compose-go v1.2.2 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/kr/pretty v0.2.0 // indirect github.com/spf13/cobra v1.4.0 diff --git a/go.sum b/go.sum index 75b2588..1638ead 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,191 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/aws/aws-sdk-go v1.34.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/compose-spec/compose-go v1.2.2 h1:y1dwl3KUTBnWPVur6EZno9zUIum6Q87/F5keljnGQB4= +github.com/compose-spec/compose-go v1.2.2/go.mod h1:pAy7Mikpeft4pxkFU565/DRHEbDfR84G6AQuiL+Hdg8= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= +github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/distribution/distribution/v3 v3.0.0-20210316161203-a01c71e2477e h1:n81KvOMrLZa+VWHwST7dun9f0G98X3zREHS1ztYzZKU= +github.com/distribution/distribution/v3 v3.0.0-20210316161203-a01c71e2477e/go.mod h1:xpWTC2KnJMiDLkoawhsPQcXjvwATEBcbq0xevG2YR9M= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= +github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= +google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= +gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ= diff --git a/helm/deployment.go b/helm/deployment.go index 9d6f28b..172871d 100644 --- a/helm/deployment.go +++ b/helm/deployment.go @@ -1,6 +1,10 @@ package helm -import "strings" +import ( + "strings" + + "github.com/compose-spec/compose-go/types" +) // Deployment is a k8s deployment. type Deployment struct { @@ -82,7 +86,7 @@ func NewProbe(period, initialDelaySeconds, success, failure int) *Probe { } } -func NewContainer(name, image string, environment, labels map[string]string) *Container { +func NewContainer(name, image string, environment types.MappingWithEquals, labels map[string]string) *Container { container := &Container{ Image: image, Name: name, @@ -100,7 +104,7 @@ func NewContainer(name, image string, environment, labels map[string]string) *Co for n, v := range environment { for _, name := range toServices { if name == n { - v = RELEASE_NAME + "-" + v + *v = RELEASE_NAME + "-" + *v } } container.Env[idx] = Value{Name: n, Value: v} -- 2.49.1 From cc73d57f5af0d9d7534a749e286371169e0c5a5e Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Sun, 3 Apr 2022 16:09:33 +0200 Subject: [PATCH 02/47] better loop range --- generator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/main.go b/generator/main.go index 0de5bf0..14c56df 100644 --- a/generator/main.go +++ b/generator/main.go @@ -501,7 +501,7 @@ func prepareInitContainers(name string, s types.ServiceConfig, container *helm.C // We need to detect others services, but we probably not have parsed them yet, so // we will wait for them for a while. initContainers := make([]*helm.Container, 0) - for dp, _ := range s.DependsOn { + for dp := range s.DependsOn { c := helm.NewContainer("check-"+dp, "busybox", nil, s.Labels) command := strings.ReplaceAll(strings.TrimSpace(dependScript), "__service__", dp) -- 2.49.1 From e346bb3ae43b16cd59d31d72dc46fb372b5b8dbc Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Mon, 4 Apr 2022 09:53:36 +0200 Subject: [PATCH 03/47] WIP - there still are problem with several things - ports label are not well managed - no heathcheck, have to be rewritten - many problem with generated compose file from memory --- cmd/utils.go | 2 +- compose/parser.go | 51 +++++++++++++-------- compose/types.go | 44 ------------------ generator/main.go | 102 +++++++++-------------------------------- generator/main_test.go | 13 ++++-- generator/writer.go | 31 +++++++++---- 6 files changed, 87 insertions(+), 156 deletions(-) delete mode 100644 compose/types.go diff --git a/cmd/utils.go b/cmd/utils.go index 308eeed..6eaeda2 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -97,7 +97,7 @@ func Convert(composeFile, appVersion, appName, chartDir string, force bool) { fmt.Println("No compose file given") return } - _, err := os.Stat(ComposeFile) + _, err := os.Stat(composeFile) if err != nil { fmt.Println("No compose file found") os.Exit(1) diff --git a/compose/parser.go b/compose/parser.go index 294fa1e..7538d7f 100644 --- a/compose/parser.go +++ b/compose/parser.go @@ -3,11 +3,10 @@ package compose import ( "log" "os" - "strings" + "path/filepath" "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/types" - "gopkg.in/yaml.v3" ) const ( @@ -16,7 +15,8 @@ const ( // Parser is a docker-compose parser. type Parser struct { - Data *types.Project + Data *types.Project + temporary *string } var Appname = "" @@ -24,30 +24,44 @@ var Appname = "" // NewParser create a Parser and parse the file given in filename. If filename is empty, we try to parse the content[0] argument that should be a valid YAML content. func NewParser(filename string, content ...string) *Parser { - c := NewCompose() - if filename != "" { - f, err := os.Open(filename) + p := &Parser{} + + if len(content) > 0 { + //write it in a temporary file + tmp, err := os.MkdirTemp(os.TempDir(), "katenary-") if err != nil { log.Fatal(err) } - dec := yaml.NewDecoder(f) - err = dec.Decode(c) + tmpfile, err := os.Create(filepath.Join(tmp, "tmp.yml")) if err != nil { log.Fatal(err) } - } else { - dec := yaml.NewDecoder(strings.NewReader(content[0])) - err := dec.Decode(c) - if err != nil { - log.Fatal(err) + tmpfile.WriteString(content[0]) + tmpfile.Close() + filename = tmpfile.Name() + p.temporary = &tmp + cli.DefaultFileNames = append([]string{filename}, cli.DefaultFileNames...) + } + // if filename is not in cli Default files, add it + if len(filename) > 0 { + found := false + for _, f := range cli.DefaultFileNames { + if f == filename { + found = true + break + } + } + // add the file at first position + if !found { + cli.DefaultFileNames = append([]string{filename}, cli.DefaultFileNames...) } } - - p := &Parser{} + log.Println(cli.DefaultFileNames) return p } +// Parse using compose-go parser, adapt a bit the Project and set Appname. func (p *Parser) Parse(appname string) { // Reminder: @@ -66,11 +80,12 @@ func (p *Parser) Parse(appname string) { proj, err := cli.ProjectFromOptions(options) if err != nil { - log.Fatal(err) + log.Fatal("Failed to create project", err) } Appname = proj.Name - p.Data = proj - + if p.temporary != nil { + defer os.RemoveAll(*p.temporary) + } } diff --git a/compose/types.go b/compose/types.go deleted file mode 100644 index f9c6238..0000000 --- a/compose/types.go +++ /dev/null @@ -1,44 +0,0 @@ -package compose - -// Compose is a complete docker-compse representation. -type Compose struct { - Version string `yaml:"version"` - Services map[string]*Service `yaml:"services"` - Volumes map[string]interface{} `yaml:"volumes"` -} - -// NewCompose resturs a Compose object. -func NewCompose() *Compose { - c := &Compose{} - c.Services = make(map[string]*Service) - c.Volumes = make(map[string]interface{}) - return c -} - -// HealthCheck manage generic type to handle TCP, HTTP and TCP health check. -type HealthCheck struct { - Test []string `yaml:"-"` - RawTest interface{} `yaml:"test"` - Interval string `yaml:"interval"` - Timeout string `yaml:"timeout"` - Retries int `yaml:"retries"` - StartPeriod string `yaml:"start_period"` -} - -// Service represent a "service" in a docker-compose file. -type Service struct { - Image string `yaml:"image"` - Ports []string `yaml:"ports"` - Environment map[string]string `yaml:"-"` - RawEnvironment interface{} `yaml:"environment"` - Labels map[string]string `yaml:"labels"` - DependsOn []string `yaml:"depends_on"` - Volumes []string `yaml:"volumes"` - Expose []int `yaml:"expose"` - EnvFiles []string `yaml:"-"` - RawEnvFiles interface{} `yaml:"env_file"` - RawBuild interface{} `yaml:"build"` - HealthCheck *HealthCheck `yaml:"healthcheck"` - Command []string `yaml:"-"` - RawCommand interface{} `yaml:"command"` -} diff --git a/generator/main.go b/generator/main.go index 14c56df..a013df6 100644 --- a/generator/main.go +++ b/generator/main.go @@ -11,9 +11,6 @@ import ( "strconv" "strings" "sync" - "time" - - "errors" "github.com/compose-spec/compose-go/types" ) @@ -123,33 +120,6 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se } } - // Special case, it there is no "ports", so there is no associated services... - // But... some other deployment can wait for it, so we alert that this deployment hasn't got any - // associated service. - if len(s.Ports) == 0 { - // alert any current or **future** waiters that this service is not exposed - go func() { - defer func() { - // recover from panic - if r := recover(); r != nil { - // log the stack trace - fmt.Println(r) - } - }() - for { - select { - case <-time.Tick(1 * time.Millisecond): - locker.Lock() - for _, c := range serviceWaiters[name] { - c <- -1 - close(c) - } - locker.Unlock() - } - } - }() - } - // add the volumes in Values if len(VolumeValues[name]) > 0 { locker.Lock() @@ -166,6 +136,8 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se // prepareContainer assigns image, command, env, and labels to a container. func prepareContainer(container *helm.Container, service types.ServiceConfig, servicename string) { + locker.Lock() + defer locker.Unlock() // if there is no image name, this should fail! if service.Image == "" { log.Fatal(ICON_PACKAGE+" No image name for service ", servicename) @@ -186,11 +158,8 @@ func generateServicesAndIngresses(name string, s types.ServiceConfig) []interfac logger.Magenta(ICON_SERVICE+" Generating service for ", name) ks := helm.NewService(name) - for i, p := range s.Ports { + for _, p := range s.Ports { ks.Spec.Ports = append(ks.Spec.Ports, helm.NewServicePort(int(p.Target), int(p.Target))) - if i == 0 { - detected(name, int(p.Target)) - } } ks.Spec.Selector = buildSelector(name, s) @@ -253,49 +222,6 @@ func createIngress(name string, port int, s types.ServiceConfig) *helm.Ingress { return ingress } -// This function is called when a possible service is detected, it append the port in a map to make others -// to be able to get the service name. It also try to send the data to any "waiter" for this service. -func detected(name string, port int) { - locker.Lock() - defer locker.Unlock() - if _, ok := servicesMap[name]; ok { - return - } - servicesMap[name] = port - go func() { - locker.Lock() - defer locker.Unlock() - if cx, ok := serviceWaiters[name]; ok { - for _, c := range cx { - c <- port - } - } - }() -} - -func getPort(name string) (int, error) { - if v, ok := servicesMap[name]; ok { - return v, nil - } - return -1, errors.New("Not found") -} - -// Waits for a service to be discovered. Sometimes, a deployment depends on another one. See the detected() function. -func waitPort(name string) chan int { - locker.Lock() - defer locker.Unlock() - c := make(chan int, 0) - serviceWaiters[name] = append(serviceWaiters[name], c) - go func() { - locker.Lock() - defer locker.Unlock() - if v, ok := servicesMap[name]; ok { - c <- v - } - }() - return c -} - // Build the selector for the service. func buildSelector(name string, s types.ServiceConfig) map[string]string { return map[string]string{ @@ -355,6 +281,20 @@ func generateContainerPorts(s types.ServiceConfig, name string, container *helm. exists[int(port.Target)] = name } + // Get ports from label + if v, ok := s.Labels[helm.LABEL_PORT]; ok { + // split port by "," + ports := strings.Split(v, ",") + for _, port := range ports { + port, _ := strconv.Atoi(port) + container.Ports = append(container.Ports, &helm.ContainerPort{ + Name: name, + ContainerPort: port, + }) + exists[port] = name + } + } + // manage the "expose" section to be a NodePort in Kubernetes for _, expose := range s.Expose { @@ -506,9 +446,11 @@ func prepareInitContainers(name string, s types.ServiceConfig, container *helm.C command := strings.ReplaceAll(strings.TrimSpace(dependScript), "__service__", dp) foundPort := -1 - if defaultPort, err := getPort(dp); err != nil { - // BUG: Sometimes the chan remains opened - foundPort = <-waitPort(dp) + locker.Lock() + defer locker.Unlock() + if defaultPort, ok := servicesMap[dp]; !ok { + logger.Redf("Error while getting port for service %s\n", dp) + os.Exit(1) } else { foundPort = defaultPort } diff --git a/generator/main_test.go b/generator/main_test.go index dcd4bfc..df43dd0 100644 --- a/generator/main_test.go +++ b/generator/main_test.go @@ -5,10 +5,13 @@ import ( "katenary/compose" "katenary/helm" "katenary/logger" + "log" "os" "path/filepath" "strings" "testing" + + "github.com/compose-spec/compose-go/cli" ) const DOCKER_COMPOSE_YML = `version: '3' @@ -92,17 +95,19 @@ volumes: driver: local ` +var defaultCliFiles = cli.DefaultFileNames + func init() { logger.NOLOG = true } func setUp(t *testing.T) (string, *compose.Parser) { + cli.DefaultFileNames = defaultCliFiles p := compose.NewParser("", DOCKER_COMPOSE_YML) p.Parse("testapp") // create a temporary directory tmp, err := os.MkdirTemp(os.TempDir(), "katenary-test") - t.Log("Generated ", tmp, "directory") if err != nil { t.Fatal(err) } @@ -115,7 +120,8 @@ func setUp(t *testing.T) (string, *compose.Parser) { // Check if the web2 service has got a command. func TestCommand(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + //defer os.RemoveAll(tmp) + log.Println(tmp) for _, service := range p.Data.Services { name := service.Name @@ -138,6 +144,7 @@ func TestCommand(t *testing.T) { commands = append(commands, line) } } + //log.Printf("%#v\n", commands) ok := 0 for _, command := range commands { if strings.Contains(command, "- /bin/sh") { @@ -242,7 +249,7 @@ func TestPorts(t *testing.T) { t.Log("Checking ", name, " service file") _, err := os.Stat(path) if err != nil { - t.Fatal(err) + t.Error(err) } } } diff --git a/generator/writer.go b/generator/writer.go index c0e40e5..55a0425 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -4,6 +4,7 @@ import ( "katenary/compose" "katenary/generator/writers" "katenary/helm" + "log" "os" "path/filepath" "regexp" @@ -26,33 +27,38 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi // try to create the directory err := os.MkdirAll(templatesDir, 0755) if err != nil { - panic(err) + log.Fatal(err) } files := make(map[string]chan interface{}) - // list avoided services + // Manage services, avoid linked pods and store all services port in servicesMap avoids := make(map[string]bool) + linked := make(map[string]types.ServiceConfig, 0) for _, service := range p.Data.Services { n := service.Name + + // find port and store it in servicesMap + for _, port := range service.Ports { + target := int(port.Target) + if target != 0 { + servicesMap[n] = target + } + } + + // avoid linked pods if _, ok := service.Labels[helm.LABEL_SAMEPOD]; ok { avoids[n] = true } - } - for _, s := range p.Data.Services { - name := s.Name - - // Manage emptyDir volumes - if empty, ok := s.Labels[helm.LABEL_EMPTYDIRS]; ok { + // manage emptyDir volumes + if empty, ok := service.Labels[helm.LABEL_EMPTYDIRS]; ok { //split empty list by coma emptyDirs := strings.Split(empty, ",") //append them in EmptyDirs EmptyDirs = append(EmptyDirs, emptyDirs...) } - // fetch corresponding service in "links" - linked := make(map[string]types.ServiceConfig, 0) // find service linked to this one for _, service := range p.Data.Services { n := service.Name @@ -62,6 +68,11 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi } } } + } + + // for all services in linked map, and not in avoids map, generate the service + for _, s := range p.Data.Services { + name := s.Name if _, found := avoids[name]; found { continue -- 2.49.1 From 30a79998ab87594e1e90bef50f1d37ab74ff9900 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Mon, 4 Apr 2022 13:52:28 +0200 Subject: [PATCH 04/47] Fix label ports --- compose/parser.go | 3 +-- generator/main.go | 43 ++++++++++++++++++++++++++---------------- generator/main_test.go | 6 ++---- generator/writer.go | 38 ++++++++++++++++++++++++++----------- go.mod | 1 - go.sum | 2 -- 6 files changed, 57 insertions(+), 36 deletions(-) diff --git a/compose/parser.go b/compose/parser.go index 7538d7f..427176a 100644 --- a/compose/parser.go +++ b/compose/parser.go @@ -40,7 +40,7 @@ func NewParser(filename string, content ...string) *Parser { tmpfile.Close() filename = tmpfile.Name() p.temporary = &tmp - cli.DefaultFileNames = append([]string{filename}, cli.DefaultFileNames...) + cli.DefaultFileNames = []string{filename} } // if filename is not in cli Default files, add it if len(filename) > 0 { @@ -56,7 +56,6 @@ func NewParser(filename string, content ...string) *Parser { cli.DefaultFileNames = append([]string{filename}, cli.DefaultFileNames...) } } - log.Println(cli.DefaultFileNames) return p } diff --git a/generator/main.go b/generator/main.go index a013df6..1afb380 100644 --- a/generator/main.go +++ b/generator/main.go @@ -15,10 +15,6 @@ import ( "github.com/compose-spec/compose-go/types" ) -var servicesMap = make(map[string]int) -var serviceWaiters = make(map[string][]chan int) -var locker = &sync.Mutex{} - const ( ICON_PACKAGE = "📦" ICON_SERVICE = "🔌" @@ -33,11 +29,15 @@ const ( ) // Values is kept in memory to create a values.yaml file. -var Values = make(map[string]map[string]interface{}) -var VolumeValues = make(map[string]map[string]map[string]interface{}) -var EmptyDirs = []string{} +var ( + Values = make(map[string]map[string]interface{}) + VolumeValues = make(map[string]map[string]map[string]interface{}) + EmptyDirs = []string{} + servicesMap = make(map[string]int) + serviceWaiters = make(map[string][]chan int) + locker = &sync.Mutex{} -var dependScript = ` + dependScript = ` OK=0 echo "Checking __service__ port" while [ $OK != 1 ]; do @@ -48,11 +48,12 @@ echo echo "Done" ` -var madeDeployments = make(map[string]helm.Deployment, 0) + madeDeployments = make(map[string]helm.Deployment, 0) +) // Create a Deployment for a given compose.Service. It returns a list of objects: a Deployment and a possible Service (kubernetes represnetation as maps). func CreateReplicaObject(name string, s types.ServiceConfig, linked map[string]types.ServiceConfig) chan interface{} { - ret := make(chan interface{}, len(s.Ports)+len(s.Expose)+1) + ret := make(chan interface{}, len(s.Ports)+len(s.Expose)+2) go parseService(name, s, linked, ret) return ret } @@ -136,17 +137,17 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se // prepareContainer assigns image, command, env, and labels to a container. func prepareContainer(container *helm.Container, service types.ServiceConfig, servicename string) { - locker.Lock() - defer locker.Unlock() // if there is no image name, this should fail! if service.Image == "" { log.Fatal(ICON_PACKAGE+" No image name for service ", servicename) } container.Image = "{{ .Values." + servicename + ".image }}" container.Command = service.Command + locker.Lock() Values[servicename] = map[string]interface{}{ "image": service.Image, } + locker.Unlock() prepareProbes(servicename, service, container) generateContainerPorts(service, servicename, container) } @@ -159,7 +160,8 @@ func generateServicesAndIngresses(name string, s types.ServiceConfig) []interfac ks := helm.NewService(name) for _, p := range s.Ports { - ks.Spec.Ports = append(ks.Spec.Ports, helm.NewServicePort(int(p.Target), int(p.Target))) + target := int(p.Target) + ks.Spec.Ports = append(ks.Spec.Ports, helm.NewServicePort(target, target)) } ks.Spec.Selector = buildSelector(name, s) @@ -286,7 +288,10 @@ func generateContainerPorts(s types.ServiceConfig, name string, container *helm. // split port by "," ports := strings.Split(v, ",") for _, port := range ports { - port, _ := strconv.Atoi(port) + port, err := strconv.Atoi(port) + if err != nil { + log.Fatalf("The given port \"%v\" as container port in \"%s\" service is not an integer\n", v, name) + } container.Ports = append(container.Ports, &helm.ContainerPort{ Name: name, ContainerPort: port, @@ -343,7 +348,13 @@ func prepareVolumes(deployment, name string, s types.ServiceConfig, container *h } if isCM { // check if the volname path points on a file, if so, we need to add subvolume to the interface - stat, _ := os.Stat(volname) + stat, err := os.Stat(volname) + if err != nil { + logger.ActivateColors = true + logger.Redf("An error occured reading volume path %s\n", err.Error()) + logger.ActivateColors = false + continue + } pointToFile := "" if !stat.IsDir() { pointToFile = filepath.Base(volname) @@ -447,13 +458,13 @@ func prepareInitContainers(name string, s types.ServiceConfig, container *helm.C foundPort := -1 locker.Lock() - defer locker.Unlock() if defaultPort, ok := servicesMap[dp]; !ok { logger.Redf("Error while getting port for service %s\n", dp) os.Exit(1) } else { foundPort = defaultPort } + locker.Unlock() if foundPort == -1 { log.Fatalf( "ERROR, the %s service is waiting for %s port number, "+ diff --git a/generator/main_test.go b/generator/main_test.go index df43dd0..2cde4a1 100644 --- a/generator/main_test.go +++ b/generator/main_test.go @@ -5,7 +5,6 @@ import ( "katenary/compose" "katenary/helm" "katenary/logger" - "log" "os" "path/filepath" "strings" @@ -98,7 +97,7 @@ volumes: var defaultCliFiles = cli.DefaultFileNames func init() { - logger.NOLOG = true + logger.NOLOG = false } func setUp(t *testing.T) (string, *compose.Parser) { @@ -120,8 +119,7 @@ func setUp(t *testing.T) (string, *compose.Parser) { // Check if the web2 service has got a command. func TestCommand(t *testing.T) { tmp, p := setUp(t) - //defer os.RemoveAll(tmp) - log.Println(tmp) + defer os.RemoveAll(tmp) for _, service := range p.Data.Services { name := service.Name diff --git a/generator/writer.go b/generator/writer.go index 55a0425..fd61e14 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "regexp" + "strconv" "strings" "time" @@ -34,15 +35,29 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi // Manage services, avoid linked pods and store all services port in servicesMap avoids := make(map[string]bool) - linked := make(map[string]types.ServiceConfig, 0) - for _, service := range p.Data.Services { + for i, service := range p.Data.Services { n := service.Name + if ports, ok := service.Labels[helm.LABEL_PORT]; ok { + if service.Ports == nil { + service.Ports = make([]types.ServicePortConfig, 0) + } + for _, port := range strings.Split(ports, ",") { + target, err := strconv.Atoi(port) + if err != nil { + log.Fatal(err) + } + service.Ports = append(service.Ports, types.ServicePortConfig{ + Target: uint32(target), + }) + } + } // find port and store it in servicesMap for _, port := range service.Ports { target := int(port.Target) if target != 0 { servicesMap[n] = target + break } } @@ -58,16 +73,8 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi //append them in EmptyDirs EmptyDirs = append(EmptyDirs, emptyDirs...) } + p.Data.Services[i] = service - // find service linked to this one - for _, service := range p.Data.Services { - n := service.Name - for _, label := range service.Labels { - if label == helm.LABEL_SAMEPOD { - linked[n] = service - } - } - } } // for all services in linked map, and not in avoids map, generate the service @@ -77,6 +84,15 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi if _, found := avoids[name]; found { continue } + linked := make(map[string]types.ServiceConfig, 0) + // find service + for _, service := range p.Data.Services { + n := service.Name + if linkname, ok := service.Labels[helm.LABEL_SAMEPOD]; ok && linkname == name { + linked[n] = service + } + } + files[name] = CreateReplicaObject(name, s, linked) } diff --git a/go.mod b/go.mod index f5e777e..4e007b6 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.16 require ( github.com/compose-spec/compose-go v1.2.2 - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/kr/pretty v0.2.0 // indirect github.com/spf13/cobra v1.4.0 golang.org/x/mod v0.5.1 diff --git a/go.sum b/go.sum index 1638ead..ac3e4ea 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,6 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -- 2.49.1 From 8f3a9aedcd65b014c5c878de2ce6a1fea612f6b2 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Mon, 4 Apr 2022 13:53:13 +0200 Subject: [PATCH 05/47] Remove noisy logs --- generator/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/main_test.go b/generator/main_test.go index 2cde4a1..50caf52 100644 --- a/generator/main_test.go +++ b/generator/main_test.go @@ -97,7 +97,7 @@ volumes: var defaultCliFiles = cli.DefaultFileNames func init() { - logger.NOLOG = false + logger.NOLOG = true } func setUp(t *testing.T) (string, *compose.Parser) { -- 2.49.1 From 5e8cf47ab6f12eb5a3482381b21994bfb65e9890 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Tue, 5 Apr 2022 08:05:33 +0200 Subject: [PATCH 06/47] WIP try to fix race condition and tests --- compose/parser.go | 30 +++++++++---------- generator/main.go | 51 ++++++++++++++++++++++---------- generator/main_test.go | 67 ++++++++++++++++++++++++++++++++++-------- generator/writer.go | 14 +++++++++ 4 files changed, 118 insertions(+), 44 deletions(-) diff --git a/compose/parser.go b/compose/parser.go index 427176a..ca95fd7 100644 --- a/compose/parser.go +++ b/compose/parser.go @@ -1,6 +1,7 @@ package compose import ( + "io/ioutil" "log" "os" "path/filepath" @@ -19,27 +20,24 @@ type Parser struct { temporary *string } -var Appname = "" +var ( + Appname = "" + CURRENT_DIR, _ = os.Getwd() +) // NewParser create a Parser and parse the file given in filename. If filename is empty, we try to parse the content[0] argument that should be a valid YAML content. func NewParser(filename string, content ...string) *Parser { p := &Parser{} - if len(content) > 0 { - //write it in a temporary file - tmp, err := os.MkdirTemp(os.TempDir(), "katenary-") + if len(content) > 0 { // mainly for the tests... + dir := filepath.Dir(filename) + err := os.MkdirAll(dir, 0755) if err != nil { log.Fatal(err) } - tmpfile, err := os.Create(filepath.Join(tmp, "tmp.yml")) - if err != nil { - log.Fatal(err) - } - tmpfile.WriteString(content[0]) - tmpfile.Close() - filename = tmpfile.Name() - p.temporary = &tmp + p.temporary = &dir + ioutil.WriteFile(filename, []byte(content[0]), 0644) cli.DefaultFileNames = []string{filename} } // if filename is not in cli Default files, add it @@ -84,7 +82,9 @@ func (p *Parser) Parse(appname string) { Appname = proj.Name p.Data = proj - if p.temporary != nil { - defer os.RemoveAll(*p.temporary) - } + CURRENT_DIR = p.Data.WorkingDir +} + +func GetCurrentDir() string { + return CURRENT_DIR } diff --git a/generator/main.go b/generator/main.go index 1afb380..95c0285 100644 --- a/generator/main.go +++ b/generator/main.go @@ -3,6 +3,7 @@ package generator import ( "fmt" "io/ioutil" + "katenary/compose" "katenary/helm" "katenary/logger" "log" @@ -123,9 +124,7 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se // add the volumes in Values if len(VolumeValues[name]) > 0 { - locker.Lock() - Values[name]["persistence"] = VolumeValues[name] - locker.Unlock() + AddValues(name, map[string]interface{}{"persistence": VolumeValues[name]}) } // the deployment is ready, give it @@ -143,11 +142,7 @@ func prepareContainer(container *helm.Container, service types.ServiceConfig, se } container.Image = "{{ .Values." + servicename + ".image }}" container.Command = service.Command - locker.Lock() - Values[servicename] = map[string]interface{}{ - "image": service.Image, - } - locker.Unlock() + AddValues(servicename, map[string]interface{}{"image": service.Image}) prepareProbes(servicename, service, container) generateContainerPorts(service, servicename, container) } @@ -195,11 +190,14 @@ func generateServicesAndIngresses(name string, s types.ServiceConfig) []interfac // Create an ingress. func createIngress(name string, port int, s types.ServiceConfig) *helm.Ingress { ingress := helm.NewIngress(name) - Values[name]["ingress"] = map[string]interface{}{ + + ingressVal := map[string]interface{}{ "class": "nginx", "host": name + "." + helm.Appname + ".tld", "enabled": false, } + AddValues(name, map[string]interface{}{"ingress": ingressVal}) + ingress.Spec.Rules = []helm.IngressRule{ { Host: fmt.Sprintf("{{ .Values.%s.ingress.host }}", name), @@ -425,15 +423,10 @@ func prepareVolumes(deployment, name string, s types.ServiceConfig, container *h }) logger.Yellow(ICON_STORE+" Generate volume values", volname, "for container named", name, "in deployment", deployment) - locker.Lock() - if _, ok := VolumeValues[deployment]; !ok { - VolumeValues[deployment] = make(map[string]map[string]interface{}) - } - VolumeValues[deployment][volname] = map[string]interface{}{ + AddVolumeValues(deployment, volname, map[string]interface{}{ "enabled": false, "capacity": "1Gi", - } - locker.Unlock() + }) if _, ok := madePVC[deployment+volname]; !ok { madePVC[deployment+volname] = true @@ -521,6 +514,8 @@ func prepareEnvFromFiles(name string, s types.ServiceConfig, container *helm.Con logger.Bluef(ICON_SECRET+" Generating secret %s\n", cf) store = helm.NewSecret(cf) } + + envfile = filepath.Join(compose.GetCurrentDir(), envfile) if err := store.AddEnvFile(envfile); err != nil { logger.ActivateColors = true logger.Red(err.Error()) @@ -542,3 +537,27 @@ func prepareEnvFromFiles(name string, s types.ServiceConfig, container *helm.Con ret <- store } } + +func AddValues(servicename string, values map[string]interface{}) { + locker.Lock() + defer locker.Unlock() + + if _, ok := values[servicename]; !ok { + Values[servicename] = make(map[string]interface{}) + } + + for k, v := range values { + Values[servicename][k] = v + } + +} + +func AddVolumeValues(deployment string, volname string, values map[string]interface{}) { + locker.Lock() + defer locker.Unlock() + + if _, ok := VolumeValues[deployment]; !ok { + VolumeValues[deployment] = make(map[string]map[string]interface{}) + } + VolumeValues[deployment][volname] = values +} diff --git a/generator/main_test.go b/generator/main_test.go index 50caf52..080591a 100644 --- a/generator/main_test.go +++ b/generator/main_test.go @@ -89,37 +89,79 @@ services: - SOME_ENV_VAR=some_value - ANOTHER_ENV_VAR=another_value + # use environment file + useenvfile: + image: nginx + env_file: + - config/env + volumes: data: driver: local ` var defaultCliFiles = cli.DefaultFileNames +var TMP_DIR = "" +var TMPWORK_DIR = "" func init() { - logger.NOLOG = true + logger.NOLOG = len(os.Getenv("NOLOG")) < 1 } func setUp(t *testing.T) (string, *compose.Parser) { cli.DefaultFileNames = defaultCliFiles - p := compose.NewParser("", DOCKER_COMPOSE_YML) - p.Parse("testapp") // create a temporary directory - tmp, err := os.MkdirTemp(os.TempDir(), "katenary-test") + tmp, err := os.MkdirTemp(os.TempDir(), "katenary-test-") if err != nil { t.Fatal(err) } + tmpwork, err := os.MkdirTemp(os.TempDir(), "katenary-test-work-") + if err != nil { + t.Fatal(err) + } + + composefile := filepath.Join(tmpwork, "docker-compose.yaml") + p := compose.NewParser(composefile, DOCKER_COMPOSE_YML) + + // create envfile for "useenvfile" service + err = os.Mkdir(filepath.Join(tmpwork, "config"), 0777) + if err != nil { + t.Fatal(err) + } + envfile := filepath.Join(tmpwork, "config", "env") + fp, err := os.Create(envfile) + if err != nil { + t.Fatal("MKFILE", err) + } + fp.WriteString("FILEENV1=some_value\n") + fp.WriteString("FILEENV2=another_value\n") + fp.Close() + + TMP_DIR = tmp + TMPWORK_DIR = tmpwork + + p.Parse("testapp") + Generate(p, "test-0", "testapp", "1.2.3", DOCKER_COMPOSE_YML, tmp) return tmp, p } +func tearDown() { + if len(TMP_DIR) > 0 { + os.RemoveAll(TMP_DIR) + } + if len(TMPWORK_DIR) > 0 { + os.RemoveAll(TMPWORK_DIR) + } +} + // Check if the web2 service has got a command. func TestCommand(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -142,7 +184,6 @@ func TestCommand(t *testing.T) { commands = append(commands, line) } } - //log.Printf("%#v\n", commands) ok := 0 for _, command := range commands { if strings.Contains(command, "- /bin/sh") { @@ -165,7 +206,7 @@ func TestCommand(t *testing.T) { // Check if environment is correctly set. func TestEnvs(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -202,7 +243,7 @@ func TestEnvs(t *testing.T) { // Check if the same pod is not deployed twice. func TestSamePod(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -228,7 +269,7 @@ func TestSamePod(t *testing.T) { // Check if the ports are correctly set. func TestPorts(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -256,7 +297,7 @@ func TestPorts(t *testing.T) { // Check if the volumes are correctly set. func TestPVC(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -277,7 +318,7 @@ func TestPVC(t *testing.T) { //Check if web service has got a ingress. func TestIngress(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -298,7 +339,7 @@ func TestIngress(t *testing.T) { // Check unmapped volumes func TestUnmappedVolumes(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() for _, service := range p.Data.Services { name := service.Name @@ -319,7 +360,7 @@ func TestUnmappedVolumes(t *testing.T) { // Check if service using equal sign for environment works func TestEqualSignOnEnv(t *testing.T) { tmp, p := setUp(t) - defer os.RemoveAll(tmp) + defer tearDown() // if the name is eqenv, the service should habe environment for _, service := range p.Data.Services { diff --git a/generator/writer.go b/generator/writer.go index fd61e14..dc35b52 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -18,6 +18,16 @@ import ( var PrefixRE = regexp.MustCompile(`\{\{.*\}\}-?`) +func portExists(port int, ports []types.ServicePortConfig) bool { + for _, p := range ports { + if p.Target == uint32(port) { + log.Println("portExists:", port, p.Target) + return true + } + } + return false +} + func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFile, dirName string) { // make the appname global (yes... ugly but easy) @@ -47,6 +57,10 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi if err != nil { log.Fatal(err) } + if portExists(target, service.Ports) { + continue + } + log.Println("Add port to service:", n, target) service.Ports = append(service.Ports, types.ServicePortConfig{ Target: uint32(target), }) -- 2.49.1 From bd133431bfb1310c65e653b6c0a9308a5f934af9 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Tue, 5 Apr 2022 08:22:53 +0200 Subject: [PATCH 07/47] Ports labels were doubled --- generator/main.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/generator/main.go b/generator/main.go index 95c0285..3fd652e 100644 --- a/generator/main.go +++ b/generator/main.go @@ -281,23 +281,6 @@ func generateContainerPorts(s types.ServiceConfig, name string, container *helm. exists[int(port.Target)] = name } - // Get ports from label - if v, ok := s.Labels[helm.LABEL_PORT]; ok { - // split port by "," - ports := strings.Split(v, ",") - for _, port := range ports { - port, err := strconv.Atoi(port) - if err != nil { - log.Fatalf("The given port \"%v\" as container port in \"%s\" service is not an integer\n", v, name) - } - container.Ports = append(container.Ports, &helm.ContainerPort{ - Name: name, - ContainerPort: port, - }) - exists[port] = name - } - } - // manage the "expose" section to be a NodePort in Kubernetes for _, expose := range s.Expose { -- 2.49.1 From 1975b8fb898d9793298fd10bc4f1e3cfb1b12c78 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Tue, 5 Apr 2022 09:17:18 +0200 Subject: [PATCH 08/47] WIP Fix probes --- generator/main.go | 96 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 84 insertions(+), 12 deletions(-) diff --git a/generator/main.go b/generator/main.go index 3fd652e..b181f62 100644 --- a/generator/main.go +++ b/generator/main.go @@ -7,6 +7,7 @@ import ( "katenary/helm" "katenary/logger" "log" + "net/url" "os" "path/filepath" "strconv" @@ -63,37 +64,37 @@ func CreateReplicaObject(name string, s types.ServiceConfig, linked map[string]t func parseService(name string, s types.ServiceConfig, linked map[string]types.ServiceConfig, ret chan interface{}) { logger.Magenta(ICON_PACKAGE+" Generating deployment for ", name) - o := helm.NewDeployment(name) + deployment := helm.NewDeployment(name) container := helm.NewContainer(name, s.Image, s.Environment, s.Labels) prepareContainer(container, s, name) prepareEnvFromFiles(name, s, container, ret) // Set the container to the deployment - o.Spec.Template.Spec.Containers = []*helm.Container{container} + deployment.Spec.Template.Spec.Containers = []*helm.Container{container} // Prepare volumes madePVC := make(map[string]bool) - o.Spec.Template.Spec.Volumes = prepareVolumes(name, name, s, container, madePVC, ret) + deployment.Spec.Template.Spec.Volumes = prepareVolumes(name, name, s, container, madePVC, ret) // Now, for "depends_on" section, it's a bit tricky to get dependencies, see the function below. - o.Spec.Template.Spec.InitContainers = prepareInitContainers(name, s, container) + deployment.Spec.Template.Spec.InitContainers = prepareInitContainers(name, s, container) // Add selectors selectors := buildSelector(name, s) - o.Spec.Selector = map[string]interface{}{ + deployment.Spec.Selector = map[string]interface{}{ "matchLabels": selectors, } - o.Spec.Template.Metadata.Labels = selectors + deployment.Spec.Template.Metadata.Labels = selectors // Now, the linked services for lname, link := range linked { container := helm.NewContainer(lname, link.Image, link.Environment, link.Labels) prepareContainer(container, link, lname) prepareEnvFromFiles(lname, link, container, ret) - o.Spec.Template.Spec.Containers = append(o.Spec.Template.Spec.Containers, container) - o.Spec.Template.Spec.Volumes = append(o.Spec.Template.Spec.Volumes, prepareVolumes(name, lname, link, container, madePVC, ret)...) - o.Spec.Template.Spec.InitContainers = append(o.Spec.Template.Spec.InitContainers, prepareInitContainers(lname, link, container)...) + deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, container) + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, prepareVolumes(name, lname, link, container, madePVC, ret)...) + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, prepareInitContainers(lname, link, container)...) //append ports and expose ports to the deployment, to be able to generate them in the Service file if len(link.Ports) > 0 || len(link.Expose) > 0 { s.Ports = append(s.Ports, link.Ports...) @@ -104,7 +105,7 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se // Remove duplicates in volumes volumes := make([]map[string]interface{}, 0) done := make(map[string]bool) - for _, vol := range o.Spec.Template.Spec.Volumes { + for _, vol := range deployment.Spec.Template.Spec.Volumes { name := vol["name"].(string) if _, ok := done[name]; ok { continue @@ -113,7 +114,7 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se volumes = append(volumes, vol) } } - o.Spec.Template.Spec.Volumes = volumes + deployment.Spec.Template.Spec.Volumes = volumes // Then, create Services and possible Ingresses for ingress labels, "ports" and "expose" section if len(s.Ports) > 0 || len(s.Expose) > 0 { @@ -128,7 +129,7 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se } // the deployment is ready, give it - ret <- o + ret <- deployment // and then, we can say that it's the end ret <- nil @@ -464,6 +465,77 @@ func prepareInitContainers(name string, s types.ServiceConfig, container *helm.C // prepareProbes generate http/tcp/command probes for a service. func prepareProbes(name string, s types.ServiceConfig, container *helm.Container) { + // first, check if there is no label for the probe + if check, ok := s.Labels[helm.LABEL_HEALTHCHECK]; ok { + check = strings.TrimSpace(check) + // get the port of the "url" check + if checkurl, err := url.Parse(check); err == nil { + if err == nil { + container.LivenessProbe = buildProtoProbe(checkurl) + } + } else { + // it's a command + container.LivenessProbe = &helm.Probe{ + Exec: &helm.Exec{ + Command: []string{ + "sh", + "-c", + check, + }, + }, + } + } + return // label overrides everything + } + // if not, we will use the default one + if s.HealthCheck != nil { + container.LivenessProbe = buildCommandProbe(s) + } +} + +func buildProtoProbe(u *url.URL) *helm.Probe { + probe := helm.Probe{} + port, err := strconv.Atoi(u.Port()) + if err != nil { + port = 80 + } + switch u.Scheme { + case "http", "https": + probe.HttpGet = &helm.HttpGet{ + Path: u.Path, + Port: port, + } + case "tcp": + probe.TCP = &helm.TCP{ + Port: port, + } + default: + logger.Redf("Error while parsing healthcheck url %s\n", u.String()) + os.Exit(1) + } + return &probe +} + +func buildCommandProbe(s types.ServiceConfig) *helm.Probe { + + // Get the first element of the command from ServiceConfig + first := s.HealthCheck.Test[0] + switch first { + case "CMD", "CMD-SHELL": + // CMD or CMD-SHELL + return &helm.Probe{ + Exec: &helm.Exec{ + Command: s.HealthCheck.Test[1:], + }, + } + default: + // badly made but it should work... + return &helm.Probe{ + Exec: &helm.Exec{ + Command: []string(s.HealthCheck.Test), + }, + } + } } -- 2.49.1 From c9bdd1b718be87eed225b744879a32bf199fe994 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Tue, 5 Apr 2022 09:33:49 +0200 Subject: [PATCH 09/47] Fixup not bound volumes --- generator/main.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/generator/main.go b/generator/main.go index b181f62..beb6d21 100644 --- a/generator/main.go +++ b/generator/main.go @@ -312,6 +312,13 @@ func prepareVolumes(deployment, name string, s types.ServiceConfig, container *h volname := vol.Source volepath := vol.Target + if volname == "" { + logger.ActivateColors = true + logger.Yellowf("Warning, volume source to %s is empty for %s -- skipping\n", volepath, name) + logger.ActivateColors = false + continue + } + isCM := false for _, cmVol := range configMapsVolumes { cmVol = strings.TrimSpace(cmVol) -- 2.49.1 From fa9aebb8e5310260b66da4bed3c7a380c8fe8fe7 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 08:05:15 +0200 Subject: [PATCH 10/47] Remove doubles from envfile and env --- generator/main.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/generator/main.go b/generator/main.go index beb6d21..2bf868b 100644 --- a/generator/main.go +++ b/generator/main.go @@ -596,6 +596,19 @@ func prepareEnvFromFiles(name string, s types.ServiceConfig, container *helm.Con }, }) + // read the envfile and remove them from the container environment or secret + envs := readEnvFile(envfile) + for varname := range envs { + if !isSecret { + // remove varname from container + for i, s := range container.Env { + if s.Name == varname { + container.Env = append(container.Env[:i], container.Env[i+1:]...) + } + } + } + } + ret <- store } } @@ -623,3 +636,23 @@ func AddVolumeValues(deployment string, volname string, values map[string]interf } VolumeValues[deployment][volname] = values } + +func readEnvFile(envfilename string) map[string]string { + env := make(map[string]string) + content, err := ioutil.ReadFile(envfilename) + if err != nil { + logger.ActivateColors = true + logger.Red(err.Error()) + logger.ActivateColors = false + os.Exit(2) + } + // each value is on a separate line with KEY=value + lines := strings.Split(string(content), "\n") + for _, line := range lines { + if strings.Contains(line, "=") { + kv := strings.SplitN(line, "=", 2) + env[kv[0]] = kv[1] + } + } + return env +} -- 2.49.1 From 4c85097f2c810691217cf61deaeaafde449e99bd Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 08:07:27 +0200 Subject: [PATCH 11/47] Move command to a correct dir name --- .gitignore | 2 +- Makefile | 2 +- cmd/{ => katenary}/main.go | 0 cmd/{ => katenary}/utils.go | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename cmd/{ => katenary}/main.go (100%) rename cmd/{ => katenary}/utils.go (100%) diff --git a/.gitignore b/.gitignore index 7d31128..22c7e0c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ dist/* .cache/* chart/* docker-compose.yaml -katenary +./katenary *.env docker-compose* !examples/**/docker-compose* diff --git a/Makefile b/Makefile index bf0a0f5..f8b4877 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ PREFIX=~/.local GO=container OUT=katenary -BLD_CMD=go build -ldflags="-X 'main.Version=$(VERSION)'" -o $(OUT) ./cmd/*.go +BLD_CMD=go build -ldflags="-X 'main.Version=$(VERSION)'" -o $(OUT) ./cmd/katenary/*.go GOOS=linux GOARCH=amd64 diff --git a/cmd/main.go b/cmd/katenary/main.go similarity index 100% rename from cmd/main.go rename to cmd/katenary/main.go diff --git a/cmd/utils.go b/cmd/katenary/utils.go similarity index 100% rename from cmd/utils.go rename to cmd/katenary/utils.go -- 2.49.1 From ad336c27a165c9446804f6dfb97574b85a81985e Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 08:24:19 +0200 Subject: [PATCH 12/47] Fix use of BUILD_IMAGE --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f8b4877..5b9903f 100644 --- a/Makefile +++ b/Makefile @@ -106,10 +106,10 @@ ifeq ($(GO),local) $(BLD_CMD) else ifeq ($(CTN),podman) @podman run -e CGO_ENABLED=0 -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) \ - --rm -v $(PWD):/go/src/katenary:z -w /go/src/katenary --userns keep-id -it docker.io/golang $(BLD_CMD) + --rm -v $(PWD):/go/src/katenary:z -w /go/src/katenary --userns keep-id -it $(BUILD_IMAGE) $(BLD_CMD) else @docker run -e CGO_ENABLED=0 -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) \ - --rm -v $(PWD):/go/src/katenary:z -w /go/src/katenary --user $(shell id -u):$(shell id -g) -e HOME=/tmp -it docker.io/golang $(BLD_CMD) + --rm -v $(PWD):/go/src/katenary:z -w /go/src/katenary --user $(shell id -u):$(shell id -g) -e HOME=/tmp -it $(BUILD_IMAGE) $(BLD_CMD) endif echo "=> Stripping if possible" strip $(OUT) 2>/dev/null || echo "=> No strip available" -- 2.49.1 From 979206d3bf6b28b32dd84b86244e9ea20a74e211 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 08:56:12 +0200 Subject: [PATCH 13/47] Fix multiple problems on ingress version check --- generator/writers/ingress.go | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/generator/writers/ingress.go b/generator/writers/ingress.go index b11a552..9c00f89 100644 --- a/generator/writers/ingress.go +++ b/generator/writers/ingress.go @@ -10,8 +10,18 @@ import ( "gopkg.in/yaml.v3" ) -const classAndVersionCondition = `{{- if and .Values.__name__.ingress.class (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}` + "\n" -const versionCondition = `{{- if semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion }}` + "\n" +const ( + classAndVersionCondition = `{{- if and .Values.__name__.ingress.class (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}` + "\n" + versionCondition118 = `{{- if semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion }}` + "\n" + versionCondition119 = `{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}` + "\n" + apiVersion = `{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }}` +) func BuildIngress(ingress *helm.Ingress, name, templatesDir string) { // Set the backend for 1.18 @@ -38,6 +48,19 @@ func BuildIngress(ingress *helm.Ingress, name, templatesDir string) { backendHit := false for _, l := range lines { + // apiVersion is a pain... + if strings.Contains(l, "apiVersion:") { + l = apiVersion + } + + // pathTyype is ony for 1.19+ + if strings.Contains(l, "pathType:") { + n := CountSpaces(l) + l = strings.Repeat(" ", n) + versionCondition118 + + l + "\n" + + strings.Repeat(" ", n) + "{{- end -}}" + } + if strings.Contains(l, "ingressClassName") { // should be set only if the version of Kubernetes is 1.18-0 or higher cond := strings.ReplaceAll(classAndVersionCondition, "__name__", name) @@ -47,14 +70,14 @@ func BuildIngress(ingress *helm.Ingress, name, templatesDir string) { // manage the backend format following the Kubernetes 1.18-0 version or higher if strings.Contains(l, "service:") { n := CountSpaces(l) - l = strings.Repeat(" ", n) + versionCondition + l + l = strings.Repeat(" ", n) + versionCondition119 + l } if strings.Contains(l, "serviceName:") || strings.Contains(l, "servicePort:") { n := CountSpaces(l) if !backendHit { l = strings.Repeat(" ", n) + "{{- else }}\n" + l } else { - l = l + "\n" + strings.Repeat(" ", n) + "{{- end }}\n" + l = l + "\n" + strings.Repeat(" ", n) + "{{- end }}" } backendHit = true } -- 2.49.1 From 13f74c9e4ddfee4949e95d87772f8bbdff3e67f9 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 08:59:31 +0200 Subject: [PATCH 14/47] Optim --- generator/writers/ingress.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/generator/writers/ingress.go b/generator/writers/ingress.go index 9c00f89..aab4fab 100644 --- a/generator/writers/ingress.go +++ b/generator/writers/ingress.go @@ -42,7 +42,12 @@ func BuildIngress(ingress *helm.Ingress, name, templatesDir string) { enc.Encode(ingress) buffer.WriteString("{{- end -}}") - fp, _ := os.Create(fname) + fp, err := os.Create(fname) + if err != nil { + panic(err) + } + defer fp.Close() + content := string(buffer.Bytes()) lines := strings.Split(content, "\n") @@ -67,7 +72,7 @@ func BuildIngress(ingress *helm.Ingress, name, templatesDir string) { l = ` ` + cond + l + "\n" + ` {{- end }}` } - // manage the backend format following the Kubernetes 1.18-0 version or higher + // manage the backend format following the Kubernetes 1.19-0 version or higher if strings.Contains(l, "service:") { n := CountSpaces(l) l = strings.Repeat(" ", n) + versionCondition119 + l @@ -81,9 +86,6 @@ func BuildIngress(ingress *helm.Ingress, name, templatesDir string) { } backendHit = true } - fp.WriteString(l + "\n") } - - fp.Close() } -- 2.49.1 From 6eb8ec991570e9a644819071a4f6ad57875fc7c2 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 09:08:12 +0200 Subject: [PATCH 15/47] Fix problems of new line + add comments --- generator/writers/configmap.go | 1 + generator/writers/deployment.go | 1 + generator/writers/ingress.go | 5 +++-- generator/writers/service.go | 1 + generator/writers/storage.go | 1 + 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/generator/writers/configmap.go b/generator/writers/configmap.go index d17a889..846dabc 100644 --- a/generator/writers/configmap.go +++ b/generator/writers/configmap.go @@ -7,6 +7,7 @@ import ( "gopkg.in/yaml.v3" ) +// BuildConfigMap writes the configMap. func BuildConfigMap(c interface{}, kind, servicename, name, templatesDir string) { fname := filepath.Join(templatesDir, servicename+"."+name+"."+kind+".yaml") fp, _ := os.Create(fname) diff --git a/generator/writers/deployment.go b/generator/writers/deployment.go index ca09249..7f594ca 100644 --- a/generator/writers/deployment.go +++ b/generator/writers/deployment.go @@ -10,6 +10,7 @@ import ( "gopkg.in/yaml.v3" ) +// BuildDeployment builds a deployment. func BuildDeployment(deployment *helm.Deployment, name, templatesDir string) { kind := "deployment" fname := filepath.Join(templatesDir, name+"."+kind+".yaml") diff --git a/generator/writers/ingress.go b/generator/writers/ingress.go index aab4fab..be7b4bd 100644 --- a/generator/writers/ingress.go +++ b/generator/writers/ingress.go @@ -23,6 +23,7 @@ apiVersion: extensions/v1beta1 {{- end }}` ) +// BuildIngress generates the ingress yaml file with conditions. func BuildIngress(ingress *helm.Ingress, name, templatesDir string) { // Set the backend for 1.18 for _, b := range ingress.Spec.Rules { @@ -63,7 +64,7 @@ func BuildIngress(ingress *helm.Ingress, name, templatesDir string) { n := CountSpaces(l) l = strings.Repeat(" ", n) + versionCondition118 + l + "\n" + - strings.Repeat(" ", n) + "{{- end -}}" + strings.Repeat(" ", n) + "{{- end }}" } if strings.Contains(l, "ingressClassName") { @@ -82,7 +83,7 @@ func BuildIngress(ingress *helm.Ingress, name, templatesDir string) { if !backendHit { l = strings.Repeat(" ", n) + "{{- else }}\n" + l } else { - l = l + "\n" + strings.Repeat(" ", n) + "{{- end }}" + l = l + "\n" + strings.Repeat(" ", n) + "{{- end }}\n" } backendHit = true } diff --git a/generator/writers/service.go b/generator/writers/service.go index 2ba9b82..c898e27 100644 --- a/generator/writers/service.go +++ b/generator/writers/service.go @@ -8,6 +8,7 @@ import ( "gopkg.in/yaml.v3" ) +// BuildService writes the service (external or not). func BuildService(service *helm.Service, name, templatesDir string) { kind := "service" suffix := "" diff --git a/generator/writers/storage.go b/generator/writers/storage.go index aa1593b..ea3617f 100644 --- a/generator/writers/storage.go +++ b/generator/writers/storage.go @@ -8,6 +8,7 @@ import ( "gopkg.in/yaml.v3" ) +// BuildStorage writes the persistentVolumeClaim. func BuildStorage(storage *helm.Storage, name, templatesDir string) { kind := "pvc" name = storage.Metadata.Labels[helm.K+"/component"] -- 2.49.1 From 8ce0d64e62659b6fa38f9931ef9c19523ba2bb29 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 09:24:51 +0200 Subject: [PATCH 16/47] Fix probes --- generator/main.go | 41 ++++++++++++++++++++++------------------- helm/deployment.go | 16 +++++++++++++++- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/generator/main.go b/generator/main.go index 2bf868b..a442ec2 100644 --- a/generator/main.go +++ b/generator/main.go @@ -482,13 +482,12 @@ func prepareProbes(name string, s types.ServiceConfig, container *helm.Container } } else { // it's a command - container.LivenessProbe = &helm.Probe{ - Exec: &helm.Exec{ - Command: []string{ - "sh", - "-c", - check, - }, + container.LivenessProbe = helm.NewProbe(0, 0, 0, 0) + container.LivenessProbe.Exec = &helm.Exec{ + Command: []string{ + "sh", + "-c", + check, }, } } @@ -501,15 +500,21 @@ func prepareProbes(name string, s types.ServiceConfig, container *helm.Container } func buildProtoProbe(u *url.URL) *helm.Probe { - probe := helm.Probe{} + probe := helm.NewProbe(0, 0, 0, 0) port, err := strconv.Atoi(u.Port()) if err != nil { port = 80 } + + path := "/" + if u.Path != "" { + path = u.Path + } + switch u.Scheme { case "http", "https": probe.HttpGet = &helm.HttpGet{ - Path: u.Path, + Path: path, Port: port, } case "tcp": @@ -520,30 +525,28 @@ func buildProtoProbe(u *url.URL) *helm.Probe { logger.Redf("Error while parsing healthcheck url %s\n", u.String()) os.Exit(1) } - return &probe + return probe } func buildCommandProbe(s types.ServiceConfig) *helm.Probe { // Get the first element of the command from ServiceConfig first := s.HealthCheck.Test[0] + p := helm.NewProbe(0, 0, 0, 0) switch first { case "CMD", "CMD-SHELL": // CMD or CMD-SHELL - return &helm.Probe{ - Exec: &helm.Exec{ - Command: s.HealthCheck.Test[1:], - }, + p.Exec = &helm.Exec{ + Command: s.HealthCheck.Test[1:], } + return p default: // badly made but it should work... - return &helm.Probe{ - Exec: &helm.Exec{ - Command: []string(s.HealthCheck.Test), - }, + p.Exec = &helm.Exec{ + Command: []string(s.HealthCheck.Test), } + return p } - } // prepareEnvFromFiles generate configMap or secrets from environment files. diff --git a/helm/deployment.go b/helm/deployment.go index 172871d..d10b48e 100644 --- a/helm/deployment.go +++ b/helm/deployment.go @@ -77,13 +77,27 @@ type Probe struct { InitialDelay int `yaml:"initialDelaySeconds"` } +// Create a new Probe object that can be apply to HttpProbe or TCPProbe. func NewProbe(period, initialDelaySeconds, success, failure int) *Probe { - return &Probe{ + probe := &Probe{ Period: period, Success: success, Failure: failure, InitialDelay: initialDelaySeconds, } + + // fix default values from + // https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + if period == 0 { + probe.Period = 10 + } + if success == 0 { + probe.Success = 1 + } + if failure == 0 { + probe.Failure = 3 + } + return probe } func NewContainer(name, image string, environment types.MappingWithEquals, labels map[string]string) *Container { -- 2.49.1 From b6f2e480ba1d8b4b5b95f40399c4e1f28e42112d Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 09:43:52 +0200 Subject: [PATCH 17/47] Refacto to split types --- helm/{configMap.go => configAndSecretMap.go} | 9 ++ helm/container.go | 59 +++++++++++ helm/deployment.go | 100 ------------------- helm/ingress.go | 1 + helm/k8sbase.go | 56 +++++++++++ helm/notes.go | 1 + helm/probe.go | 51 ++++++++++ helm/service.go | 6 ++ helm/storage.go | 3 + helm/types.go | 60 ++--------- 10 files changed, 196 insertions(+), 150 deletions(-) rename helm/{configMap.go => configAndSecretMap.go} (82%) create mode 100644 helm/container.go create mode 100644 helm/k8sbase.go create mode 100644 helm/probe.go diff --git a/helm/configMap.go b/helm/configAndSecretMap.go similarity index 82% rename from helm/configMap.go rename to helm/configAndSecretMap.go index c2bdb3c..1ae3205 100644 --- a/helm/configMap.go +++ b/helm/configAndSecretMap.go @@ -13,11 +13,13 @@ type InlineConfig interface { Metadata() *Metadata } +// ConfigMap is made to represent a configMap with data. type ConfigMap struct { *K8sBase `yaml:",inline"` Data map[string]string `yaml:"data"` } +// NewConfigMap returns a new initialzed ConfigMap. func NewConfigMap(name string) *ConfigMap { base := NewBase() base.ApiVersion = "v1" @@ -30,10 +32,12 @@ func NewConfigMap(name string) *ConfigMap { } } +// Metadata returns the metadata of the configMap. func (c *ConfigMap) Metadata() *Metadata { return c.K8sBase.Metadata } +// AddEnvFile adds an environment file to the configMap. func (c *ConfigMap) AddEnvFile(file string) error { content, err := ioutil.ReadFile(file) if err != nil { @@ -57,11 +61,13 @@ func (c *ConfigMap) AddEnvFile(file string) error { } +// Secret is made to represent a secret with data. type Secret struct { *K8sBase `yaml:",inline"` Data map[string]string `yaml:"data"` } +// NewSecret returns a new initialzed Secret. func NewSecret(name string) *Secret { base := NewBase() base.ApiVersion = "v1" @@ -74,6 +80,7 @@ func NewSecret(name string) *Secret { } } +// AddEnvFile adds an environment file to the secret. func (s *Secret) AddEnvFile(file string) error { content, err := ioutil.ReadFile(file) if err != nil { @@ -96,6 +103,8 @@ func (s *Secret) AddEnvFile(file string) error { return nil } + +// Metadata returns the metadata of the secret. func (s *Secret) Metadata() *Metadata { return s.K8sBase.Metadata } diff --git a/helm/container.go b/helm/container.go new file mode 100644 index 0000000..06bb567 --- /dev/null +++ b/helm/container.go @@ -0,0 +1,59 @@ +package helm + +import ( + "strings" + + "github.com/compose-spec/compose-go/types" +) + +// ContainerPort represent a port mapping. +type ContainerPort struct { + Name string + ContainerPort int `yaml:"containerPort"` +} + +// Value represent a environment variable with name and value. +type Value struct { + Name string `yaml:"name"` + Value interface{} `yaml:"value"` +} + +// Container represent a container with name, image, and environment variables. It is used in Deployment. +type Container struct { + Name string `yaml:"name,omitempty"` + Image string `yaml:"image"` + Ports []*ContainerPort `yaml:"ports,omitempty"` + Env []Value `yaml:"env,omitempty"` + EnvFrom []map[string]map[string]string `yaml:"envFrom,omitempty"` + Command []string `yaml:"command,omitempty"` + VolumeMounts []interface{} `yaml:"volumeMounts,omitempty"` + LivenessProbe *Probe `yaml:"livenessProbe,omitempty"` +} + +// NewContainer creates a new container with name, image, labels and environment variables. +func NewContainer(name, image string, environment types.MappingWithEquals, labels map[string]string) *Container { + container := &Container{ + Image: image, + Name: name, + Env: make([]Value, len(environment)), + EnvFrom: make([]map[string]map[string]string, 0), + } + + // find bound environment variable to a service + toServices := make([]string, 0) + if bound, ok := labels[LABEL_ENV_SERVICE]; ok { + toServices = strings.Split(bound, ",") + } + + idx := 0 + for n, v := range environment { + for _, name := range toServices { + if name == n { + *v = RELEASE_NAME + "-" + *v + } + } + container.Env[idx] = Value{Name: n, Value: v} + idx++ + } + return container +} diff --git a/helm/deployment.go b/helm/deployment.go index d10b48e..bf5ff8e 100644 --- a/helm/deployment.go +++ b/helm/deployment.go @@ -1,11 +1,5 @@ package helm -import ( - "strings" - - "github.com/compose-spec/compose-go/types" -) - // Deployment is a k8s deployment. type Deployment struct { *K8sBase `yaml:",inline"` @@ -33,100 +27,6 @@ func NewDepSpec() *DepSpec { } } -type Value struct { - Name string `yaml:"name"` - Value interface{} `yaml:"value"` -} - -type ContainerPort struct { - Name string - ContainerPort int `yaml:"containerPort"` -} - -type Container struct { - Name string `yaml:"name,omitempty"` - Image string `yaml:"image"` - Ports []*ContainerPort `yaml:"ports,omitempty"` - Env []Value `yaml:"env,omitempty"` - EnvFrom []map[string]map[string]string `yaml:"envFrom,omitempty"` - Command []string `yaml:"command,omitempty"` - VolumeMounts []interface{} `yaml:"volumeMounts,omitempty"` - LivenessProbe *Probe `yaml:"livenessProbe,omitempty"` -} - -type HttpGet struct { - Path string `yaml:"path"` - Port int `yaml:"port"` -} - -type Exec struct { - Command []string `yaml:"command"` -} - -type TCP struct { - Port int `yaml:"port"` -} - -type Probe struct { - HttpGet *HttpGet `yaml:"httpGet,omitempty"` - Exec *Exec `yaml:"exec,omitempty"` - TCP *TCP `yaml:"tcp,omitempty"` - Period int `yaml:"periodSeconds"` - Success int `yaml:"successThreshold"` - Failure int `yaml:"failureThreshold"` - InitialDelay int `yaml:"initialDelaySeconds"` -} - -// Create a new Probe object that can be apply to HttpProbe or TCPProbe. -func NewProbe(period, initialDelaySeconds, success, failure int) *Probe { - probe := &Probe{ - Period: period, - Success: success, - Failure: failure, - InitialDelay: initialDelaySeconds, - } - - // fix default values from - // https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ - if period == 0 { - probe.Period = 10 - } - if success == 0 { - probe.Success = 1 - } - if failure == 0 { - probe.Failure = 3 - } - return probe -} - -func NewContainer(name, image string, environment types.MappingWithEquals, labels map[string]string) *Container { - container := &Container{ - Image: image, - Name: name, - Env: make([]Value, len(environment)), - EnvFrom: make([]map[string]map[string]string, 0), - } - - // find bound environment variable to a service - toServices := make([]string, 0) - if bound, ok := labels[LABEL_ENV_SERVICE]; ok { - toServices = strings.Split(bound, ",") - } - - idx := 0 - for n, v := range environment { - for _, name := range toServices { - if name == n { - *v = RELEASE_NAME + "-" + *v - } - } - container.Env[idx] = Value{Name: n, Value: v} - idx++ - } - return container -} - type PodSpec struct { InitContainers []*Container `yaml:"initContainers,omitempty"` Containers []*Container `yaml:"containers"` diff --git a/helm/ingress.go b/helm/ingress.go index 8301c06..88c8608 100644 --- a/helm/ingress.go +++ b/helm/ingress.go @@ -1,5 +1,6 @@ package helm +// Ingress is the kubernetes ingress object. type Ingress struct { *K8sBase `yaml:",inline"` Spec IngressSpec diff --git a/helm/k8sbase.go b/helm/k8sbase.go new file mode 100644 index 0000000..af2aa58 --- /dev/null +++ b/helm/k8sbase.go @@ -0,0 +1,56 @@ +package helm + +import ( + "crypto/sha1" + "fmt" + "io/ioutil" +) + +// Metadata is the metadata for a kubernetes object. +type Metadata struct { + Name string `yaml:"name,omitempty"` + Labels map[string]string `yaml:"labels"` + Annotations map[string]string `yaml:"annotations,omitempty"` +} + +func NewMetadata() *Metadata { + return &Metadata{ + Name: "", + Labels: make(map[string]string), + Annotations: make(map[string]string), + } +} + +// K8sBase is the base for all kubernetes objects. +type K8sBase struct { + ApiVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata *Metadata `yaml:"metadata"` +} + +// NewBase is a factory for creating a new base object with metadata, labels and annotations set to the default. +func NewBase() *K8sBase { + b := &K8sBase{ + Metadata: NewMetadata(), + } + // add some information of the build + b.Metadata.Labels[K+"/project"] = GetProjectName() + b.Metadata.Labels[K+"/release"] = RELEASE_NAME + b.Metadata.Annotations[K+"/version"] = Version + return b +} + +func (k *K8sBase) BuildSHA(filename string) { + c, _ := ioutil.ReadFile(filename) + //sum := sha256.Sum256(c) + sum := sha1.Sum(c) + k.Metadata.Annotations[K+"/docker-compose-sha1"] = fmt.Sprintf("%x", string(sum[:])) +} + +func (k *K8sBase) Get() string { + return k.Kind +} + +func (k *K8sBase) Name() string { + return k.Metadata.Name +} diff --git a/helm/notes.go b/helm/notes.go index ce3ad2e..54a33ec 100644 --- a/helm/notes.go +++ b/helm/notes.go @@ -10,6 +10,7 @@ Your application is now deployed. This may take a while to be up and responding. __list__ ` +// GenerateNotesFile generates the notes file for the helm chart. func GenerateNotesFile(ingressess map[string]*Ingress) string { list := make([]string, 0) diff --git a/helm/probe.go b/helm/probe.go new file mode 100644 index 0000000..210ae1d --- /dev/null +++ b/helm/probe.go @@ -0,0 +1,51 @@ +package helm + +// Probe is a struct that can be used to create a Liveness or Readiness probe. +type Probe struct { + HttpGet *HttpGet `yaml:"httpGet,omitempty"` + Exec *Exec `yaml:"exec,omitempty"` + TCP *TCP `yaml:"tcp,omitempty"` + Period int `yaml:"periodSeconds"` + Success int `yaml:"successThreshold"` + Failure int `yaml:"failureThreshold"` + InitialDelay int `yaml:"initialDelaySeconds"` +} + +// Create a new Probe object that can be apply to HttpProbe or TCPProbe. +func NewProbe(period, initialDelaySeconds, success, failure int) *Probe { + probe := &Probe{ + Period: period, + Success: success, + Failure: failure, + InitialDelay: initialDelaySeconds, + } + + // fix default values from + // https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + if period == 0 { + probe.Period = 10 + } + if success == 0 { + probe.Success = 1 + } + if failure == 0 { + probe.Failure = 3 + } + return probe +} + +// HttpGet is a Probe configuration to check http health. +type HttpGet struct { + Path string `yaml:"path"` + Port int `yaml:"port"` +} + +// Execis a Probe configuration to check exec health. +type Exec struct { + Command []string `yaml:"command"` +} + +// TCP is a Probe configuration to check tcp health. +type TCP struct { + Port int `yaml:"port"` +} diff --git a/helm/service.go b/helm/service.go index 6080904..2e00ca5 100644 --- a/helm/service.go +++ b/helm/service.go @@ -1,10 +1,12 @@ package helm +// Service is a Kubernetes service. type Service struct { *K8sBase `yaml:",inline"` Spec *ServiceSpec `yaml:"spec"` } +// NewService creates a new initialized service. func NewService(name string) *Service { s := &Service{ K8sBase: NewBase(), @@ -17,12 +19,14 @@ func NewService(name string) *Service { return s } +// ServicePort is a port on a service. type ServicePort struct { Protocol string `yaml:"protocol"` Port int `yaml:"port"` TargetPort int `yaml:"targetPort"` } +// NewServicePort creates a new initialized service port. func NewServicePort(port, target int) *ServicePort { return &ServicePort{ Protocol: "TCP", @@ -31,12 +35,14 @@ func NewServicePort(port, target int) *ServicePort { } } +// ServiceSpec is the spec for a service. type ServiceSpec struct { Selector map[string]string Ports []*ServicePort Type string `yaml:"type,omitempty"` } +// NewServiceSpec creates a new initialized service spec. func NewServiceSpec() *ServiceSpec { return &ServiceSpec{ Selector: make(map[string]string), diff --git a/helm/storage.go b/helm/storage.go index c10ef3b..491748e 100644 --- a/helm/storage.go +++ b/helm/storage.go @@ -1,10 +1,12 @@ package helm +// Storage is a struct for a PersistentVolumeClaim. type Storage struct { *K8sBase `yaml:",inline"` Spec *PVCSpec } +// NewPVC creates a new PersistentVolumeClaim object. func NewPVC(name, storageName string) *Storage { pvc := &Storage{} pvc.K8sBase = NewBase() @@ -24,6 +26,7 @@ func NewPVC(name, storageName string) *Storage { return pvc } +// PVCSpec is a struct for a PersistentVolumeClaim spec. type PVCSpec struct { Resouces map[string]interface{} `yaml:"resources"` AccessModes []string `yaml:"accessModes"` diff --git a/helm/types.go b/helm/types.go index bf4ccfc..0ae082d 100644 --- a/helm/types.go +++ b/helm/types.go @@ -2,9 +2,6 @@ package helm import ( "bytes" - "crypto/sha1" - "fmt" - "io/ioutil" "os" "strings" "text/template" @@ -58,65 +55,28 @@ var ( Version = "1.0" // should be set from main.Version ) +// Kinded represent an object with a kind. type Kinded interface { + + // Get must resturn the kind name. Get() string } +// Signable represents an object with a signature. type Signable interface { + + // BuildSHA must return the signature. BuildSHA(filename string) } +// Named represents an object with a name. type Named interface { + + // Name must return the name of the object (from metadata). Name() string } -type Metadata struct { - Name string `yaml:"name,omitempty"` - Labels map[string]string `yaml:"labels"` - Annotations map[string]string `yaml:"annotations,omitempty"` -} - -func NewMetadata() *Metadata { - return &Metadata{ - Name: "", - Labels: make(map[string]string), - Annotations: make(map[string]string), - } -} - -type K8sBase struct { - ApiVersion string `yaml:"apiVersion"` - Kind string `yaml:"kind"` - Metadata *Metadata `yaml:"metadata"` -} - -func NewBase() *K8sBase { - - b := &K8sBase{ - Metadata: NewMetadata(), - } - // add some information of the build - b.Metadata.Labels[K+"/project"] = GetProjectName() - b.Metadata.Labels[K+"/release"] = RELEASE_NAME - b.Metadata.Annotations[K+"/version"] = Version - return b -} - -func (k *K8sBase) BuildSHA(filename string) { - c, _ := ioutil.ReadFile(filename) - //sum := sha256.Sum256(c) - sum := sha1.Sum(c) - k.Metadata.Annotations[K+"/docker-compose-sha1"] = fmt.Sprintf("%x", string(sum[:])) -} - -func (k *K8sBase) Get() string { - return k.Kind -} - -func (k *K8sBase) Name() string { - return k.Metadata.Name -} - +// GetProjectName returns the name of the project. func GetProjectName() string { if len(Appname) > 0 { return Appname -- 2.49.1 From 910933e5f3b37b8c375ef57bb5cb2ce689822958 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 09:49:31 +0200 Subject: [PATCH 18/47] Updated dependencies --- go.mod | 6 ++- go.sum | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 124 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 4e007b6..82378f6 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,14 @@ module katenary go 1.16 require ( - github.com/compose-spec/compose-go v1.2.2 + github.com/compose-spec/compose-go v1.2.4 + github.com/distribution/distribution/v3 v3.0.0-20220504180456-7a6b9e3042bd // indirect github.com/kr/pretty v0.2.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/spf13/cobra v1.4.0 + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect golang.org/x/mod v0.5.1 + golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/go.sum b/go.sum index ac3e4ea..e448930 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,24 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.34.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -14,16 +28,27 @@ github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/compose-spec/compose-go v1.2.2 h1:y1dwl3KUTBnWPVur6EZno9zUIum6Q87/F5keljnGQB4= -github.com/compose-spec/compose-go v1.2.2/go.mod h1:pAy7Mikpeft4pxkFU565/DRHEbDfR84G6AQuiL+Hdg8= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/compose-spec/compose-go v1.2.4 h1:nzTFqM8+2J7Veao5Pq5U451thinv3U1wChIvcjX59/A= +github.com/compose-spec/compose-go v1.2.4/go.mod h1:pAy7Mikpeft4pxkFU565/DRHEbDfR84G6AQuiL+Hdg8= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/distribution/distribution/v3 v3.0.0-20210316161203-a01c71e2477e h1:n81KvOMrLZa+VWHwST7dun9f0G98X3zREHS1ztYzZKU= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/distribution/distribution/v3 v3.0.0-20210316161203-a01c71e2477e/go.mod h1:xpWTC2KnJMiDLkoawhsPQcXjvwATEBcbq0xevG2YR9M= +github.com/distribution/distribution/v3 v3.0.0-20220504180456-7a6b9e3042bd h1:KRoLSsR7wZ4H2dueR/O6BGBIXDxfOxUVmaMiu1QiQPw= +github.com/distribution/distribution/v3 v3.0.0-20220504180456-7a6b9e3042bd/go.mod h1:qLi7jGj1b5TUaYTB3ekkHiocxHJi8+3CFhXM54MGKBs= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= @@ -33,30 +58,51 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -66,13 +112,16 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -80,76 +129,118 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -157,11 +248,24 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -170,16 +274,24 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -187,3 +299,4 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -- 2.49.1 From 053292f37c2a9d88340c9aa50fc40abcf0b13d01 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 10:22:02 +0200 Subject: [PATCH 19/47] Add a "ignore" label to skip a service --- README.md | 1 + generator/writer.go | 26 +++++++++++++++++++++++ helm/labels.go | 52 +++++++++++++++++++++++++++++++++++++++++++++ helm/types.go | 48 +---------------------------------------- 4 files changed, 80 insertions(+), 47 deletions(-) create mode 100644 helm/labels.go diff --git a/README.md b/README.md index 1682cfb..6375138 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,7 @@ services: These labels could be found by `katenary show-labels`, and can be placed as "labels" inside your docker-compose file: ``` +katenary.io/ignore : ignore the container, it will not yied any object in the helm chart katenary.io/secret-envfiles : set the given file names as a secret instead of configmap katenary.io/ports : set the ports to expose as a service (coma separated) katenary.io/ingress : set the port to expose in an ingress (coma separated) diff --git a/generator/writer.go b/generator/writer.go index dc35b52..de9e54b 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -45,9 +45,35 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi // Manage services, avoid linked pods and store all services port in servicesMap avoids := make(map[string]bool) + skips := make(map[string]bool) + + for _, s := range p.Data.Services { + if s.Labels[helm.LABEL_IGNORE] == "true" { + skips[s.Name] = true + } + } + + // remove skipped services + for s := range skips { + for i, service := range p.Data.Services { + if service.Name == s { + p.Data.Services = append(p.Data.Services[:i], p.Data.Services[i+1:]...) + break + } + } + } + for i, service := range p.Data.Services { n := service.Name + // if the service depends on a skipped service, remove the link + for dep := range service.DependsOn { + if skips[dep] { + delete(service.DependsOn, dep) + } + } + + // if the service port is declared in labels, add it to the service. if ports, ok := service.Labels[helm.LABEL_PORT]; ok { if service.Ports == nil { service.Ports = make([]types.ServicePortConfig, 0) diff --git a/helm/labels.go b/helm/labels.go new file mode 100644 index 0000000..f35f320 --- /dev/null +++ b/helm/labels.go @@ -0,0 +1,52 @@ +package helm + +import ( + "bytes" + "html/template" +) + +const RELEASE_NAME = "{{ .Release.Name }}" +const ( + LABEL_ENV_SECRET = K + "/secret-envfiles" + LABEL_PORT = K + "/ports" + LABEL_INGRESS = K + "/ingress" + LABEL_ENV_SERVICE = K + "/env-to-service" + LABEL_VOL_CM = K + "/configmap-volumes" + LABEL_HEALTHCHECK = K + "/healthcheck" + LABEL_SAMEPOD = K + "/same-pod" + LABEL_EMPTYDIRS = K + "/empty-dirs" + LABEL_IGNORE = K + "/ignore" +) + +// GetLabelsDocumentation returns the documentation for the labels. +func GetLabelsDocumentation() string { + t, _ := template.New("labels").Parse(` +# Labels +{{ .LABEL_IGNORE | printf "%-33s"}}: ignore the container, it will not yied any object in the helm chart +{{.LABEL_ENV_SECRET | printf "%-33s"}}: set the given file names as a secret instead of configmap +{{.LABEL_PORT | printf "%-33s"}}: set the ports to expose as a service (coma separated) +{{.LABEL_INGRESS | printf "%-33s"}}: set the port to expose in an ingress (coma separated) +{{.LABEL_ENV_SERVICE | printf "%-33s"}}: specifies that the environment variable points on a service name (coma separated) +{{.LABEL_VOL_CM | printf "%-33s"}}: specifies that the volumes points on a configmap (coma separated) +{{.LABEL_SAMEPOD | printf "%-33s"}}: specifies that the pod should be deployed in the same pod than the given service name +{{.LABEL_EMPTYDIRS | printf "%-33s"}}: specifies that the given volume names should be "emptyDir" instead of persistentVolumeClaim (coma separated) +{{.LABEL_HEALTHCHECK | printf "%-33s"}}: specifies that the container should be monitored by a healthcheck, **it overrides the docker-compose healthcheck**. +{{ printf "%-34s" ""}} You can use these form of label values: +{{ printf "%-35s" ""}}- "http://[not used address][:port][/path]" to specify an http healthcheck +{{ printf "%-35s" ""}}- "tcp://[not used address]:port" to specify a tcp healthcheck +{{ printf "%-35s" ""}}- other string is condidered as a "command" healthcheck + `) + buff := bytes.NewBuffer(nil) + t.Execute(buff, map[string]string{ + "LABEL_ENV_SECRET": LABEL_ENV_SECRET, + "LABEL_ENV_SERVICE": LABEL_ENV_SERVICE, + "LABEL_PORT": LABEL_PORT, + "LABEL_INGRESS": LABEL_INGRESS, + "LABEL_VOL_CM": LABEL_VOL_CM, + "LABEL_HEALTHCHECK": LABEL_HEALTHCHECK, + "LABEL_SAMEPOD": LABEL_SAMEPOD, + "LABEL_EMPTYDIRS": LABEL_EMPTYDIRS, + "LABEL_IGNORE": LABEL_IGNORE, + }) + return buff.String() +} diff --git a/helm/types.go b/helm/types.go index 0ae082d..9fcd3d7 100644 --- a/helm/types.go +++ b/helm/types.go @@ -1,77 +1,31 @@ package helm import ( - "bytes" "os" "strings" - "text/template" ) const K = "katenary.io" -const RELEASE_NAME = "{{ .Release.Name }}" -const ( - LABEL_ENV_SECRET = K + "/secret-envfiles" - LABEL_PORT = K + "/ports" - LABEL_INGRESS = K + "/ingress" - LABEL_ENV_SERVICE = K + "/env-to-service" - LABEL_VOL_CM = K + "/configmap-volumes" - LABEL_HEALTHCHECK = K + "/healthcheck" - LABEL_SAMEPOD = K + "/same-pod" - LABEL_EMPTYDIRS = K + "/empty-dirs" -) - -func GetLabelsDocumentation() string { - t, _ := template.New("labels").Parse(` -# Labels -{{.LABEL_ENV_SECRET | printf "%-33s"}}: set the given file names as a secret instead of configmap -{{.LABEL_PORT | printf "%-33s"}}: set the ports to expose as a service (coma separated) -{{.LABEL_INGRESS | printf "%-33s"}}: set the port to expose in an ingress (coma separated) -{{.LABEL_ENV_SERVICE | printf "%-33s"}}: specifies that the environment variable points on a service name (coma separated) -{{.LABEL_VOL_CM | printf "%-33s"}}: specifies that the volumes points on a configmap (coma separated) -{{.LABEL_SAMEPOD | printf "%-33s"}}: specifies that the pod should be deployed in the same pod than the given service name -{{.LABEL_EMPTYDIRS | printf "%-33s"}}: specifies that the given volume names should be "emptyDir" instead of persistentVolumeClaim (coma separated) -{{.LABEL_HEALTHCHECK | printf "%-33s"}}: specifies that the container should be monitored by a healthcheck, **it overrides the docker-compose healthcheck**. -{{ printf "%-34s" ""}} You can use these form of label values: -{{ printf "%-35s" ""}}- "http://[not used address][:port][/path]" to specify an http healthcheck -{{ printf "%-35s" ""}}- "tcp://[not used address]:port" to specify a tcp healthcheck -{{ printf "%-35s" ""}}- other string is condidered as a "command" healthcheck - `) - buff := bytes.NewBuffer(nil) - t.Execute(buff, map[string]string{ - "LABEL_ENV_SECRET": LABEL_ENV_SECRET, - "LABEL_ENV_SERVICE": LABEL_ENV_SERVICE, - "LABEL_PORT": LABEL_PORT, - "LABEL_INGRESS": LABEL_INGRESS, - "LABEL_VOL_CM": LABEL_VOL_CM, - "LABEL_HEALTHCHECK": LABEL_HEALTHCHECK, - "LABEL_SAMEPOD": LABEL_SAMEPOD, - "LABEL_EMPTYDIRS": LABEL_EMPTYDIRS, - }) - return buff.String() -} var ( - Appname = "" + Appname = "" // set at runtime Version = "1.0" // should be set from main.Version ) // Kinded represent an object with a kind. type Kinded interface { - // Get must resturn the kind name. Get() string } // Signable represents an object with a signature. type Signable interface { - // BuildSHA must return the signature. BuildSHA(filename string) } // Named represents an object with a name. type Named interface { - // Name must return the name of the object (from metadata). Name() string } -- 2.49.1 From f10e09eba546d1580488fe1d76daff48218adad0 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 10:23:43 +0200 Subject: [PATCH 20/47] Formatting --- helm/labels.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/labels.go b/helm/labels.go index f35f320..bf80d7d 100644 --- a/helm/labels.go +++ b/helm/labels.go @@ -22,7 +22,7 @@ const ( func GetLabelsDocumentation() string { t, _ := template.New("labels").Parse(` # Labels -{{ .LABEL_IGNORE | printf "%-33s"}}: ignore the container, it will not yied any object in the helm chart +{{.LABEL_IGNORE | printf "%-33s"}}: ignore the container, it will not yied any object in the helm chart {{.LABEL_ENV_SECRET | printf "%-33s"}}: set the given file names as a secret instead of configmap {{.LABEL_PORT | printf "%-33s"}}: set the ports to expose as a service (coma separated) {{.LABEL_INGRESS | printf "%-33s"}}: set the port to expose in an ingress (coma separated) -- 2.49.1 From 5f2f27f72b006179a3232403cddda2e0588fb1eb Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 10:44:20 +0200 Subject: [PATCH 21/47] More documentation --- generator/writers/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/writers/utils.go b/generator/writers/utils.go index b057945..5bfa607 100644 --- a/generator/writers/utils.go +++ b/generator/writers/utils.go @@ -1,6 +1,6 @@ package writers -// IndentSize set the indentation size for yaml output. +// IndentSize set the indentation size for yaml output. Could ba changed by command line argument. var IndentSize = 2 // CountSpaces returns the number of spaces from the begining of the line. -- 2.49.1 From 0a20be9660307e4085c3a98b950c6b382a043a6d Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 10:44:43 +0200 Subject: [PATCH 22/47] Better file closing --- generator/writer.go | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/generator/writer.go b/generator/writer.go index de9e54b..56a0e58 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -45,6 +45,8 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi // Manage services, avoid linked pods and store all services port in servicesMap avoids := make(map[string]bool) + + // Manage services to not export skips := make(map[string]bool) for _, s := range p.Data.Services { @@ -190,17 +192,24 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi } } // Create the values.yaml file - fp, _ := os.Create(filepath.Join(dirName, "values.yaml")) - enc := yaml.NewEncoder(fp) - enc.SetIndent(2) + valueFile, err := os.Create(filepath.Join(dirName, "values.yaml")) + if err != nil { + log.Fatal(err) + } + defer valueFile.Close() + enc := yaml.NewEncoder(valueFile) + enc.SetIndent(writers.IndentSize) enc.Encode(Values) - fp.Close() // Create tht Chart.yaml file - fp, _ = os.Create(filepath.Join(dirName, "Chart.yaml")) - fp.WriteString(`# Create on ` + time.Now().Format(time.RFC3339) + "\n") - fp.WriteString(`# Katenary command line: ` + strings.Join(os.Args, " ") + "\n") - enc = yaml.NewEncoder(fp) + chartFile, err := os.Create(filepath.Join(dirName, "Chart.yaml")) + if err != nil { + log.Fatal(err) + } + defer chartFile.Close() + chartFile.WriteString(`# Create on ` + time.Now().Format(time.RFC3339) + "\n") + chartFile.WriteString(`# Katenary command line: ` + strings.Join(os.Args, " ") + "\n") + enc = yaml.NewEncoder(chartFile) enc.SetIndent(writers.IndentSize) enc.Encode(map[string]interface{}{ "apiVersion": "v2", @@ -210,10 +219,12 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi "version": "0.1.0", "appVersion": appVersion, }) - fp.Close() // And finally, create a NOTE.txt file - fp, _ = os.Create(filepath.Join(templatesDir, "NOTES.txt")) - fp.WriteString(helm.GenerateNotesFile(ingresses)) - fp.Close() + noteFile, err := os.Create(filepath.Join(templatesDir, "NOTES.txt")) + if err != nil { + log.Fatal(err) + } + defer noteFile.Close() + noteFile.WriteString(helm.GenerateNotesFile(ingresses)) } -- 2.49.1 From 71befefea8393487da45e9504334dab6c0be1eac Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 10:45:41 +0200 Subject: [PATCH 23/47] Better file closing --- generator/writer.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/generator/writer.go b/generator/writer.go index 56a0e58..c60e5f8 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -183,11 +183,14 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi default: fname := filepath.Join(templatesDir, n+"."+kind+".yaml") - fp, _ := os.Create(fname) + fp, err := os.Create(fname) + if err != nil { + log.Fatal(err) + } + defer fp.Close() enc := yaml.NewEncoder(fp) enc.SetIndent(2) enc.Encode(c) - fp.Close() } } } -- 2.49.1 From 710b46061d22ff285920cd571be7451de6bd4a7e Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 10:46:22 +0200 Subject: [PATCH 24/47] Forgotten to use configured indent size --- generator/writer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/writer.go b/generator/writer.go index c60e5f8..85ed9fb 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -189,7 +189,7 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi } defer fp.Close() enc := yaml.NewEncoder(fp) - enc.SetIndent(2) + enc.SetIndent(writers.IndentSize) enc.Encode(c) } } -- 2.49.1 From ddcc3d00d3deda3635f66c666f2b9d7e67d0f53b Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 10:48:14 +0200 Subject: [PATCH 25/47] Remove debug log --- generator/writer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/generator/writer.go b/generator/writer.go index 85ed9fb..da36d76 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -88,7 +88,6 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi if portExists(target, service.Ports) { continue } - log.Println("Add port to service:", n, target) service.Ports = append(service.Ports, types.ServicePortConfig{ Target: uint32(target), }) -- 2.49.1 From bfe6738348ea89ed663950ea5cf1f702d99540cc Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 11:33:02 +0200 Subject: [PATCH 26/47] Use pointer for Value, change RELEASE_NAME const --- generator/main.go | 19 +++++++++---------- generator/main_test.go | 4 ++-- helm/configAndSecretMap.go | 4 ++-- helm/container.go | 8 ++++---- helm/deployment.go | 2 +- helm/ingress.go | 2 +- helm/k8sbase.go | 2 +- helm/labels.go | 2 +- helm/service.go | 2 +- helm/storage.go | 2 +- 10 files changed, 23 insertions(+), 24 deletions(-) diff --git a/generator/main.go b/generator/main.go index a442ec2..d35bf03 100644 --- a/generator/main.go +++ b/generator/main.go @@ -26,10 +26,6 @@ const ( ICON_INGRESS = "🌐" ) -const ( - RELEASE_NAME = helm.RELEASE_NAME -) - // Values is kept in memory to create a values.yaml file. var ( Values = make(map[string]map[string]interface{}) @@ -44,7 +40,7 @@ OK=0 echo "Checking __service__ port" while [ $OK != 1 ]; do echo -n "." - nc -z ` + RELEASE_NAME + `-__service__ __port__ 2>&1 >/dev/null && OK=1 || sleep 1 + nc -z ` + helm.ReleaseNameTpl + `-__service__ __port__ 2>&1 >/dev/null && OK=1 || sleep 1 done echo echo "Done" @@ -70,7 +66,7 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se prepareContainer(container, s, name) prepareEnvFromFiles(name, s, container, ret) - // Set the container to the deployment + // Set the containers to the deployment deployment.Spec.Template.Spec.Containers = []*helm.Container{container} // Prepare volumes @@ -208,7 +204,7 @@ func createIngress(name string, port int, s types.ServiceConfig) *helm.Ingress { PathType: "Prefix", Backend: &helm.IngressBackend{ Service: helm.IngressService{ - Name: RELEASE_NAME + "-" + name, + Name: helm.ReleaseNameTpl + "-" + name, Port: map[string]interface{}{ "number": port, }, @@ -227,7 +223,7 @@ func createIngress(name string, port int, s types.ServiceConfig) *helm.Ingress { func buildSelector(name string, s types.ServiceConfig) map[string]string { return map[string]string{ "katenary.io/component": name, - "katenary.io/release": RELEASE_NAME, + "katenary.io/release": helm.ReleaseNameTpl, } } @@ -355,7 +351,7 @@ func prepareVolumes(deployment, name string, s types.ServiceConfig, container *h volname = strings.Replace(volname, "./", "", 1) volname = strings.ReplaceAll(volname, "/", "-") volname = strings.ReplaceAll(volname, ".", "-") - cm.K8sBase.Metadata.Name = RELEASE_NAME + "-" + volname + "-" + name + cm.K8sBase.Metadata.Name = helm.ReleaseNameTpl + "-" + volname + "-" + name // build a configmap from the volume path volumes = append(volumes, map[string]interface{}{ @@ -405,7 +401,7 @@ func prepareVolumes(deployment, name string, s types.ServiceConfig, container *h volumes = append(volumes, map[string]interface{}{ "name": volname, "persistentVolumeClaim": map[string]string{ - "claimName": RELEASE_NAME + "-" + volname, + "claimName": helm.ReleaseNameTpl + "-" + volname, }, }) mountPoints = append(mountPoints, map[string]interface{}{ @@ -499,6 +495,7 @@ func prepareProbes(name string, s types.ServiceConfig, container *helm.Container } } +// buildProtoProbe builds a probe from a url that can be http or tcp. func buildProtoProbe(u *url.URL) *helm.Probe { probe := helm.NewProbe(0, 0, 0, 0) port, err := strconv.Atoi(u.Port()) @@ -616,6 +613,7 @@ func prepareEnvFromFiles(name string, s types.ServiceConfig, container *helm.Con } } +// AddValues adds values to the values.yaml map. func AddValues(servicename string, values map[string]interface{}) { locker.Lock() defer locker.Unlock() @@ -630,6 +628,7 @@ func AddValues(servicename string, values map[string]interface{}) { } +// AddVolumeValues add a volume to the values.yaml map for the given deployment name. func AddVolumeValues(deployment string, volname string, values map[string]interface{}) { locker.Lock() defer locker.Unlock() diff --git a/generator/main_test.go b/generator/main_test.go index 080591a..2342cf6 100644 --- a/generator/main_test.go +++ b/generator/main_test.go @@ -227,8 +227,8 @@ func TestEnvs(t *testing.T) { } if next { matched = true - if !strings.Contains(line, helm.RELEASE_NAME+"-database") { - t.Error("DB_HOST variable should be set to " + helm.RELEASE_NAME + "-database") + if !strings.Contains(line, helm.ReleaseNameTpl+"-database") { + t.Error("DB_HOST variable should be set to " + helm.ReleaseNameTpl + "-database") } break } diff --git a/helm/configAndSecretMap.go b/helm/configAndSecretMap.go index 1ae3205..90b6b10 100644 --- a/helm/configAndSecretMap.go +++ b/helm/configAndSecretMap.go @@ -24,7 +24,7 @@ func NewConfigMap(name string) *ConfigMap { base := NewBase() base.ApiVersion = "v1" base.Kind = "ConfigMap" - base.Metadata.Name = RELEASE_NAME + "-" + name + base.Metadata.Name = ReleaseNameTpl + "-" + name base.Metadata.Labels[K+"/component"] = name return &ConfigMap{ K8sBase: base, @@ -72,7 +72,7 @@ func NewSecret(name string) *Secret { base := NewBase() base.ApiVersion = "v1" base.Kind = "Secret" - base.Metadata.Name = RELEASE_NAME + "-" + name + base.Metadata.Name = ReleaseNameTpl + "-" + name base.Metadata.Labels[K+"/component"] = name return &Secret{ K8sBase: base, diff --git a/helm/container.go b/helm/container.go index 06bb567..9c3cb52 100644 --- a/helm/container.go +++ b/helm/container.go @@ -23,7 +23,7 @@ type Container struct { Name string `yaml:"name,omitempty"` Image string `yaml:"image"` Ports []*ContainerPort `yaml:"ports,omitempty"` - Env []Value `yaml:"env,omitempty"` + Env []*Value `yaml:"env,omitempty"` EnvFrom []map[string]map[string]string `yaml:"envFrom,omitempty"` Command []string `yaml:"command,omitempty"` VolumeMounts []interface{} `yaml:"volumeMounts,omitempty"` @@ -35,7 +35,7 @@ func NewContainer(name, image string, environment types.MappingWithEquals, label container := &Container{ Image: image, Name: name, - Env: make([]Value, len(environment)), + Env: make([]*Value, len(environment)), EnvFrom: make([]map[string]map[string]string, 0), } @@ -49,10 +49,10 @@ func NewContainer(name, image string, environment types.MappingWithEquals, label for n, v := range environment { for _, name := range toServices { if name == n { - *v = RELEASE_NAME + "-" + *v + *v = ReleaseNameTpl + "-" + *v } } - container.Env[idx] = Value{Name: n, Value: v} + container.Env[idx] = &Value{Name: n, Value: v} idx++ } return container diff --git a/helm/deployment.go b/helm/deployment.go index bf5ff8e..d1dec8a 100644 --- a/helm/deployment.go +++ b/helm/deployment.go @@ -8,7 +8,7 @@ type Deployment struct { func NewDeployment(name string) *Deployment { d := &Deployment{K8sBase: NewBase(), Spec: NewDepSpec()} - d.K8sBase.Metadata.Name = RELEASE_NAME + "-" + name + d.K8sBase.Metadata.Name = ReleaseNameTpl + "-" + name d.K8sBase.ApiVersion = "apps/v1" d.K8sBase.Kind = "Deployment" d.K8sBase.Metadata.Labels[K+"/component"] = name diff --git a/helm/ingress.go b/helm/ingress.go index 88c8608..f2ad8e8 100644 --- a/helm/ingress.go +++ b/helm/ingress.go @@ -9,7 +9,7 @@ type Ingress struct { func NewIngress(name string) *Ingress { i := &Ingress{} i.K8sBase = NewBase() - i.K8sBase.Metadata.Name = RELEASE_NAME + "-" + name + i.K8sBase.Metadata.Name = ReleaseNameTpl + "-" + name i.K8sBase.Kind = "Ingress" i.ApiVersion = "networking.k8s.io/v1" i.K8sBase.Metadata.Labels[K+"/component"] = name diff --git a/helm/k8sbase.go b/helm/k8sbase.go index af2aa58..4f54188 100644 --- a/helm/k8sbase.go +++ b/helm/k8sbase.go @@ -35,7 +35,7 @@ func NewBase() *K8sBase { } // add some information of the build b.Metadata.Labels[K+"/project"] = GetProjectName() - b.Metadata.Labels[K+"/release"] = RELEASE_NAME + b.Metadata.Labels[K+"/release"] = ReleaseNameTpl b.Metadata.Annotations[K+"/version"] = Version return b } diff --git a/helm/labels.go b/helm/labels.go index bf80d7d..bbab538 100644 --- a/helm/labels.go +++ b/helm/labels.go @@ -5,7 +5,7 @@ import ( "html/template" ) -const RELEASE_NAME = "{{ .Release.Name }}" +const ReleaseNameTpl = "{{ .Release.Name }}" const ( LABEL_ENV_SECRET = K + "/secret-envfiles" LABEL_PORT = K + "/ports" diff --git a/helm/service.go b/helm/service.go index 2e00ca5..f1a7ec1 100644 --- a/helm/service.go +++ b/helm/service.go @@ -12,7 +12,7 @@ func NewService(name string) *Service { K8sBase: NewBase(), Spec: NewServiceSpec(), } - s.K8sBase.Metadata.Name = RELEASE_NAME + "-" + name + s.K8sBase.Metadata.Name = ReleaseNameTpl + "-" + name s.K8sBase.Kind = "Service" s.K8sBase.ApiVersion = "v1" s.K8sBase.Metadata.Labels[K+"/component"] = name diff --git a/helm/storage.go b/helm/storage.go index 491748e..f0a14de 100644 --- a/helm/storage.go +++ b/helm/storage.go @@ -13,7 +13,7 @@ func NewPVC(name, storageName string) *Storage { pvc.K8sBase.Kind = "PersistentVolumeClaim" pvc.K8sBase.Metadata.Labels[K+"/pvc-name"] = storageName pvc.K8sBase.ApiVersion = "v1" - pvc.K8sBase.Metadata.Name = RELEASE_NAME + "-" + storageName + pvc.K8sBase.Metadata.Name = ReleaseNameTpl + "-" + storageName pvc.K8sBase.Metadata.Labels[K+"/component"] = name pvc.Spec = &PVCSpec{ Resouces: map[string]interface{}{ -- 2.49.1 From d72ab144005eeab86b3bbb077866976e1760a234 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 12:16:04 +0200 Subject: [PATCH 27/47] Better Probe builder - we didn't get probe configuration from compose, it's now fixed - the HealthCheck object is tricky because it overrides `time.Duration`, we are using specific initialization in NewProbeWithDuration - it is easier to pass the ServiceConfig to NewProbeFromService --- generator/main.go | 11 +++++---- helm/probe.go | 63 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/generator/main.go b/generator/main.go index d35bf03..90ef1e5 100644 --- a/generator/main.go +++ b/generator/main.go @@ -471,14 +471,15 @@ func prepareProbes(name string, s types.ServiceConfig, container *helm.Container // first, check if there is no label for the probe if check, ok := s.Labels[helm.LABEL_HEALTHCHECK]; ok { check = strings.TrimSpace(check) + p := helm.NewProbeFromService(&s) // get the port of the "url" check if checkurl, err := url.Parse(check); err == nil { if err == nil { - container.LivenessProbe = buildProtoProbe(checkurl) + container.LivenessProbe = buildProtoProbe(p, checkurl) } } else { // it's a command - container.LivenessProbe = helm.NewProbe(0, 0, 0, 0) + container.LivenessProbe = p container.LivenessProbe.Exec = &helm.Exec{ Command: []string{ "sh", @@ -496,8 +497,7 @@ func prepareProbes(name string, s types.ServiceConfig, container *helm.Container } // buildProtoProbe builds a probe from a url that can be http or tcp. -func buildProtoProbe(u *url.URL) *helm.Probe { - probe := helm.NewProbe(0, 0, 0, 0) +func buildProtoProbe(probe *helm.Probe, u *url.URL) *helm.Probe { port, err := strconv.Atoi(u.Port()) if err != nil { port = 80 @@ -529,7 +529,8 @@ func buildCommandProbe(s types.ServiceConfig) *helm.Probe { // Get the first element of the command from ServiceConfig first := s.HealthCheck.Test[0] - p := helm.NewProbe(0, 0, 0, 0) + + p := helm.NewProbeFromService(&s) switch first { case "CMD", "CMD-SHELL": // CMD or CMD-SHELL diff --git a/helm/probe.go b/helm/probe.go index 210ae1d..bc80e4b 100644 --- a/helm/probe.go +++ b/helm/probe.go @@ -1,18 +1,24 @@ package helm +import ( + "time" + + "github.com/compose-spec/compose-go/types" +) + // Probe is a struct that can be used to create a Liveness or Readiness probe. type Probe struct { HttpGet *HttpGet `yaml:"httpGet,omitempty"` Exec *Exec `yaml:"exec,omitempty"` TCP *TCP `yaml:"tcp,omitempty"` - Period int `yaml:"periodSeconds"` - Success int `yaml:"successThreshold"` - Failure int `yaml:"failureThreshold"` - InitialDelay int `yaml:"initialDelaySeconds"` + Period float64 `yaml:"periodSeconds"` + InitialDelay float64 `yaml:"initialDelaySeconds"` + Success uint64 `yaml:"successThreshold"` + Failure uint64 `yaml:"failureThreshold"` } // Create a new Probe object that can be apply to HttpProbe or TCPProbe. -func NewProbe(period, initialDelaySeconds, success, failure int) *Probe { +func NewProbe(period, initialDelaySeconds float64, success, failure uint64) *Probe { probe := &Probe{ Period: period, Success: success, @@ -34,6 +40,53 @@ func NewProbe(period, initialDelaySeconds, success, failure int) *Probe { return probe } +// NewProbeWithDuration creates a new Probe object with the given duration from types. +func NewProbeWithDuration(period, initialDelaySeconds *types.Duration, success, failure *uint64) *Probe { + + if period == nil { + d := types.Duration(0 * time.Second) + period = &d + } + + if initialDelaySeconds == nil { + d := types.Duration(0 * time.Second) + initialDelaySeconds = &d + } + + if success == nil { + s := uint64(0) + success = &s + } + + if failure == nil { + f := uint64(0) + failure = &f + } + + p, err := time.ParseDuration(period.String()) + if err != nil { + p = time.Second * 10 + } + + i, err := time.ParseDuration(initialDelaySeconds.String()) + if err != nil { + i = time.Second * 0 + } + + return NewProbe(p.Seconds(), i.Seconds(), *success, *failure) + +} + +// NewProbeFromService creates a new Probe object from a ServiceConfig. +func NewProbeFromService(s *types.ServiceConfig) *Probe { + if s == nil || s.HealthCheck == nil { + return NewProbe(0, 0, 0, 0) + } + + return NewProbeWithDuration(s.HealthCheck.Interval, s.HealthCheck.StartPeriod, nil, s.HealthCheck.Retries) + +} + // HttpGet is a Probe configuration to check http health. type HttpGet struct { Path string `yaml:"path"` -- 2.49.1 From ee5a18ae86e6adb43877e66539c63382e13ecf30 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 12:21:09 +0200 Subject: [PATCH 28/47] Formatting and comment --- generator/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/generator/main.go b/generator/main.go index 90ef1e5..f00e66d 100644 --- a/generator/main.go +++ b/generator/main.go @@ -468,7 +468,7 @@ func prepareInitContainers(name string, s types.ServiceConfig, container *helm.C // prepareProbes generate http/tcp/command probes for a service. func prepareProbes(name string, s types.ServiceConfig, container *helm.Container) { - // first, check if there is no label for the probe + // first, check if there a label for the probe if check, ok := s.Labels[helm.LABEL_HEALTHCHECK]; ok { check = strings.TrimSpace(check) p := helm.NewProbeFromService(&s) @@ -490,6 +490,7 @@ func prepareProbes(name string, s types.ServiceConfig, container *helm.Container } return // label overrides everything } + // if not, we will use the default one if s.HealthCheck != nil { container.LivenessProbe = buildCommandProbe(s) -- 2.49.1 From fe451574396f1cba3b358bd0df77e97a46871973 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 13:22:39 +0200 Subject: [PATCH 29/47] Fixed Values that was badly initialized --- generator/main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/generator/main.go b/generator/main.go index f00e66d..41d2ae6 100644 --- a/generator/main.go +++ b/generator/main.go @@ -620,14 +620,13 @@ func AddValues(servicename string, values map[string]interface{}) { locker.Lock() defer locker.Unlock() - if _, ok := values[servicename]; !ok { + if _, ok := Values[servicename]; !ok { Values[servicename] = make(map[string]interface{}) } for k, v := range values { Values[servicename][k] = v } - } // AddVolumeValues add a volume to the values.yaml map for the given deployment name. -- 2.49.1 From dbe9fc25ea9f52199147972de35daed41742df24 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 13:55:33 +0200 Subject: [PATCH 30/47] Add mapenv label that is more agnostic - that means that katenary.io/env-to-service is now DEPRECATED - the yaml style in label is OK, that allows more possibilities and adaptation for users --- generator/main.go | 30 ++++++++++++++++++++++++++++++ helm/container.go | 12 ++++++++++++ helm/labels.go | 9 +++++++-- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/generator/main.go b/generator/main.go index 41d2ae6..f0a8aea 100644 --- a/generator/main.go +++ b/generator/main.go @@ -15,6 +15,7 @@ import ( "sync" "github.com/compose-spec/compose-go/types" + "gopkg.in/yaml.v3" ) const ( @@ -52,6 +53,7 @@ echo "Done" // Create a Deployment for a given compose.Service. It returns a list of objects: a Deployment and a possible Service (kubernetes represnetation as maps). func CreateReplicaObject(name string, s types.ServiceConfig, linked map[string]types.ServiceConfig) chan interface{} { ret := make(chan interface{}, len(s.Ports)+len(s.Expose)+2) + rebuildEnvMap(&s) go parseService(name, s, linked, ret) return ret } @@ -659,3 +661,31 @@ func readEnvFile(envfilename string) map[string]string { } return env } + +// rebuildEnvMap will get all LABEL_MAP_ENV to rebuild the env map with tpl. +func rebuildEnvMap(s *types.ServiceConfig) { + mapenv, ok := s.Labels[helm.LABEL_MAP_ENV] + if !ok { + return + } + + // the mapenv is a YAML string + var envmap map[string]string + err := yaml.Unmarshal([]byte(mapenv), &envmap) + if err != nil { + logger.ActivateColors = true + logger.Red(err.Error()) + logger.ActivateColors = false + return + } + + // rebuild the env map + for k, v := range envmap { + _, ok := s.Environment[k] + if !ok { + continue + } + s.Environment[k] = &v + } + +} diff --git a/helm/container.go b/helm/container.go index 9c3cb52..fbc480a 100644 --- a/helm/container.go +++ b/helm/container.go @@ -1,6 +1,7 @@ package helm import ( + "katenary/logger" "strings" "github.com/compose-spec/compose-go/types" @@ -44,6 +45,17 @@ func NewContainer(name, image string, environment types.MappingWithEquals, label if bound, ok := labels[LABEL_ENV_SERVICE]; ok { toServices = strings.Split(bound, ",") } + if len(toServices) > 0 { + // warn, it's deprecated now + logger.ActivateColors = true + logger.Yellowf( + "[deprecated] in \"%s\" service: label %s is deprecated, please use %s instead\n", + name, + LABEL_ENV_SERVICE, + LABEL_MAP_ENV, + ) + logger.ActivateColors = false + } idx := 0 for n, v := range environment { diff --git a/helm/labels.go b/helm/labels.go index bbab538..cb7ee99 100644 --- a/helm/labels.go +++ b/helm/labels.go @@ -7,15 +7,18 @@ import ( const ReleaseNameTpl = "{{ .Release.Name }}" const ( + LABEL_MAP_ENV = K + "/mapenv" LABEL_ENV_SECRET = K + "/secret-envfiles" LABEL_PORT = K + "/ports" LABEL_INGRESS = K + "/ingress" - LABEL_ENV_SERVICE = K + "/env-to-service" LABEL_VOL_CM = K + "/configmap-volumes" LABEL_HEALTHCHECK = K + "/healthcheck" LABEL_SAMEPOD = K + "/same-pod" LABEL_EMPTYDIRS = K + "/empty-dirs" LABEL_IGNORE = K + "/ignore" + + //deprecated: use LABEL_MAP_ENV instead + LABEL_ENV_SERVICE = K + "/env-to-service" ) // GetLabelsDocumentation returns the documentation for the labels. @@ -24,9 +27,10 @@ func GetLabelsDocumentation() string { # Labels {{.LABEL_IGNORE | printf "%-33s"}}: ignore the container, it will not yied any object in the helm chart {{.LABEL_ENV_SECRET | printf "%-33s"}}: set the given file names as a secret instead of configmap +{{.LABEL_MAP_ENV | printf "%-33s"}}: map environment variable to a template string (yaml style) {{.LABEL_PORT | printf "%-33s"}}: set the ports to expose as a service (coma separated) {{.LABEL_INGRESS | printf "%-33s"}}: set the port to expose in an ingress (coma separated) -{{.LABEL_ENV_SERVICE | printf "%-33s"}}: specifies that the environment variable points on a service name (coma separated) +{{.LABEL_ENV_SERVICE | printf "%-33s"}}: DEPRECATED use {{ .LABEL_MAP_ENV }} instead - specifies that the environment variable points on a service name (coma separated) {{.LABEL_VOL_CM | printf "%-33s"}}: specifies that the volumes points on a configmap (coma separated) {{.LABEL_SAMEPOD | printf "%-33s"}}: specifies that the pod should be deployed in the same pod than the given service name {{.LABEL_EMPTYDIRS | printf "%-33s"}}: specifies that the given volume names should be "emptyDir" instead of persistentVolumeClaim (coma separated) @@ -47,6 +51,7 @@ func GetLabelsDocumentation() string { "LABEL_SAMEPOD": LABEL_SAMEPOD, "LABEL_EMPTYDIRS": LABEL_EMPTYDIRS, "LABEL_IGNORE": LABEL_IGNORE, + "LABEL_MAP_ENV": LABEL_MAP_ENV, }) return buff.String() } -- 2.49.1 From 28694603569734793f4bad3b13beae5d58992ad0 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 14:04:09 +0200 Subject: [PATCH 31/47] The mapenv label, documented, fixed --- README.md | 12 ++++++++---- generator/main.go | 12 ++++-------- helm/labels.go | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 6375138..05e8940 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ This project is partially made at [Smile](https://www.smile.eu) You can download the binaries from the [Release](https://github.com/metal3d/katenary/releases) section. Copy the binary and rename it to `katenary`. Place the binary inside your `PATH`. You should now be able to call the `katenary` command. +You can of course get the binary with `go install -u github.com/metal3d/katenary/cmd/katenary/...` but the `main` branch is continuously updated. It's preferable to use releases. You can use this commands on Linux: @@ -125,7 +126,7 @@ What can be interpreted by Katenary: - `env_file` list will create a configMap object per environemnt file (⚠ todo: the "to-service" label doesn't work with configMap for now) - some labels can help to bind values, for example: - `katenary.io/ingress: 80` will expose the port 80 in a ingress - - `katenary.io/env-to-service: VARNAME` will convert the value to a variable `{{ .Release.Name }}-VARNAME` - it's usefull when you want to pass the name of a service as a variable (think about the service name for mysql to pass to a container that wants to connect to this) + - `katenary.io/mapenv: |`: allow to map environment to something else than the given value in the compose file Exemple of a possible `docker-compose.yaml` file: @@ -144,10 +145,12 @@ services: # because it's the "exposed" port - database labels: - # explain to katenary that "DB_HOST" value is variable (using release name) - katenary.io/env-to-service: DB_HOST # expose the port 80 as an ingress katenary.io/ingress: 80 + # make adaptations, DB_HOST environment is actually the service name + # to hit (note the yaml style, start with "|") + katenary.io/mapenv: | + DB_HOST: {{ .Release.Name }}-database database: image: mariadb:10 env_file: @@ -168,9 +171,9 @@ These labels could be found by `katenary show-labels`, and can be placed as "lab ``` katenary.io/ignore : ignore the container, it will not yied any object in the helm chart katenary.io/secret-envfiles : set the given file names as a secret instead of configmap +katenary.io/mapenv : map environment variable to a template string (yaml style) katenary.io/ports : set the ports to expose as a service (coma separated) katenary.io/ingress : set the port to expose in an ingress (coma separated) -katenary.io/env-to-service : specifies that the environment variable points on a service name (coma separated) katenary.io/configmap-volumes : specifies that the volumes points on a configmap (coma separated) katenary.io/same-pod : specifies that the pod should be deployed in the same pod than the given service name katenary.io/empty-dirs : specifies that the given volume names should be "emptyDir" instead of persistentVolumeClaim (coma separated) @@ -179,6 +182,7 @@ katenary.io/healthcheck : specifies that the container should be monito - "http://[not used address][:port][/path]" to specify an http healthcheck - "tcp://[not used address]:port" to specify a tcp healthcheck - other string is condidered as a "command" healthcheck +katenary.io/env-to-service : DEPRECATED use katenary.io/mapenv instead - specifies that the environment variable points on a service name (coma separated) ``` # What a name... diff --git a/generator/main.go b/generator/main.go index f0a8aea..28c4be5 100644 --- a/generator/main.go +++ b/generator/main.go @@ -53,7 +53,7 @@ echo "Done" // Create a Deployment for a given compose.Service. It returns a list of objects: a Deployment and a possible Service (kubernetes represnetation as maps). func CreateReplicaObject(name string, s types.ServiceConfig, linked map[string]types.ServiceConfig) chan interface{} { ret := make(chan interface{}, len(s.Ports)+len(s.Expose)+2) - rebuildEnvMap(&s) + applyEnvMapLabel(&s) go parseService(name, s, linked, ret) return ret } @@ -662,8 +662,8 @@ func readEnvFile(envfilename string) map[string]string { return env } -// rebuildEnvMap will get all LABEL_MAP_ENV to rebuild the env map with tpl. -func rebuildEnvMap(s *types.ServiceConfig) { +// applyEnvMapLabel will get all LABEL_MAP_ENV to rebuild the env map with tpl. +func applyEnvMapLabel(s *types.ServiceConfig) { mapenv, ok := s.Labels[helm.LABEL_MAP_ENV] if !ok { return @@ -679,12 +679,8 @@ func rebuildEnvMap(s *types.ServiceConfig) { return } - // rebuild the env map + // add in envmap for k, v := range envmap { - _, ok := s.Environment[k] - if !ok { - continue - } s.Environment[k] = &v } diff --git a/helm/labels.go b/helm/labels.go index cb7ee99..0abad59 100644 --- a/helm/labels.go +++ b/helm/labels.go @@ -30,7 +30,6 @@ func GetLabelsDocumentation() string { {{.LABEL_MAP_ENV | printf "%-33s"}}: map environment variable to a template string (yaml style) {{.LABEL_PORT | printf "%-33s"}}: set the ports to expose as a service (coma separated) {{.LABEL_INGRESS | printf "%-33s"}}: set the port to expose in an ingress (coma separated) -{{.LABEL_ENV_SERVICE | printf "%-33s"}}: DEPRECATED use {{ .LABEL_MAP_ENV }} instead - specifies that the environment variable points on a service name (coma separated) {{.LABEL_VOL_CM | printf "%-33s"}}: specifies that the volumes points on a configmap (coma separated) {{.LABEL_SAMEPOD | printf "%-33s"}}: specifies that the pod should be deployed in the same pod than the given service name {{.LABEL_EMPTYDIRS | printf "%-33s"}}: specifies that the given volume names should be "emptyDir" instead of persistentVolumeClaim (coma separated) @@ -39,6 +38,7 @@ func GetLabelsDocumentation() string { {{ printf "%-35s" ""}}- "http://[not used address][:port][/path]" to specify an http healthcheck {{ printf "%-35s" ""}}- "tcp://[not used address]:port" to specify a tcp healthcheck {{ printf "%-35s" ""}}- other string is condidered as a "command" healthcheck +{{.LABEL_ENV_SERVICE | printf "%-33s"}}: DEPRECATED use {{ .LABEL_MAP_ENV }} instead - specifies that the environment variable points on a service name (coma separated) `) buff := bytes.NewBuffer(nil) t.Execute(buff, map[string]string{ -- 2.49.1 From a8565b55a7f5d8e824d12fe8a3088fec24e109ad Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 14:20:19 +0200 Subject: [PATCH 32/47] Fix examples --- examples/basic/README.md | 2 +- examples/basic/docker-compose.yaml | 6 ++- examples/ghost/README.md | 9 ++++ examples/ghost/chart/ghost/Chart.yaml | 8 ++++ .../ghost/chart/ghost/templates/NOTES.txt | 8 ++++ .../ghost/templates/blog.deployment.yaml | 33 +++++++++++++++ .../chart/ghost/templates/blog.ingress.yaml | 42 +++++++++++++++++++ .../chart/ghost/templates/blog.service.yaml | 19 +++++++++ examples/ghost/chart/ghost/values.yaml | 6 +++ examples/ghost/docker-compose.yaml | 30 +++++++++++++ 10 files changed, 160 insertions(+), 3 deletions(-) create mode 100644 examples/ghost/README.md create mode 100644 examples/ghost/chart/ghost/Chart.yaml create mode 100644 examples/ghost/chart/ghost/templates/NOTES.txt create mode 100644 examples/ghost/chart/ghost/templates/blog.deployment.yaml create mode 100644 examples/ghost/chart/ghost/templates/blog.ingress.yaml create mode 100644 examples/ghost/chart/ghost/templates/blog.service.yaml create mode 100644 examples/ghost/chart/ghost/values.yaml create mode 100644 examples/ghost/docker-compose.yaml diff --git a/examples/basic/README.md b/examples/basic/README.md index 893b7ee..1b3974a 100644 --- a/examples/basic/README.md +++ b/examples/basic/README.md @@ -5,6 +5,6 @@ This is a basic example of what can do Katenary with standard docker-compose fil In this example: - `depends_on` yield a `initContainer` in the webapp ddeployment to wait for database -- so we need to declare the listened port inside `database` container as we don't use it with docker-compose- also, we needed to declare that `DB_HOST` is actually a service name +- so we need to declare the listened port inside `database` container as we don't use it with docker-compose- also, we needed to declare that `DB_HOST` is actually a service name using `mapenv` label Take a look on [chart/basic](chart/basic) directory to see what `katenary convert` command has generated. diff --git a/examples/basic/docker-compose.yaml b/examples/basic/docker-compose.yaml index ce9dd35..3e9b3fc 100644 --- a/examples/basic/docker-compose.yaml +++ b/examples/basic/docker-compose.yaml @@ -1,7 +1,8 @@ version: "3" +# this example is absolutely not working, it's an example to see how it is converted +# by Katenary services: - webapp: image: php:7-apache environment: @@ -12,7 +13,8 @@ services: # expose an ingress katenary.io/ingress: 80 # DB_HOST is actually a service name - katenary.io/env-to-service: DB_HOST + katenary.io/mapenv: | + DB_HOST: {{ .Release.Name }}-database depends_on: - database diff --git a/examples/ghost/README.md b/examples/ghost/README.md new file mode 100644 index 0000000..58135d4 --- /dev/null +++ b/examples/ghost/README.md @@ -0,0 +1,9 @@ +# Example with Ghost + +[Ghost](https://ghost.org/) is a simple but powerfull blog engine. It is very nice to test some behaviors with Docker or Podman. + +The given `docker-compose.yaml` file here declares a stand-alone blog service. To help using it, we use [Patwae](https://pathwae.net) reverse-proxy to listend http://ghost.example.localhost + +The problem to solve is that the `url` environment variable correspond to the Ingress host when we will convert it to Helm Chart. So, we use the `mapenv` label to declare that `url` is actually `{{ .Values.blog.ingress.host }}` value. + +Note that we also `ignore` pathwae because we don't need it in our Helm Chart. diff --git a/examples/ghost/chart/ghost/Chart.yaml b/examples/ghost/chart/ghost/Chart.yaml new file mode 100644 index 0000000..f4732b0 --- /dev/null +++ b/examples/ghost/chart/ghost/Chart.yaml @@ -0,0 +1,8 @@ +# Create on 2022-05-05T14:16:27+02:00 +# Katenary command line: /tmp/go-build669507924/b001/exe/main convert +apiVersion: v2 +appVersion: 0.0.1 +description: A helm chart for ghost +name: ghost +type: application +version: 0.1.0 diff --git a/examples/ghost/chart/ghost/templates/NOTES.txt b/examples/ghost/chart/ghost/templates/NOTES.txt new file mode 100644 index 0000000..10ce5b3 --- /dev/null +++ b/examples/ghost/chart/ghost/templates/NOTES.txt @@ -0,0 +1,8 @@ + +Congratulations, + +Your application is now deployed. This may take a while to be up and responding. + +{{ if .Values.blog.ingress.enabled -}} +- blog is accessible on : http://{{ .Values.blog.ingress.host }} +{{- end }} diff --git a/examples/ghost/chart/ghost/templates/blog.deployment.yaml b/examples/ghost/chart/ghost/templates/blog.deployment.yaml new file mode 100644 index 0000000..6378e0d --- /dev/null +++ b/examples/ghost/chart/ghost/templates/blog.deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: '{{ .Release.Name }}-blog' + labels: + katenary.io/component: blog + katenary.io/project: ghost + katenary.io/release: '{{ .Release.Name }}' + annotations: + katenary.io/docker-compose-sha1: 0c2bbf548ff569c3dc5d77dc158e98bbe86fb5d4 + katenary.io/version: master +spec: + replicas: 1 + selector: + matchLabels: + katenary.io/component: blog + katenary.io/release: '{{ .Release.Name }}' + template: + metadata: + labels: + katenary.io/component: blog + katenary.io/release: '{{ .Release.Name }}' + spec: + containers: + - name: blog + image: '{{ .Values.blog.image }}' + ports: + - name: blog + containerPort: 2368 + env: + - name: url + value: http://{{ .Values.blog.ingress.host }} + diff --git a/examples/ghost/chart/ghost/templates/blog.ingress.yaml b/examples/ghost/chart/ghost/templates/blog.ingress.yaml new file mode 100644 index 0000000..43c804d --- /dev/null +++ b/examples/ghost/chart/ghost/templates/blog.ingress.yaml @@ -0,0 +1,42 @@ +{{- if .Values.blog.ingress.enabled -}} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: '{{ .Release.Name }}-blog' + labels: + katenary.io/component: blog + katenary.io/project: ghost + katenary.io/release: '{{ .Release.Name }}' + annotations: + katenary.io/docker-compose-sha1: 0c2bbf548ff569c3dc5d77dc158e98bbe86fb5d4 + katenary.io/version: master +spec: + {{- if and .Values.blog.ingress.class (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: '{{ .Values.blog.ingress.class }}' + {{- end }} + rules: + - host: '{{ .Values.blog.ingress.host }}' + http: + paths: + - path: / + {{- if semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} + service: + name: '{{ .Release.Name }}-blog' + port: + number: 2368 + {{- else }} + serviceName: '{{ .Release.Name }}-blog' + servicePort: 2368 + {{- end }} + +{{- end -}} diff --git a/examples/ghost/chart/ghost/templates/blog.service.yaml b/examples/ghost/chart/ghost/templates/blog.service.yaml new file mode 100644 index 0000000..5c54299 --- /dev/null +++ b/examples/ghost/chart/ghost/templates/blog.service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: '{{ .Release.Name }}-blog' + labels: + katenary.io/component: blog + katenary.io/project: ghost + katenary.io/release: '{{ .Release.Name }}' + annotations: + katenary.io/docker-compose-sha1: 0c2bbf548ff569c3dc5d77dc158e98bbe86fb5d4 + katenary.io/version: master +spec: + selector: + katenary.io/component: blog + katenary.io/release: '{{ .Release.Name }}' + ports: + - protocol: TCP + port: 2368 + targetPort: 2368 diff --git a/examples/ghost/chart/ghost/values.yaml b/examples/ghost/chart/ghost/values.yaml new file mode 100644 index 0000000..6ef57af --- /dev/null +++ b/examples/ghost/chart/ghost/values.yaml @@ -0,0 +1,6 @@ +blog: + image: ghost + ingress: + class: nginx + enabled: false + host: blog.ghost.tld diff --git a/examples/ghost/docker-compose.yaml b/examples/ghost/docker-compose.yaml new file mode 100644 index 0000000..67472f7 --- /dev/null +++ b/examples/ghost/docker-compose.yaml @@ -0,0 +1,30 @@ +version: "3" + +services: + blog: + image: ghost + environment: + # this is OK for local test, but not with Helm + # because the URL depends on Ingress + url: http://ghost.example.localhost + labels: + katenary.io/ports: 2368 + katenary.io/ingress: 2368 + # ... so we declare that "url" is actually + # the ingress host + katenary.io/mapenv: | + url: http://{{ .Values.blog.ingress.host }} + + proxy: + # A simple proxy for localhost + image: quay.io/pathwae/proxy + environment: + CONFIG: | + ghost.example.localhost: + to: http://blog:2368 + ports: + - 80:80 + labels: + # we don't want this in Helm because we will use + # an ingress + katenary.io/ignore: true -- 2.49.1 From 1e8f417418519b6b57ee2bf53c131befda5e8e34 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Thu, 5 May 2022 19:02:13 +0200 Subject: [PATCH 33/47] Set environment variables to values.yaml --- generator/main.go | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/generator/main.go b/generator/main.go index 28c4be5..0499cd3 100644 --- a/generator/main.go +++ b/generator/main.go @@ -53,7 +53,6 @@ echo "Done" // Create a Deployment for a given compose.Service. It returns a list of objects: a Deployment and a possible Service (kubernetes represnetation as maps). func CreateReplicaObject(name string, s types.ServiceConfig, linked map[string]types.ServiceConfig) chan interface{} { ret := make(chan interface{}, len(s.Ports)+len(s.Expose)+2) - applyEnvMapLabel(&s) go parseService(name, s, linked, ret) return ret } @@ -62,8 +61,11 @@ func CreateReplicaObject(name string, s types.ServiceConfig, linked map[string]t func parseService(name string, s types.ServiceConfig, linked map[string]types.ServiceConfig, ret chan interface{}) { logger.Magenta(ICON_PACKAGE+" Generating deployment for ", name) - deployment := helm.NewDeployment(name) + // adapt env + applyEnvMapLabel(&s) + setEnvToValues(name, &s) + deployment := helm.NewDeployment(name) container := helm.NewContainer(name, s.Image, s.Environment, s.Labels) prepareContainer(container, s, name) prepareEnvFromFiles(name, s, container, ret) @@ -87,6 +89,8 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se // Now, the linked services for lname, link := range linked { + applyEnvMapLabel(&link) + setEnvToValues(lname, &link) container := helm.NewContainer(lname, link.Image, link.Environment, link.Labels) prepareContainer(container, link, lname) prepareEnvFromFiles(lname, link, container, ret) @@ -683,5 +687,19 @@ func applyEnvMapLabel(s *types.ServiceConfig) { for k, v := range envmap { s.Environment[k] = &v } - +} + +// setEnvToValues will set the environment variables to the values.yaml map. +func setEnvToValues(name string, s *types.ServiceConfig) { + // crete the "environment" key + env := make(map[string]interface{}) + for k, v := range s.Environment { + env[k] = v + } + + AddValues(name, map[string]interface{}{"environment": env}) + for k := range s.Environment { + v := "{{ .Values." + name + ".environment." + k + " }}" + s.Environment[k] = &v + } } -- 2.49.1 From 1b9ac9ca8a0ebecd4a1b9978aff2bf21c80e491f Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Fri, 6 May 2022 09:55:12 +0200 Subject: [PATCH 34/47] Changed behavio on environment - variables are all in values.yaml as "template string". This means that we can now set values to reference others (useful with mapenv label) - we can also set any variable as a secret --- generator/main.go | 79 +++++++++++++++++++++++++++++++++----- generator/main_test.go | 16 ++++---- helm/configAndSecretMap.go | 12 +++++- helm/labels.go | 3 ++ 4 files changed, 92 insertions(+), 18 deletions(-) diff --git a/generator/main.go b/generator/main.go index 0499cd3..2a5fc35 100644 --- a/generator/main.go +++ b/generator/main.go @@ -59,14 +59,27 @@ func CreateReplicaObject(name string, s types.ServiceConfig, linked map[string]t // This function will try to yied deployment and services based on a service from the compose file structure. func parseService(name string, s types.ServiceConfig, linked map[string]types.ServiceConfig, ret chan interface{}) { + // TODO: this function is a mess, need a complete refactorisation + logger.Magenta(ICON_PACKAGE+" Generating deployment for ", name) + deployment := helm.NewDeployment(name) // adapt env applyEnvMapLabel(&s) - setEnvToValues(name, &s) - deployment := helm.NewDeployment(name) + // create a container for the deployment. container := helm.NewContainer(name, s.Image, s.Environment, s.Labels) + + // If some variables are secrets, set them now ! + if secretFile := setSecretVar(name, &s, container); secretFile != nil { + ret <- secretFile + container.EnvFrom = append(container.EnvFrom, map[string]map[string]string{ + "secretRef": { + "name": secretFile.Metadata().Name, + }, + }) + } + setEnvToValues(name, &s, container) prepareContainer(container, s, name) prepareEnvFromFiles(name, s, container, ret) @@ -90,8 +103,18 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se // Now, the linked services for lname, link := range linked { applyEnvMapLabel(&link) - setEnvToValues(lname, &link) container := helm.NewContainer(lname, link.Image, link.Environment, link.Labels) + + // If some variables are secrets, set them now ! + if secretFile := setSecretVar(lname, &link, container); secretFile != nil { + ret <- secretFile + container.EnvFrom = append(container.EnvFrom, map[string]map[string]string{ + "secretRef": { + "name": secretFile.Metadata().Name, + }, + }) + } + setEnvToValues(lname, &link, container) prepareContainer(container, link, lname) prepareEnvFromFiles(lname, link, container, ret) deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, container) @@ -321,23 +344,23 @@ func prepareVolumes(deployment, name string, s types.ServiceConfig, container *h continue } - isCM := false + isConfigMap := false for _, cmVol := range configMapsVolumes { cmVol = strings.TrimSpace(cmVol) if volname == cmVol { - isCM = true + isConfigMap = true break } } - if !isCM && (strings.HasPrefix(volname, ".") || strings.HasPrefix(volname, "/")) { + if !isConfigMap && (strings.HasPrefix(volname, ".") || strings.HasPrefix(volname, "/")) { // local volume cannt be mounted logger.ActivateColors = true logger.Redf("You cannot, at this time, have local volume in %s deployment\n", name) logger.ActivateColors = false continue } - if isCM { + if isConfigMap { // check if the volname path points on a file, if so, we need to add subvolume to the interface stat, err := os.Stat(volname) if err != nil { @@ -690,16 +713,52 @@ func applyEnvMapLabel(s *types.ServiceConfig) { } // setEnvToValues will set the environment variables to the values.yaml map. -func setEnvToValues(name string, s *types.ServiceConfig) { +func setEnvToValues(name string, s *types.ServiceConfig, c *helm.Container) { // crete the "environment" key + env := make(map[string]interface{}) for k, v := range s.Environment { env[k] = v } + if len(env) == 0 { + return + } AddValues(name, map[string]interface{}{"environment": env}) - for k := range s.Environment { - v := "{{ .Values." + name + ".environment." + k + " }}" + for k := range env { + v := "{{ tpl .Values." + name + ".environment." + k + " . }}" s.Environment[k] = &v + for _, c := range c.Env { + if c.Name == k { + c.Value = v + } + } } } + +func setSecretVar(name string, s *types.ServiceConfig, c *helm.Container) *helm.Secret { + // get the list of secret vars + secretvars, ok := s.Labels[helm.LABEL_SECRETVARS] + if !ok { + return nil + } + + store := helm.NewSecret(name + "-secrets") + for _, secretvar := range strings.Split(secretvars, ",") { + // get the value from env + _, ok := s.Environment[secretvar] + if !ok { + continue + } + // add the secret + store.AddEnv(secretvar, ".Values."+name+".environment."+secretvar) + envs := c.Env + for i, env := range envs { + if env.Name == secretvar { + envs = append(envs[:i], envs[i+1:]...) + } + } + c.Env = envs + } + return store +} diff --git a/generator/main_test.go b/generator/main_test.go index 2342cf6..c02f17f 100644 --- a/generator/main_test.go +++ b/generator/main_test.go @@ -50,7 +50,8 @@ services: ANOTHER_ENV_VAR: another_value DB_HOST: database labels: - katenary.io/env-to-service: DB_HOST + katenary.io/mapenv: | + DB_HOST: {{ .Release.Name }}-database database: image: mysql:5.7 @@ -221,20 +222,20 @@ func TestEnvs(t *testing.T) { lines, _ := ioutil.ReadAll(fp) next := false for _, line := range strings.Split(string(lines), "\n") { - if strings.Contains(line, "DB_HOST") { + if !next && strings.Contains(line, "name: DB_HOST") { next = true continue - } - if next { + } else if next && strings.Contains(line, "value:") { matched = true - if !strings.Contains(line, helm.ReleaseNameTpl+"-database") { - t.Error("DB_HOST variable should be set to " + helm.ReleaseNameTpl + "-database") + if !strings.Contains(line, "{{ tpl .Values.php.environment.DB_HOST . }}") { + t.Error("DB_HOST variable should be set to {{ tpl .Values.php.environment.DB_HOST . }}", line, string(lines)) } break } } if !matched { t.Error("DB_HOST variable not found in ", path) + t.Log(string(lines)) } } } @@ -382,8 +383,9 @@ func TestEqualSignOnEnv(t *testing.T) { match++ } } - if match != 2 { + if match != 4 { // because the value points on .Values... t.Error("eqenv service should have 2 environment variables") + t.Log(string(lines)) } } } diff --git a/helm/configAndSecretMap.go b/helm/configAndSecretMap.go index 90b6b10..b319257 100644 --- a/helm/configAndSecretMap.go +++ b/helm/configAndSecretMap.go @@ -10,6 +10,7 @@ import ( // InlineConfig is made to represent a configMap or a secret type InlineConfig interface { AddEnvFile(filename string) error + AddEnv(key, val string) error Metadata() *Metadata } @@ -56,9 +57,12 @@ func (c *ConfigMap) AddEnvFile(file string) error { } c.Data[parts[0]] = parts[1] } - return nil +} +func (c *ConfigMap) AddEnv(key, val string) error { + c.Data[key] = val + return nil } // Secret is made to represent a secret with data. @@ -108,3 +112,9 @@ func (s *Secret) AddEnvFile(file string) error { func (s *Secret) Metadata() *Metadata { return s.K8sBase.Metadata } + +// AddEnv adds an environment variable to the secret. +func (s *Secret) AddEnv(key, val string) error { + s.Data[key] = fmt.Sprintf(`{{ %s | b64enc }}`, val) + return nil +} diff --git a/helm/labels.go b/helm/labels.go index 0abad59..6fd824d 100644 --- a/helm/labels.go +++ b/helm/labels.go @@ -16,6 +16,7 @@ const ( LABEL_SAMEPOD = K + "/same-pod" LABEL_EMPTYDIRS = K + "/empty-dirs" LABEL_IGNORE = K + "/ignore" + LABEL_SECRETVARS = K + "/secret-vars" //deprecated: use LABEL_MAP_ENV instead LABEL_ENV_SERVICE = K + "/env-to-service" @@ -26,6 +27,7 @@ func GetLabelsDocumentation() string { t, _ := template.New("labels").Parse(` # Labels {{.LABEL_IGNORE | printf "%-33s"}}: ignore the container, it will not yied any object in the helm chart +{{.LABEL_SECRETVARS | printf "%-33s"}}: secret variables to push on a secret file {{.LABEL_ENV_SECRET | printf "%-33s"}}: set the given file names as a secret instead of configmap {{.LABEL_MAP_ENV | printf "%-33s"}}: map environment variable to a template string (yaml style) {{.LABEL_PORT | printf "%-33s"}}: set the ports to expose as a service (coma separated) @@ -52,6 +54,7 @@ func GetLabelsDocumentation() string { "LABEL_EMPTYDIRS": LABEL_EMPTYDIRS, "LABEL_IGNORE": LABEL_IGNORE, "LABEL_MAP_ENV": LABEL_MAP_ENV, + "LABEL_SECRETVARS": LABEL_SECRETVARS, }) return buff.String() } -- 2.49.1 From 2721cb113680cefbdcc13143f3a4110f0d684df7 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Fri, 6 May 2022 13:42:21 +0200 Subject: [PATCH 35/47] Fixed updated versions --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 82378f6..6685316 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/compose-spec/compose-go v1.2.4 - github.com/distribution/distribution/v3 v3.0.0-20220504180456-7a6b9e3042bd // indirect + github.com/distribution/distribution/v3 v3.0.0-20220505155552-985711c1f414 // indirect github.com/kr/pretty v0.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/spf13/cobra v1.4.0 diff --git a/go.sum b/go.sum index e448930..153f8ab 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,8 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/distribution/distribution/v3 v3.0.0-20210316161203-a01c71e2477e/go.mod h1:xpWTC2KnJMiDLkoawhsPQcXjvwATEBcbq0xevG2YR9M= github.com/distribution/distribution/v3 v3.0.0-20220504180456-7a6b9e3042bd h1:KRoLSsR7wZ4H2dueR/O6BGBIXDxfOxUVmaMiu1QiQPw= github.com/distribution/distribution/v3 v3.0.0-20220504180456-7a6b9e3042bd/go.mod h1:qLi7jGj1b5TUaYTB3ekkHiocxHJi8+3CFhXM54MGKBs= +github.com/distribution/distribution/v3 v3.0.0-20220505155552-985711c1f414 h1:KfVB1Z5fm10trO24Rn5Zzocd8sTm5k/gS24ijxQ1aJU= +github.com/distribution/distribution/v3 v3.0.0-20220505155552-985711c1f414/go.mod h1:2oyLKljQFnsI1tzJxjUg4GI+HEpDfzFP3LrGM04rKg0= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -- 2.49.1 From 3031f37509cd6bb8cfb04d63a820da97bb9ef183 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Fri, 6 May 2022 13:42:53 +0200 Subject: [PATCH 36/47] Refactorisation and improvements - image of container is now splitted in repository and tag (in values and deployment). The "tag" is tested in deployment - add "chart-version" option Code: - globalize the PVC generation - ensure types for environment values - refactored to make generic the container creation in a deployment - avoiding race condition on ServiceConfig by using a copy (then a pointer of this copy) --- cmd/katenary/main.go | 5 +- cmd/katenary/utils.go | 5 +- generator/main.go | 243 +++++++++++++++++++++-------------- generator/main_test.go | 9 +- generator/writer.go | 21 +-- generator/writers/storage.go | 11 +- helm/container.go | 6 +- helm/k8sbase.go | 2 + helm/storage.go | 21 +++ 9 files changed, 208 insertions(+), 115 deletions(-) diff --git a/cmd/katenary/main.go b/cmd/katenary/main.go index 1b0df68..712f057 100644 --- a/cmd/katenary/main.go +++ b/cmd/katenary/main.go @@ -65,18 +65,21 @@ func main() { appversion := c.Flag("app-version").Value.String() composeFile := c.Flag("compose-file").Value.String() appName := c.Flag("app-name").Value.String() + chartVersion := c.Flag("chart-version").Value.String() chartDir := c.Flag("output-dir").Value.String() indentation, err := strconv.Atoi(c.Flag("indent-size").Value.String()) if err != nil { writers.IndentSize = indentation } - Convert(composeFile, appversion, appName, chartDir, force) + Convert(composeFile, appversion, appName, chartDir, chartVersion, force) }, } convertCmd.Flags().BoolP( "force", "f", false, "force overwrite of existing output files") convertCmd.Flags().StringP( "app-version", "a", AppVersion, "app version") + convertCmd.Flags().StringP( + "chart-version", "v", ChartVersion, "chart version") convertCmd.Flags().StringP( "compose-file", "c", ComposeFile, "docker compose file") convertCmd.Flags().StringP( diff --git a/cmd/katenary/utils.go b/cmd/katenary/utils.go index 6eaeda2..cead5d8 100644 --- a/cmd/katenary/utils.go +++ b/cmd/katenary/utils.go @@ -17,6 +17,7 @@ var ( AppName = "MyApp" ChartsDir = "chart" AppVersion = "0.0.1" + ChartVersion = "0.1.0" ) func init() { @@ -92,7 +93,7 @@ func detectGitVersion() (string, error) { return defaulVersion, errors.New("git log failed") } -func Convert(composeFile, appVersion, appName, chartDir string, force bool) { +func Convert(composeFile, appVersion, appName, chartDir, chartVersion string, force bool) { if len(composeFile) == 0 { fmt.Println("No compose file given") return @@ -138,6 +139,6 @@ func Convert(composeFile, appVersion, appName, chartDir string, force bool) { } // start generator - generator.Generate(p, Version, appName, appVersion, ComposeFile, dirname) + generator.Generate(p, Version, appName, appVersion, chartVersion, ComposeFile, dirname) } diff --git a/generator/main.go b/generator/main.go index 2a5fc35..f2d92cb 100644 --- a/generator/main.go +++ b/generator/main.go @@ -10,6 +10,7 @@ import ( "net/url" "os" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -18,6 +19,8 @@ import ( "gopkg.in/yaml.v3" ) +type EnvVal = helm.EnvValue + const ( ICON_PACKAGE = "📦" ICON_SERVICE = "🔌" @@ -29,12 +32,11 @@ const ( // Values is kept in memory to create a values.yaml file. var ( - Values = make(map[string]map[string]interface{}) - VolumeValues = make(map[string]map[string]map[string]interface{}) - EmptyDirs = []string{} - servicesMap = make(map[string]int) - serviceWaiters = make(map[string][]chan int) - locker = &sync.Mutex{} + Values = make(map[string]map[string]interface{}) + VolumeValues = make(map[string]map[string]map[string]EnvVal) + EmptyDirs = []string{} + servicesMap = make(map[string]int) + locker = &sync.Mutex{} dependScript = ` OK=0 @@ -50,48 +52,22 @@ echo "Done" madeDeployments = make(map[string]helm.Deployment, 0) ) -// Create a Deployment for a given compose.Service. It returns a list of objects: a Deployment and a possible Service (kubernetes represnetation as maps). -func CreateReplicaObject(name string, s types.ServiceConfig, linked map[string]types.ServiceConfig) chan interface{} { - ret := make(chan interface{}, len(s.Ports)+len(s.Expose)+2) - go parseService(name, s, linked, ret) +// Create a Deployment for a given compose.Service. It returns a list chan +// of HelmFileGenerator which will be used to generate the files (deployment, secrets, configMap...). +func CreateReplicaObject(name string, s types.ServiceConfig, linked map[string]types.ServiceConfig) HelmFileGenerator { + ret := make(chan HelmFile, runtime.NumCPU()) + // there is a bug woth typs.ServiceConfig if we use the pointer. So we need to dereference it. + go buildDeployment(name, &s, linked, ret) return ret } // This function will try to yied deployment and services based on a service from the compose file structure. -func parseService(name string, s types.ServiceConfig, linked map[string]types.ServiceConfig, ret chan interface{}) { - // TODO: this function is a mess, need a complete refactorisation +func buildDeployment(name string, s *types.ServiceConfig, linked map[string]types.ServiceConfig, fileGeneratorChan HelmFileGenerator) { logger.Magenta(ICON_PACKAGE+" Generating deployment for ", name) deployment := helm.NewDeployment(name) - // adapt env - applyEnvMapLabel(&s) - - // create a container for the deployment. - container := helm.NewContainer(name, s.Image, s.Environment, s.Labels) - - // If some variables are secrets, set them now ! - if secretFile := setSecretVar(name, &s, container); secretFile != nil { - ret <- secretFile - container.EnvFrom = append(container.EnvFrom, map[string]map[string]string{ - "secretRef": { - "name": secretFile.Metadata().Name, - }, - }) - } - setEnvToValues(name, &s, container) - prepareContainer(container, s, name) - prepareEnvFromFiles(name, s, container, ret) - - // Set the containers to the deployment - deployment.Spec.Template.Spec.Containers = []*helm.Container{container} - - // Prepare volumes - madePVC := make(map[string]bool) - deployment.Spec.Template.Spec.Volumes = prepareVolumes(name, name, s, container, madePVC, ret) - - // Now, for "depends_on" section, it's a bit tricky to get dependencies, see the function below. - deployment.Spec.Template.Spec.InitContainers = prepareInitContainers(name, s, container) + newContainerForDeployment(name, name, deployment, s, fileGeneratorChan) // Add selectors selectors := buildSelector(name, s) @@ -100,27 +76,11 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se } deployment.Spec.Template.Metadata.Labels = selectors - // Now, the linked services + // Now, the linked services (same pod) for lname, link := range linked { - applyEnvMapLabel(&link) - container := helm.NewContainer(lname, link.Image, link.Environment, link.Labels) - - // If some variables are secrets, set them now ! - if secretFile := setSecretVar(lname, &link, container); secretFile != nil { - ret <- secretFile - container.EnvFrom = append(container.EnvFrom, map[string]map[string]string{ - "secretRef": { - "name": secretFile.Metadata().Name, - }, - }) - } - setEnvToValues(lname, &link, container) - prepareContainer(container, link, lname) - prepareEnvFromFiles(lname, link, container, ret) - deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, container) - deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, prepareVolumes(name, lname, link, container, madePVC, ret)...) - deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, prepareInitContainers(lname, link, container)...) - //append ports and expose ports to the deployment, to be able to generate them in the Service file + newContainerForDeployment(name, lname, deployment, &link, fileGeneratorChan) + // append ports and expose ports to the deployment, + // to be able to generate them in the Service file if len(link.Ports) > 0 || len(link.Expose) > 0 { s.Ports = append(s.Ports, link.Ports...) s.Expose = append(s.Expose, link.Expose...) @@ -144,39 +104,57 @@ func parseService(name string, s types.ServiceConfig, linked map[string]types.Se // Then, create Services and possible Ingresses for ingress labels, "ports" and "expose" section if len(s.Ports) > 0 || len(s.Expose) > 0 { for _, s := range generateServicesAndIngresses(name, s) { - ret <- s + if s != nil { + fileGeneratorChan <- s + } } } // add the volumes in Values if len(VolumeValues[name]) > 0 { - AddValues(name, map[string]interface{}{"persistence": VolumeValues[name]}) + AddValues(name, map[string]EnvVal{"persistence": VolumeValues[name]}) } // the deployment is ready, give it - ret <- deployment + fileGeneratorChan <- deployment // and then, we can say that it's the end - ret <- nil + fileGeneratorChan <- nil } // prepareContainer assigns image, command, env, and labels to a container. -func prepareContainer(container *helm.Container, service types.ServiceConfig, servicename string) { +func prepareContainer(container *helm.Container, service *types.ServiceConfig, servicename string) { // if there is no image name, this should fail! if service.Image == "" { log.Fatal(ICON_PACKAGE+" No image name for service ", servicename) } - container.Image = "{{ .Values." + servicename + ".image }}" + + // Get the image tag + imageParts := strings.Split(service.Image, ":") + tag := "" + if len(imageParts) == 2 { + container.Image = imageParts[0] + tag = imageParts[1] + } + + vtag := ".Values." + servicename + ".repository.tag" + container.Image = `{{ .Values.` + servicename + `.repository.image }}` + + `{{ if ne ` + vtag + ` "" }}:{{ ` + vtag + ` }}{{ end }}` container.Command = service.Command - AddValues(servicename, map[string]interface{}{"image": service.Image}) + AddValues(servicename, map[string]EnvVal{ + "repository": map[string]EnvVal{ + "image": imageParts[0], + "tag": tag, + }, + }) prepareProbes(servicename, service, container) generateContainerPorts(service, servicename, container) } // Create a service (k8s). -func generateServicesAndIngresses(name string, s types.ServiceConfig) []interface{} { +func generateServicesAndIngresses(name string, s *types.ServiceConfig) []HelmFile { - ret := make([]interface{}, 0) // can handle helm.Service or helm.Ingress + ret := make([]HelmFile, 0) // can handle helm.Service or helm.Ingress logger.Magenta(ICON_SERVICE+" Generating service for ", name) ks := helm.NewService(name) @@ -214,7 +192,7 @@ func generateServicesAndIngresses(name string, s types.ServiceConfig) []interfac } // Create an ingress. -func createIngress(name string, port int, s types.ServiceConfig) *helm.Ingress { +func createIngress(name string, port int, s *types.ServiceConfig) *helm.Ingress { ingress := helm.NewIngress(name) ingressVal := map[string]interface{}{ @@ -222,7 +200,7 @@ func createIngress(name string, port int, s types.ServiceConfig) *helm.Ingress { "host": name + "." + helm.Appname + ".tld", "enabled": false, } - AddValues(name, map[string]interface{}{"ingress": ingressVal}) + AddValues(name, map[string]EnvVal{"ingress": ingressVal}) ingress.Spec.Rules = []helm.IngressRule{ { @@ -249,7 +227,7 @@ func createIngress(name string, port int, s types.ServiceConfig) *helm.Ingress { } // Build the selector for the service. -func buildSelector(name string, s types.ServiceConfig) map[string]string { +func buildSelector(name string, s *types.ServiceConfig) map[string]string { return map[string]string{ "katenary.io/component": name, "katenary.io/release": helm.ReleaseNameTpl, @@ -290,7 +268,7 @@ func buildCMFromPath(path string) *helm.ConfigMap { } // generateContainerPorts add the container ports of a service. -func generateContainerPorts(s types.ServiceConfig, name string, container *helm.Container) { +func generateContainerPorts(s *types.ServiceConfig, name string, container *helm.Container) { exists := make(map[int]string) for _, port := range s.Ports { @@ -323,7 +301,7 @@ func generateContainerPorts(s types.ServiceConfig, name string, container *helm. } // prepareVolumes add the volumes of a service. -func prepareVolumes(deployment, name string, s types.ServiceConfig, container *helm.Container, madePVC map[string]bool, ret chan interface{}) []map[string]interface{} { +func prepareVolumes(deployment, name string, s *types.ServiceConfig, container *helm.Container, fileGeneratorChan HelmFileGenerator) []map[string]interface{} { volumes := make([]map[string]interface{}, 0) mountPoints := make([]interface{}, 0) @@ -401,7 +379,9 @@ func prepareVolumes(deployment, name string, s types.ServiceConfig, container *h "mountPath": volepath, }) } - ret <- cm + if cm != nil { + fileGeneratorChan <- cm + } } else { // rmove minus sign from volume name volname = strings.ReplaceAll(volname, "-", "") @@ -439,15 +419,13 @@ func prepareVolumes(deployment, name string, s types.ServiceConfig, container *h }) logger.Yellow(ICON_STORE+" Generate volume values", volname, "for container named", name, "in deployment", deployment) - AddVolumeValues(deployment, volname, map[string]interface{}{ + AddVolumeValues(deployment, volname, map[string]EnvVal{ "enabled": false, "capacity": "1Gi", }) - if _, ok := madePVC[deployment+volname]; !ok { - madePVC[deployment+volname] = true - pvc := helm.NewPVC(deployment, volname) - ret <- pvc + if pvc := helm.NewPVC(deployment, volname); pvc != nil { + fileGeneratorChan <- pvc } } } @@ -456,7 +434,7 @@ func prepareVolumes(deployment, name string, s types.ServiceConfig, container *h } // prepareInitContainers add the init containers of a service. -func prepareInitContainers(name string, s types.ServiceConfig, container *helm.Container) []*helm.Container { +func prepareInitContainers(name string, s *types.ServiceConfig, container *helm.Container) []*helm.Container { // We need to detect others services, but we probably not have parsed them yet, so // we will wait for them for a while. @@ -496,11 +474,11 @@ func prepareInitContainers(name string, s types.ServiceConfig, container *helm.C } // prepareProbes generate http/tcp/command probes for a service. -func prepareProbes(name string, s types.ServiceConfig, container *helm.Container) { +func prepareProbes(name string, s *types.ServiceConfig, container *helm.Container) { // first, check if there a label for the probe if check, ok := s.Labels[helm.LABEL_HEALTHCHECK]; ok { check = strings.TrimSpace(check) - p := helm.NewProbeFromService(&s) + p := helm.NewProbeFromService(s) // get the port of the "url" check if checkurl, err := url.Parse(check); err == nil { if err == nil { @@ -555,12 +533,12 @@ func buildProtoProbe(probe *helm.Probe, u *url.URL) *helm.Probe { return probe } -func buildCommandProbe(s types.ServiceConfig) *helm.Probe { +func buildCommandProbe(s *types.ServiceConfig) *helm.Probe { // Get the first element of the command from ServiceConfig first := s.HealthCheck.Test[0] - p := helm.NewProbeFromService(&s) + p := helm.NewProbeFromService(s) switch first { case "CMD", "CMD-SHELL": // CMD or CMD-SHELL @@ -578,7 +556,7 @@ func buildCommandProbe(s types.ServiceConfig) *helm.Probe { } // prepareEnvFromFiles generate configMap or secrets from environment files. -func prepareEnvFromFiles(name string, s types.ServiceConfig, container *helm.Container, ret chan interface{}) { +func prepareEnvFromFiles(name string, s *types.ServiceConfig, container *helm.Container, fileGeneratorChan HelmFileGenerator) { // prepare secrets secretsFiles := make([]string, 0) @@ -640,12 +618,14 @@ func prepareEnvFromFiles(name string, s types.ServiceConfig, container *helm.Con } } - ret <- store + if store != nil { + fileGeneratorChan <- store + } } } // AddValues adds values to the values.yaml map. -func AddValues(servicename string, values map[string]interface{}) { +func AddValues(servicename string, values map[string]EnvVal) { locker.Lock() defer locker.Unlock() @@ -659,18 +639,18 @@ func AddValues(servicename string, values map[string]interface{}) { } // AddVolumeValues add a volume to the values.yaml map for the given deployment name. -func AddVolumeValues(deployment string, volname string, values map[string]interface{}) { +func AddVolumeValues(deployment string, volname string, values map[string]EnvVal) { locker.Lock() defer locker.Unlock() if _, ok := VolumeValues[deployment]; !ok { - VolumeValues[deployment] = make(map[string]map[string]interface{}) + VolumeValues[deployment] = make(map[string]map[string]EnvVal) } VolumeValues[deployment][volname] = values } -func readEnvFile(envfilename string) map[string]string { - env := make(map[string]string) +func readEnvFile(envfilename string) map[string]EnvVal { + env := make(map[string]EnvVal) content, err := ioutil.ReadFile(envfilename) if err != nil { logger.ActivateColors = true @@ -690,14 +670,17 @@ func readEnvFile(envfilename string) map[string]string { } // applyEnvMapLabel will get all LABEL_MAP_ENV to rebuild the env map with tpl. -func applyEnvMapLabel(s *types.ServiceConfig) { +func applyEnvMapLabel(s *types.ServiceConfig, c *helm.Container) { + + locker.Lock() + defer locker.Unlock() mapenv, ok := s.Labels[helm.LABEL_MAP_ENV] if !ok { return } // the mapenv is a YAML string - var envmap map[string]string + var envmap map[string]EnvVal err := yaml.Unmarshal([]byte(mapenv), &envmap) if err != nil { logger.ActivateColors = true @@ -708,7 +691,18 @@ func applyEnvMapLabel(s *types.ServiceConfig) { // add in envmap for k, v := range envmap { - s.Environment[k] = &v + vstring := fmt.Sprintf("%v", v) + s.Environment[k] = &vstring + touched := false + for _, env := range c.Env { + if env.Name == k { + env.Value = v + touched = true + } + } + if !touched { + c.Env = append(c.Env, &helm.Value{Name: k, Value: v}) + } } } @@ -716,7 +710,7 @@ func applyEnvMapLabel(s *types.ServiceConfig) { func setEnvToValues(name string, s *types.ServiceConfig, c *helm.Container) { // crete the "environment" key - env := make(map[string]interface{}) + env := make(map[string]EnvVal) for k, v := range s.Environment { env[k] = v } @@ -724,19 +718,26 @@ func setEnvToValues(name string, s *types.ServiceConfig, c *helm.Container) { return } - AddValues(name, map[string]interface{}{"environment": env}) + AddValues(name, map[string]EnvVal{"environment": env}) for k := range env { v := "{{ tpl .Values." + name + ".environment." + k + " . }}" s.Environment[k] = &v + touched := false for _, c := range c.Env { if c.Name == k { c.Value = v + touched = true } } + if !touched { + c.Env = append(c.Env, &helm.Value{Name: k, Value: v}) + } } } func setSecretVar(name string, s *types.ServiceConfig, c *helm.Container) *helm.Secret { + locker.Lock() + defer locker.Unlock() // get the list of secret vars secretvars, ok := s.Labels[helm.LABEL_SECRETVARS] if !ok { @@ -762,3 +763,51 @@ func setSecretVar(name string, s *types.ServiceConfig, c *helm.Container) *helm. } return store } + +// Generate a container in deployment with all needed objects (volumes, secrets, env, ...). +// The deployName shoud be the name of the deployment, we cannot get it from Metadata as this is a variable name. +func newContainerForDeployment(deployName, containerName string, deployment *helm.Deployment, s *types.ServiceConfig, fileGeneratorChan HelmFileGenerator) *helm.Container { + container := helm.NewContainer(containerName, s.Image, s.Environment, s.Labels) + + applyEnvMapLabel(s, container) + if secretFile := setSecretVar(containerName, s, container); secretFile != nil { + fileGeneratorChan <- secretFile + container.EnvFrom = append(container.EnvFrom, map[string]map[string]string{ + "secretRef": { + "name": secretFile.Metadata().Name, + }, + }) + } + setEnvToValues(containerName, s, container) + prepareContainer(container, s, deployName) + prepareEnvFromFiles(deployName, s, container, fileGeneratorChan) + + // add the container in deployment + if deployment.Spec.Template.Spec.Containers == nil { + deployment.Spec.Template.Spec.Containers = make([]*helm.Container, 0) + } + deployment.Spec.Template.Spec.Containers = append( + deployment.Spec.Template.Spec.Containers, + container, + ) + + // add the volumes + if deployment.Spec.Template.Spec.Volumes == nil { + deployment.Spec.Template.Spec.Volumes = make([]map[string]interface{}, 0) + } + deployment.Spec.Template.Spec.Volumes = append( + deployment.Spec.Template.Spec.Volumes, + prepareVolumes(deployName, containerName, s, container, fileGeneratorChan)..., + ) + + // add init containers + if deployment.Spec.Template.Spec.InitContainers == nil { + deployment.Spec.Template.Spec.InitContainers = make([]*helm.Container, 0) + } + deployment.Spec.Template.Spec.InitContainers = append( + deployment.Spec.Template.Spec.InitContainers, + prepareInitContainers(containerName, s, container)..., + ) + + return container +} diff --git a/generator/main_test.go b/generator/main_test.go index c02f17f..7a57006 100644 --- a/generator/main_test.go +++ b/generator/main_test.go @@ -98,7 +98,6 @@ services: volumes: data: - driver: local ` var defaultCliFiles = cli.DefaultFileNames @@ -110,6 +109,10 @@ func init() { } func setUp(t *testing.T) (string, *compose.Parser) { + + // cleanup "made" files + helm.ResetMadePVC() + cli.DefaultFileNames = defaultCliFiles // create a temporary directory @@ -145,7 +148,7 @@ func setUp(t *testing.T) (string, *compose.Parser) { p.Parse("testapp") - Generate(p, "test-0", "testapp", "1.2.3", DOCKER_COMPOSE_YML, tmp) + Generate(p, "test-0", "testapp", "1.2.3", "4.5.6", DOCKER_COMPOSE_YML, tmp) return tmp, p } @@ -310,6 +313,8 @@ func TestPVC(t *testing.T) { t.Log("Checking ", name, " pvc file") _, err := os.Stat(path) if err != nil { + list, _ := filepath.Glob(tmp + "/templates/*") + t.Log(list) t.Fatal(err) } } diff --git a/generator/writer.go b/generator/writer.go index da36d76..6db0bba 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -16,6 +16,9 @@ import ( "gopkg.in/yaml.v3" ) +type HelmFile interface{} +type HelmFileGenerator chan HelmFile + var PrefixRE = regexp.MustCompile(`\{\{.*\}\}-?`) func portExists(port int, ports []types.ServicePortConfig) bool { @@ -28,7 +31,7 @@ func portExists(port int, ports []types.ServicePortConfig) bool { return false } -func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFile, dirName string) { +func Generate(p *compose.Parser, katernayVersion, appName, appVersion, chartVersion, composeFile, dirName string) { // make the appname global (yes... ugly but easy) helm.Appname = appName @@ -41,7 +44,7 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi log.Fatal(err) } - files := make(map[string]chan interface{}) + files := make(map[string]HelmFileGenerator) // Manage services, avoid linked pods and store all services port in servicesMap avoids := make(map[string]bool) @@ -140,21 +143,21 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi // to generate notes, we need to keep an Ingresses list ingresses := make(map[string]*helm.Ingress) - for n, f := range files { - for c := range f { - if c == nil { + for n, generator := range files { + for helmFile := range generator { + if helmFile == nil { break } - kind := c.(helm.Kinded).Get() + kind := helmFile.(helm.Kinded).Get() kind = strings.ToLower(kind) // Add a SHA inside the generated file, it's only // to make it easy to check it the compose file corresponds to the // generated helm chart - c.(helm.Signable).BuildSHA(composeFile) + helmFile.(helm.Signable).BuildSHA(composeFile) // Some types need special fixes in yaml generation - switch c := c.(type) { + switch c := helmFile.(type) { case *helm.Storage: // For storage, we need to add a "condition" to activate it writers.BuildStorage(c, n, templatesDir) @@ -218,7 +221,7 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, composeFi "name": appName, "description": "A helm chart for " + appName, "type": "application", - "version": "0.1.0", + "version": chartVersion, "appVersion": appVersion, }) diff --git a/generator/writers/storage.go b/generator/writers/storage.go index ea3617f..2201c01 100644 --- a/generator/writers/storage.go +++ b/generator/writers/storage.go @@ -2,6 +2,7 @@ package writers import ( "katenary/helm" + "log" "os" "path/filepath" @@ -14,12 +15,18 @@ func BuildStorage(storage *helm.Storage, name, templatesDir string) { name = storage.Metadata.Labels[helm.K+"/component"] pvcname := storage.Metadata.Labels[helm.K+"/pvc-name"] fname := filepath.Join(templatesDir, name+"-"+pvcname+"."+kind+".yaml") - fp, _ := os.Create(fname) + fp, err := os.Create(fname) + if err != nil { + log.Fatal(err) + } + defer fp.Close() volname := storage.K8sBase.Metadata.Labels[helm.K+"/pvc-name"] fp.WriteString("{{ if .Values." + name + ".persistence." + volname + ".enabled }}\n") enc := yaml.NewEncoder(fp) enc.SetIndent(IndentSize) - enc.Encode(storage) + if err := enc.Encode(storage); err != nil { + log.Fatal(err) + } fp.WriteString("{{- end -}}") } diff --git a/helm/container.go b/helm/container.go index fbc480a..9bfa370 100644 --- a/helm/container.go +++ b/helm/container.go @@ -7,6 +7,8 @@ import ( "github.com/compose-spec/compose-go/types" ) +type EnvValue interface{} + // ContainerPort represent a port mapping. type ContainerPort struct { Name string @@ -15,8 +17,8 @@ type ContainerPort struct { // Value represent a environment variable with name and value. type Value struct { - Name string `yaml:"name"` - Value interface{} `yaml:"value"` + Name string `yaml:"name"` + Value EnvValue `yaml:"value"` } // Container represent a container with name, image, and environment variables. It is used in Deployment. diff --git a/helm/k8sbase.go b/helm/k8sbase.go index 4f54188..81a8b42 100644 --- a/helm/k8sbase.go +++ b/helm/k8sbase.go @@ -47,10 +47,12 @@ func (k *K8sBase) BuildSHA(filename string) { k.Metadata.Annotations[K+"/docker-compose-sha1"] = fmt.Sprintf("%x", string(sum[:])) } +// Get returns the Kind. func (k *K8sBase) Get() string { return k.Kind } +// Name returns the name of the object from Metadata. func (k *K8sBase) Name() string { return k.Metadata.Name } diff --git a/helm/storage.go b/helm/storage.go index f0a14de..e09e82f 100644 --- a/helm/storage.go +++ b/helm/storage.go @@ -1,5 +1,20 @@ package helm +import "sync" + +var ( + made = make(map[string]bool) + locker = sync.Mutex{} +) + +// ResetMadePVC resets the cache of made PVCs. +// Useful in tests only. +func ResetMadePVC() { + locker.Lock() + defer locker.Unlock() + made = make(map[string]bool) +} + // Storage is a struct for a PersistentVolumeClaim. type Storage struct { *K8sBase `yaml:",inline"` @@ -8,6 +23,12 @@ type Storage struct { // NewPVC creates a new PersistentVolumeClaim object. func NewPVC(name, storageName string) *Storage { + locker.Lock() + defer locker.Unlock() + if _, ok := made[name+storageName]; ok { + return nil + } + made[name+storageName] = true pvc := &Storage{} pvc.K8sBase = NewBase() pvc.K8sBase.Kind = "PersistentVolumeClaim" -- 2.49.1 From f6126e5ce1253090050485b10c9d5b4766407bc3 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Fri, 6 May 2022 15:13:29 +0200 Subject: [PATCH 37/47] Make it possible to first use compose.yaml files I prefer podman :p --- cmd/katenary/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/katenary/utils.go b/cmd/katenary/utils.go index cead5d8..bc0d8b4 100644 --- a/cmd/katenary/utils.go +++ b/cmd/katenary/utils.go @@ -12,7 +12,7 @@ import ( ) var ( - composeFiles = []string{"docker-compose.yaml", "docker-compose.yml"} + composeFiles = []string{"compose.yml", "compose.yaml", "docker-compose.yaml", "docker-compose.yml"} ComposeFile = "" AppName = "MyApp" ChartsDir = "chart" -- 2.49.1 From d4f89f60c9d6cbc9f5adb980c61280b5d7b45814 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Fri, 6 May 2022 15:14:57 +0200 Subject: [PATCH 38/47] Remove label "env-to-services" Deprecated and really complex to manage with the new mapenv label. --- helm/container.go | 15 ++++----------- helm/labels.go | 2 -- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/helm/container.go b/helm/container.go index 9bfa370..f202a0c 100644 --- a/helm/container.go +++ b/helm/container.go @@ -51,7 +51,10 @@ func NewContainer(name, image string, environment types.MappingWithEquals, label // warn, it's deprecated now logger.ActivateColors = true logger.Yellowf( - "[deprecated] in \"%s\" service: label %s is deprecated, please use %s instead\n", + "[deprecated] in \"%s\" service: label %s is deprecated and **ignored**, please use %s instead\n"+ + "e.g.\n"+ + " labels:\n"+ + " FOO: {{ .Release.Name }}-fooservice\n", name, LABEL_ENV_SERVICE, LABEL_MAP_ENV, @@ -59,15 +62,5 @@ func NewContainer(name, image string, environment types.MappingWithEquals, label logger.ActivateColors = false } - idx := 0 - for n, v := range environment { - for _, name := range toServices { - if name == n { - *v = ReleaseNameTpl + "-" + *v - } - } - container.Env[idx] = &Value{Name: n, Value: v} - idx++ - } return container } diff --git a/helm/labels.go b/helm/labels.go index 6fd824d..4b07921 100644 --- a/helm/labels.go +++ b/helm/labels.go @@ -40,12 +40,10 @@ func GetLabelsDocumentation() string { {{ printf "%-35s" ""}}- "http://[not used address][:port][/path]" to specify an http healthcheck {{ printf "%-35s" ""}}- "tcp://[not used address]:port" to specify a tcp healthcheck {{ printf "%-35s" ""}}- other string is condidered as a "command" healthcheck -{{.LABEL_ENV_SERVICE | printf "%-33s"}}: DEPRECATED use {{ .LABEL_MAP_ENV }} instead - specifies that the environment variable points on a service name (coma separated) `) buff := bytes.NewBuffer(nil) t.Execute(buff, map[string]string{ "LABEL_ENV_SECRET": LABEL_ENV_SECRET, - "LABEL_ENV_SERVICE": LABEL_ENV_SERVICE, "LABEL_PORT": LABEL_PORT, "LABEL_INGRESS": LABEL_INGRESS, "LABEL_VOL_CM": LABEL_VOL_CM, -- 2.49.1 From 0f5b66d209985e5e2569c25343f3f801ab0b9e07 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Fri, 6 May 2022 15:21:47 +0200 Subject: [PATCH 39/47] Trim spaces on coma separated values Maybe a function to ensure this... later... --- generator/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/generator/main.go b/generator/main.go index f2d92cb..cec719f 100644 --- a/generator/main.go +++ b/generator/main.go @@ -573,6 +573,7 @@ func prepareEnvFromFiles(name string, s *types.ServiceConfig, container *helm.Co cf := f + "-" + name isSecret := false for _, s := range secretsFiles { + s = strings.TrimSpace(s) if s == envfile { isSecret = true } @@ -746,6 +747,7 @@ func setSecretVar(name string, s *types.ServiceConfig, c *helm.Container) *helm. store := helm.NewSecret(name + "-secrets") for _, secretvar := range strings.Split(secretvars, ",") { + secretvar = strings.TrimSpace(secretvar) // get the value from env _, ok := s.Environment[secretvar] if !ok { -- 2.49.1 From baeb8a612aeb3685de0e7e6bc027948056155cc1 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Fri, 6 May 2022 15:22:33 +0200 Subject: [PATCH 40/47] Add more doc + deprecate env-to-service --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 05e8940..d7472ed 100644 --- a/README.md +++ b/README.md @@ -157,11 +157,15 @@ services: # this will create a configMap - my_env.env environment: + MARIADB_USER: foo MARIADB_ROOT_PASSWORD: foobar + MARIADB_PASSWORD: bar labels: # no need to declare this port in docker-compose # but katenary will need it katenary.io/ports: 3306 + # these variables are secrets + katenary.io/secret-vars: MARIADB_ROOT_PASSWORD, MARIADB_PASSWORD ``` # Labels @@ -170,6 +174,7 @@ These labels could be found by `katenary show-labels`, and can be placed as "lab ``` katenary.io/ignore : ignore the container, it will not yied any object in the helm chart +katenary.io/secret-vars : secret variables to push on a secret file katenary.io/secret-envfiles : set the given file names as a secret instead of configmap katenary.io/mapenv : map environment variable to a template string (yaml style) katenary.io/ports : set the ports to expose as a service (coma separated) @@ -182,7 +187,6 @@ katenary.io/healthcheck : specifies that the container should be monito - "http://[not used address][:port][/path]" to specify an http healthcheck - "tcp://[not used address]:port" to specify a tcp healthcheck - other string is condidered as a "command" healthcheck -katenary.io/env-to-service : DEPRECATED use katenary.io/mapenv instead - specifies that the environment variable points on a service name (coma separated) ``` # What a name... -- 2.49.1 From b1b96d83184c624901b0012c345e4854eba25b0c Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Fri, 6 May 2022 20:18:10 +0200 Subject: [PATCH 41/47] A big commit to fix more things, see details - ingress can now have annotations - configmaps are better named and there is a label to know from where the content is taken - fixed bad "nil" pointer checks - we force to not resolve paths from compose file, this should be a problem when we call the compose file from outside. So, TODO: check this! - the "project" label is now the Chart Name, not a static string - minor fixes --- compose/parser.go | 2 +- generator/main.go | 34 ++++++++++++++++++++-------------- generator/writer.go | 15 +++++++++++++-- generator/writers/configmap.go | 2 +- generator/writers/ingress.go | 9 +++++++++ helm/configAndSecretMap.go | 10 ++++++++-- helm/container.go | 1 - helm/k8sbase.go | 17 ++++++++++++++++- 8 files changed, 68 insertions(+), 22 deletions(-) diff --git a/compose/parser.go b/compose/parser.go index ca95fd7..81e54bf 100644 --- a/compose/parser.go +++ b/compose/parser.go @@ -69,7 +69,7 @@ func (p *Parser) Parse(appname string) { cli.WithDefaultConfigPath, cli.WithNormalization(true), cli.WithInterpolation(true), - cli.WithResolvedPaths(true), + //cli.WithResolvedPaths(true), ) if err != nil { log.Fatal(err) diff --git a/generator/main.go b/generator/main.go index cec719f..7fc8fea 100644 --- a/generator/main.go +++ b/generator/main.go @@ -195,11 +195,15 @@ func generateServicesAndIngresses(name string, s *types.ServiceConfig) []HelmFil func createIngress(name string, port int, s *types.ServiceConfig) *helm.Ingress { ingress := helm.NewIngress(name) + annotations := map[string]string{} ingressVal := map[string]interface{}{ - "class": "nginx", - "host": name + "." + helm.Appname + ".tld", - "enabled": false, + "class": "nginx", + "host": name + "." + helm.Appname + ".tld", + "enabled": false, + "annotations": annotations, } + + // add Annotations in values AddValues(name, map[string]EnvVal{"ingress": ingressVal}) ingress.Spec.Rules = []helm.IngressRule{ @@ -235,7 +239,7 @@ func buildSelector(name string, s *types.ServiceConfig) map[string]string { } // buildCMFromPath generates a ConfigMap from a path. -func buildCMFromPath(path string) *helm.ConfigMap { +func buildCMFromPath(name, path string) *helm.ConfigMap { stat, err := os.Stat(path) if err != nil { return nil @@ -262,7 +266,7 @@ func buildCMFromPath(path string) *helm.ConfigMap { } } - cm := helm.NewConfigMap("") + cm := helm.NewConfigMap(name, path) cm.Data = files return cm } @@ -354,11 +358,11 @@ func prepareVolumes(deployment, name string, s *types.ServiceConfig, container * } // the volume is a path and it's explicitally asked to be a configmap in labels - cm := buildCMFromPath(volname) + cm := buildCMFromPath(name, volname) volname = strings.Replace(volname, "./", "", 1) volname = strings.ReplaceAll(volname, "/", "-") volname = strings.ReplaceAll(volname, ".", "-") - cm.K8sBase.Metadata.Name = helm.ReleaseNameTpl + "-" + volname + "-" + name + cm.K8sBase.Metadata.Name = helm.ReleaseNameTpl + "-" + name + "-" + volname // build a configmap from the volume path volumes = append(volumes, map[string]interface{}{ @@ -570,7 +574,6 @@ func prepareEnvFromFiles(name string, s *types.ServiceConfig, container *helm.Co f = strings.ReplaceAll(f, ".env", "") f = strings.ReplaceAll(f, ".", "") f = strings.ReplaceAll(f, "/", "") - cf := f + "-" + name isSecret := false for _, s := range secretsFiles { s = strings.TrimSpace(s) @@ -580,11 +583,11 @@ func prepareEnvFromFiles(name string, s *types.ServiceConfig, container *helm.Co } var store helm.InlineConfig if !isSecret { - logger.Bluef(ICON_CONF+" Generating configMap %s\n", cf) - store = helm.NewConfigMap(cf) + logger.Bluef(ICON_CONF+" Generating configMap from %s\n", envfile) + store = helm.NewConfigMap(name, envfile) } else { - logger.Bluef(ICON_SECRET+" Generating secret %s\n", cf) - store = helm.NewSecret(cf) + logger.Bluef(ICON_SECRET+" Generating secret from %s\n", envfile) + store = helm.NewSecret(name, envfile) } envfile = filepath.Join(compose.GetCurrentDir(), envfile) @@ -620,7 +623,7 @@ func prepareEnvFromFiles(name string, s *types.ServiceConfig, container *helm.Co } if store != nil { - fileGeneratorChan <- store + fileGeneratorChan <- store.(HelmFile) } } } @@ -695,6 +698,9 @@ func applyEnvMapLabel(s *types.ServiceConfig, c *helm.Container) { vstring := fmt.Sprintf("%v", v) s.Environment[k] = &vstring touched := false + if c.Env != nil { + c.Env = make([]*helm.Value, 0) + } for _, env := range c.Env { if env.Name == k { env.Value = v @@ -745,7 +751,7 @@ func setSecretVar(name string, s *types.ServiceConfig, c *helm.Container) *helm. return nil } - store := helm.NewSecret(name + "-secrets") + store := helm.NewSecret(name, "") for _, secretvar := range strings.Split(secretvars, ",") { secretvar = strings.TrimSpace(secretvar) // get the value from env diff --git a/generator/writer.go b/generator/writer.go index 6db0bba..b661ddd 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -16,7 +16,10 @@ import ( "gopkg.in/yaml.v3" ) -type HelmFile interface{} +type HelmFile interface { + GetType() string + GetPathRessource() string +} type HelmFileGenerator chan HelmFile var PrefixRE = regexp.MustCompile(`\{\{.*\}\}-?`) @@ -179,7 +182,15 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, chartVers case *helm.ConfigMap, *helm.Secret: // there could be several files, so let's force the filename - name := c.(helm.Named).Name() + name := c.(helm.Named).Name() + "-" + c.GetType() + suffix := c.GetPathRessource() + if suffix != "" { + charToRemove := []string{"/", ".", " "} + for _, char := range charToRemove { + suffix = strings.Replace(suffix, char, "-", -1) + } + } + name += suffix name = PrefixRE.ReplaceAllString(name, "") writers.BuildConfigMap(c, kind, n, name, templatesDir) diff --git a/generator/writers/configmap.go b/generator/writers/configmap.go index 846dabc..d045f1d 100644 --- a/generator/writers/configmap.go +++ b/generator/writers/configmap.go @@ -9,7 +9,7 @@ import ( // BuildConfigMap writes the configMap. func BuildConfigMap(c interface{}, kind, servicename, name, templatesDir string) { - fname := filepath.Join(templatesDir, servicename+"."+name+"."+kind+".yaml") + fname := filepath.Join(templatesDir, name+"."+kind+".yaml") fp, _ := os.Create(fname) enc := yaml.NewEncoder(fp) enc.SetIndent(IndentSize) diff --git a/generator/writers/ingress.go b/generator/writers/ingress.go index be7b4bd..fbfdc60 100644 --- a/generator/writers/ingress.go +++ b/generator/writers/ingress.go @@ -59,6 +59,15 @@ func BuildIngress(ingress *helm.Ingress, name, templatesDir string) { l = apiVersion } + // add annotations linked to the Values + if strings.Contains(l, "annotations:") { + n := CountSpaces(l) + IndentSize + l += "\n" + strings.Repeat(" ", n) + "{{- range $k, $v := .Values.__name__.ingress.annotations }}\n" + l += strings.Repeat(" ", n) + "{{ $k }}: {{ $v }}\n" + l += strings.Repeat(" ", n) + "{{- end }}" + l = strings.ReplaceAll(l, "__name__", name) + } + // pathTyype is ony for 1.19+ if strings.Contains(l, "pathType:") { n := CountSpaces(l) diff --git a/helm/configAndSecretMap.go b/helm/configAndSecretMap.go index b319257..86ff0b7 100644 --- a/helm/configAndSecretMap.go +++ b/helm/configAndSecretMap.go @@ -21,12 +21,15 @@ type ConfigMap struct { } // NewConfigMap returns a new initialzed ConfigMap. -func NewConfigMap(name string) *ConfigMap { +func NewConfigMap(name, path string) *ConfigMap { base := NewBase() base.ApiVersion = "v1" base.Kind = "ConfigMap" base.Metadata.Name = ReleaseNameTpl + "-" + name base.Metadata.Labels[K+"/component"] = name + if path != "" { + base.Metadata.Labels[K+"/path"] = path + } return &ConfigMap{ K8sBase: base, Data: make(map[string]string), @@ -72,12 +75,15 @@ type Secret struct { } // NewSecret returns a new initialzed Secret. -func NewSecret(name string) *Secret { +func NewSecret(name, path string) *Secret { base := NewBase() base.ApiVersion = "v1" base.Kind = "Secret" base.Metadata.Name = ReleaseNameTpl + "-" + name base.Metadata.Labels[K+"/component"] = name + if path != "" { + base.Metadata.Labels[K+"/path"] = path + } return &Secret{ K8sBase: base, Data: make(map[string]string), diff --git a/helm/container.go b/helm/container.go index f202a0c..05441fa 100644 --- a/helm/container.go +++ b/helm/container.go @@ -38,7 +38,6 @@ func NewContainer(name, image string, environment types.MappingWithEquals, label container := &Container{ Image: image, Name: name, - Env: make([]*Value, len(environment)), EnvFrom: make([]map[string]map[string]string, 0), } diff --git a/helm/k8sbase.go b/helm/k8sbase.go index 81a8b42..df95877 100644 --- a/helm/k8sbase.go +++ b/helm/k8sbase.go @@ -4,6 +4,7 @@ import ( "crypto/sha1" "fmt" "io/ioutil" + "strings" ) // Metadata is the metadata for a kubernetes object. @@ -34,7 +35,7 @@ func NewBase() *K8sBase { Metadata: NewMetadata(), } // add some information of the build - b.Metadata.Labels[K+"/project"] = GetProjectName() + b.Metadata.Labels[K+"/project"] = "{{ .Chart.Name }}" b.Metadata.Labels[K+"/release"] = ReleaseNameTpl b.Metadata.Annotations[K+"/version"] = Version return b @@ -56,3 +57,17 @@ func (k *K8sBase) Get() string { func (k *K8sBase) Name() string { return k.Metadata.Name } + +func (k *K8sBase) GetType() string { + if n, ok := k.Metadata.Labels[K+"/type"]; ok { + return n + } + return strings.ToLower(k.Kind) +} + +func (k *K8sBase) GetPathRessource() string { + if p, ok := k.Metadata.Labels[K+"/path"]; ok { + return p + } + return "" +} -- 2.49.1 From 2e1950174cc2baa7cb4a514657eda6dee67d5e20 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Sat, 7 May 2022 10:07:10 +0200 Subject: [PATCH 42/47] New label to get volumes from another service - fix container name - the volume-from label works only with "same-pod" label of course. It allows the use of the same volume from another service to be shared in a same pod. E.g. php-fpm + nginx --- generator/main.go | 93 ++++++++++++++++++++++++++++++++++++++++++----- helm/labels.go | 3 ++ 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/generator/main.go b/generator/main.go index 7fc8fea..a2d152a 100644 --- a/generator/main.go +++ b/generator/main.go @@ -359,9 +359,7 @@ func prepareVolumes(deployment, name string, s *types.ServiceConfig, container * // the volume is a path and it's explicitally asked to be a configmap in labels cm := buildCMFromPath(name, volname) - volname = strings.Replace(volname, "./", "", 1) - volname = strings.ReplaceAll(volname, "/", "-") - volname = strings.ReplaceAll(volname, ".", "-") + volname = PathToName(volname) cm.K8sBase.Metadata.Name = helm.ReleaseNameTpl + "-" + name + "-" + volname // build a configmap from the volume path @@ -402,7 +400,7 @@ func prepareVolumes(deployment, name string, s *types.ServiceConfig, container * "name": volname, "mountPath": volepath, }) - container.VolumeMounts = mountPoints + container.VolumeMounts = append(container.VolumeMounts, mountPoints...) isEmptyDir = true break } @@ -433,7 +431,7 @@ func prepareVolumes(deployment, name string, s *types.ServiceConfig, container * } } } - container.VolumeMounts = mountPoints + container.VolumeMounts = append(container.VolumeMounts, mountPoints...) return volumes } @@ -570,10 +568,8 @@ func prepareEnvFromFiles(name string, s *types.ServiceConfig, container *helm.Co // manage environment files (env_file in compose) for _, envfile := range s.EnvFile { - f := strings.ReplaceAll(envfile, "_", "-") + f := PathToName(envfile) f = strings.ReplaceAll(f, ".env", "") - f = strings.ReplaceAll(f, ".", "") - f = strings.ReplaceAll(f, "/", "") isSecret := false for _, s := range secretsFiles { s = strings.TrimSpace(s) @@ -787,7 +783,7 @@ func newContainerForDeployment(deployName, containerName string, deployment *hel }) } setEnvToValues(containerName, s, container) - prepareContainer(container, s, deployName) + prepareContainer(container, s, containerName) prepareEnvFromFiles(deployName, s, container, fileGeneratorChan) // add the container in deployment @@ -803,6 +799,9 @@ func newContainerForDeployment(deployName, containerName string, deployment *hel if deployment.Spec.Template.Spec.Volumes == nil { deployment.Spec.Template.Spec.Volumes = make([]map[string]interface{}, 0) } + // manage LABEL_VOLUMEFROM + addVolumeFrom(deployment, container, s) + // and then we can add other volumes deployment.Spec.Template.Spec.Volumes = append( deployment.Spec.Template.Spec.Volumes, prepareVolumes(deployName, containerName, s, container, fileGeneratorChan)..., @@ -819,3 +818,79 @@ func newContainerForDeployment(deployName, containerName string, deployment *hel return container } + +// addVolumeFrom takes the LABEL_VOLUMEFROM to get volumes from another container. This can only work with +// container that has got LABEL_SAMEPOD as we need to get the volumes from another container in the same deployment. +func addVolumeFrom(deployment *helm.Deployment, container *helm.Container, s *types.ServiceConfig) { + labelfrom, ok := s.Labels[helm.LABEL_VOLUMEFROM] + if !ok { + return + } + + // decode Yaml from the label + var volumesFrom map[string]map[string]string + err := yaml.Unmarshal([]byte(labelfrom), &volumesFrom) + if err != nil { + logger.ActivateColors = true + logger.Red(err.Error()) + logger.ActivateColors = false + return + } + + // for each declared volume "from", we will find it from the deployment volumes and add it to the container. + // Then, to avoid duplicates, we will remove it from the ServiceConfig object. + for name, volumes := range volumesFrom { + for volumeName := range volumes { + initianame := volumeName + volumeName = PathToName(volumeName) + // get the volume from the deployment container "name" + var ctn *helm.Container + for _, c := range deployment.Spec.Template.Spec.Containers { + if c.Name == name { + ctn = c + break + } + } + if ctn == nil { + logger.ActivateColors = true + logger.Red("VolumeFrom: container %s not found", name) + logger.ActivateColors = false + continue + } + // get the volume from the container + for _, v := range ctn.VolumeMounts { + switch v := v.(type) { + case map[string]interface{}: + if v["name"] == volumeName { + if container.VolumeMounts == nil { + container.VolumeMounts = make([]interface{}, 0) + } + // make a copy of the volume mount and then add it to the VolumeMounts + var mountpoint = make(map[string]interface{}) + for k, v := range v { + mountpoint[k] = v + } + container.VolumeMounts = append(container.VolumeMounts, mountpoint) + + // remove the volume from the ServiceConfig + volumes := s.Volumes + for i, vol := range volumes { + if vol.Source == initianame { + volumes = append(volumes[:i], volumes[i+1:]...) + break + } + } + s.Volumes = volumes + } + } + } + } + } +} + +func PathToName(path string) string { + path = strings.TrimPrefix(path, "./") + path = strings.ReplaceAll(path, ".", "-") + path = strings.ReplaceAll(path, "/", "-") + return path +} diff --git a/helm/labels.go b/helm/labels.go index 4b07921..d9fc397 100644 --- a/helm/labels.go +++ b/helm/labels.go @@ -14,6 +14,7 @@ const ( LABEL_VOL_CM = K + "/configmap-volumes" LABEL_HEALTHCHECK = K + "/healthcheck" LABEL_SAMEPOD = K + "/same-pod" + LABEL_VOLUMEFROM = K + "/volume-from" LABEL_EMPTYDIRS = K + "/empty-dirs" LABEL_IGNORE = K + "/ignore" LABEL_SECRETVARS = K + "/secret-vars" @@ -34,6 +35,7 @@ func GetLabelsDocumentation() string { {{.LABEL_INGRESS | printf "%-33s"}}: set the port to expose in an ingress (coma separated) {{.LABEL_VOL_CM | printf "%-33s"}}: specifies that the volumes points on a configmap (coma separated) {{.LABEL_SAMEPOD | printf "%-33s"}}: specifies that the pod should be deployed in the same pod than the given service name +{{.LABEL_VOLUMEFROM | printf "%-33s"}}: specifies that the volumes to be mounted from the given service (yaml style) {{.LABEL_EMPTYDIRS | printf "%-33s"}}: specifies that the given volume names should be "emptyDir" instead of persistentVolumeClaim (coma separated) {{.LABEL_HEALTHCHECK | printf "%-33s"}}: specifies that the container should be monitored by a healthcheck, **it overrides the docker-compose healthcheck**. {{ printf "%-34s" ""}} You can use these form of label values: @@ -49,6 +51,7 @@ func GetLabelsDocumentation() string { "LABEL_VOL_CM": LABEL_VOL_CM, "LABEL_HEALTHCHECK": LABEL_HEALTHCHECK, "LABEL_SAMEPOD": LABEL_SAMEPOD, + "LABEL_VOLUMEFROM": LABEL_VOLUMEFROM, "LABEL_EMPTYDIRS": LABEL_EMPTYDIRS, "LABEL_IGNORE": LABEL_IGNORE, "LABEL_MAP_ENV": LABEL_MAP_ENV, -- 2.49.1 From a4b848a83cc0e52e284bd18cddebb0fc46c0634c Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Sat, 7 May 2022 17:33:43 +0200 Subject: [PATCH 43/47] Fix bad log --- generator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/main.go b/generator/main.go index a2d152a..5070a4d 100644 --- a/generator/main.go +++ b/generator/main.go @@ -853,7 +853,7 @@ func addVolumeFrom(deployment *helm.Deployment, container *helm.Container, s *ty } if ctn == nil { logger.ActivateColors = true - logger.Red("VolumeFrom: container %s not found", name) + logger.Redf("VolumeFrom: container %s not found", name) logger.ActivateColors = false continue } -- 2.49.1 From 2b1dc683030707f77c29fdbef6a2d9a6ec282a44 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Sat, 7 May 2022 17:39:03 +0200 Subject: [PATCH 44/47] Fix slice index removing --- generator/main.go | 17 +++++++++-------- generator/writer.go | 3 ++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/generator/main.go b/generator/main.go index 5070a4d..4ae44af 100644 --- a/generator/main.go +++ b/generator/main.go @@ -613,6 +613,7 @@ func prepareEnvFromFiles(name string, s *types.ServiceConfig, container *helm.Co for i, s := range container.Env { if s.Name == varname { container.Env = append(container.Env[:i], container.Env[i+1:]...) + i-- } } } @@ -757,13 +758,14 @@ func setSecretVar(name string, s *types.ServiceConfig, c *helm.Container) *helm. } // add the secret store.AddEnv(secretvar, ".Values."+name+".environment."+secretvar) - envs := c.Env - for i, env := range envs { + for i, env := range c.Env { if env.Name == secretvar { - envs = append(envs[:i], envs[i+1:]...) + c.Env = append(c.Env[:i], c.Env[i+1:]...) + i-- } } - c.Env = envs + // remove env from ServiceConfig + delete(s.Environment, secretvar) } return store } @@ -873,14 +875,13 @@ func addVolumeFrom(deployment *helm.Deployment, container *helm.Container, s *ty container.VolumeMounts = append(container.VolumeMounts, mountpoint) // remove the volume from the ServiceConfig - volumes := s.Volumes - for i, vol := range volumes { + for i, vol := range s.Volumes { if vol.Source == initianame { - volumes = append(volumes[:i], volumes[i+1:]...) + s.Volumes = append(s.Volumes[:i], s.Volumes[i+1:]...) + i-- break } } - s.Volumes = volumes } } } diff --git a/generator/writer.go b/generator/writer.go index b661ddd..6ed7dc3 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -61,11 +61,12 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, chartVers } } - // remove skipped services + // remove skipped services from the parsed data for s := range skips { for i, service := range p.Data.Services { if service.Name == s { p.Data.Services = append(p.Data.Services[:i], p.Data.Services[i+1:]...) + i-- break } } -- 2.49.1 From e7e4746774a755c1b68241edf005deb3741c26c9 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Sat, 7 May 2022 17:39:52 +0200 Subject: [PATCH 45/47] Better path to name + more comments --- generator/main.go | 8 ++++++-- generator/writer.go | 13 +++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/generator/main.go b/generator/main.go index 4ae44af..9bec10b 100644 --- a/generator/main.go +++ b/generator/main.go @@ -10,6 +10,7 @@ import ( "net/url" "os" "path/filepath" + "regexp" "runtime" "strconv" "strings" @@ -889,9 +890,12 @@ func addVolumeFrom(deployment *helm.Deployment, container *helm.Container, s *ty } } +// replaceChars replaces some chars in a string. +const replaceChars = `[^a-zA-Z0-9._-]` + +// PathToName transform a path to a yaml name. func PathToName(path string) string { path = strings.TrimPrefix(path, "./") - path = strings.ReplaceAll(path, ".", "-") - path = strings.ReplaceAll(path, "/", "-") + path = regexp.MustCompile(replaceChars).ReplaceAllString(path, "-") return path } diff --git a/generator/writer.go b/generator/writer.go index 6ed7dc3..ec0c372 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -16,10 +16,14 @@ import ( "gopkg.in/yaml.v3" ) +// HelmFile represents a helm file from helm package that has got some necessary methods +// to generate a helm file. type HelmFile interface { GetType() string GetPathRessource() string } + +// HelmFileGenerator is a chanel of HelmFile. type HelmFileGenerator chan HelmFile var PrefixRE = regexp.MustCompile(`\{\{.*\}\}-?`) @@ -34,6 +38,7 @@ func portExists(port int, ports []types.ServicePortConfig) bool { return false } +// Generate get a parsed compose file, and generate the helm files. func Generate(p *compose.Parser, katernayVersion, appName, appVersion, chartVersion, composeFile, dirName string) { // make the appname global (yes... ugly but easy) @@ -55,6 +60,7 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, chartVers // Manage services to not export skips := make(map[string]bool) + // remove ignored services for _, s := range p.Data.Services { if s.Labels[helm.LABEL_IGNORE] == "true" { skips[s.Name] = true @@ -185,12 +191,7 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, chartVers // there could be several files, so let's force the filename name := c.(helm.Named).Name() + "-" + c.GetType() suffix := c.GetPathRessource() - if suffix != "" { - charToRemove := []string{"/", ".", " "} - for _, char := range charToRemove { - suffix = strings.Replace(suffix, char, "-", -1) - } - } + suffix = PathToName(suffix) name += suffix name = PrefixRE.ReplaceAllString(name, "") writers.BuildConfigMap(c, kind, n, name, templatesDir) -- 2.49.1 From 7e861618bb2331e5200e4a0f008b2b0af77b2cd4 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Sun, 8 May 2022 08:31:43 +0200 Subject: [PATCH 46/47] Remove useless temporary slices --- generator/writer.go | 63 +++++++++++++++++---------------------------- 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/generator/writer.go b/generator/writer.go index ec0c372..9afec43 100644 --- a/generator/writer.go +++ b/generator/writer.go @@ -52,42 +52,25 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, chartVers log.Fatal(err) } - files := make(map[string]HelmFileGenerator) - - // Manage services, avoid linked pods and store all services port in servicesMap - avoids := make(map[string]bool) - - // Manage services to not export - skips := make(map[string]bool) - - // remove ignored services - for _, s := range p.Data.Services { - if s.Labels[helm.LABEL_IGNORE] == "true" { - skips[s.Name] = true - } - } + generators := make(map[string]HelmFileGenerator) // remove skipped services from the parsed data - for s := range skips { - for i, service := range p.Data.Services { - if service.Name == s { - p.Data.Services = append(p.Data.Services[:i], p.Data.Services[i+1:]...) - i-- - break - } + for i, service := range p.Data.Services { + if v, ok := service.Labels[helm.LABEL_IGNORE]; !ok || v != "true" { + continue + } + p.Data.Services = append(p.Data.Services[:i], p.Data.Services[i+1:]...) + i-- + + // find this service in others as "depends_on" and remove it + for _, service2 := range p.Data.Services { + delete(service2.DependsOn, service.Name) } } for i, service := range p.Data.Services { n := service.Name - // if the service depends on a skipped service, remove the link - for dep := range service.DependsOn { - if skips[dep] { - delete(service.DependsOn, dep) - } - } - // if the service port is declared in labels, add it to the service. if ports, ok := service.Labels[helm.LABEL_PORT]; ok { if service.Ports == nil { @@ -115,15 +98,13 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, chartVers } } - // avoid linked pods - if _, ok := service.Labels[helm.LABEL_SAMEPOD]; ok { - avoids[n] = true - } - // manage emptyDir volumes if empty, ok := service.Labels[helm.LABEL_EMPTYDIRS]; ok { //split empty list by coma emptyDirs := strings.Split(empty, ",") + for i, emptyDir := range emptyDirs { + emptyDirs[i] = strings.TrimSpace(emptyDir) + } //append them in EmptyDirs EmptyDirs = append(EmptyDirs, emptyDirs...) } @@ -131,15 +112,17 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, chartVers } - // for all services in linked map, and not in avoids map, generate the service + // for all services in linked map, and not in samePods map, generate the service for _, s := range p.Data.Services { name := s.Name - if _, found := avoids[name]; found { + // do not make a deployment for services declared to be in the same pod than another + if _, ok := s.Labels[helm.LABEL_SAMEPOD]; ok { continue } + + // find services that is in the same pod linked := make(map[string]types.ServiceConfig, 0) - // find service for _, service := range p.Data.Services { n := service.Name if linkname, ok := service.Labels[helm.LABEL_SAMEPOD]; ok && linkname == name { @@ -147,15 +130,15 @@ func Generate(p *compose.Parser, katernayVersion, appName, appVersion, chartVers } } - files[name] = CreateReplicaObject(name, s, linked) + generators[name] = CreateReplicaObject(name, s, linked) } // to generate notes, we need to keep an Ingresses list ingresses := make(map[string]*helm.Ingress) - for n, generator := range files { - for helmFile := range generator { - if helmFile == nil { + for n, generator := range generators { // generators is a map : name -> generator + for helmFile := range generator { // generator is a chan + if helmFile == nil { // generator finished break } kind := helmFile.(helm.Kinded).Get() -- 2.49.1 From 333d051d4967ba9ecacffd1f65507aef082587cf Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Sun, 8 May 2022 09:11:31 +0200 Subject: [PATCH 47/47] Fix problems with relative/absolute path --- compose/parser.go | 3 ++- generator/main.go | 33 ++++++++++++--------------------- generator/utils.go | 22 ++++++++++++++++++++++ 3 files changed, 36 insertions(+), 22 deletions(-) create mode 100644 generator/utils.go diff --git a/compose/parser.go b/compose/parser.go index 81e54bf..86b0cb5 100644 --- a/compose/parser.go +++ b/compose/parser.go @@ -69,8 +69,9 @@ func (p *Parser) Parse(appname string) { cli.WithDefaultConfigPath, cli.WithNormalization(true), cli.WithInterpolation(true), - //cli.WithResolvedPaths(true), + cli.WithResolvedPaths(true), ) + if err != nil { log.Fatal(err) } diff --git a/generator/main.go b/generator/main.go index 9bec10b..4d2a252 100644 --- a/generator/main.go +++ b/generator/main.go @@ -10,7 +10,6 @@ import ( "net/url" "os" "path/filepath" - "regexp" "runtime" "strconv" "strings" @@ -239,8 +238,8 @@ func buildSelector(name string, s *types.ServiceConfig) map[string]string { } } -// buildCMFromPath generates a ConfigMap from a path. -func buildCMFromPath(name, path string) *helm.ConfigMap { +// buildConfigMapFromPath generates a ConfigMap from a path. +func buildConfigMapFromPath(name, path string) *helm.ConfigMap { stat, err := os.Stat(path) if err != nil { return nil @@ -267,7 +266,7 @@ func buildCMFromPath(name, path string) *helm.ConfigMap { } } - cm := helm.NewConfigMap(name, path) + cm := helm.NewConfigMap(name, GetRelPath(path)) cm.Data = files return cm } @@ -313,6 +312,9 @@ func prepareVolumes(deployment, name string, s *types.ServiceConfig, container * configMapsVolumes := make([]string, 0) if v, ok := s.Labels[helm.LABEL_VOL_CM]; ok { configMapsVolumes = strings.Split(v, ",") + for i, cm := range configMapsVolumes { + configMapsVolumes[i] = strings.TrimSpace(cm) + } } for _, vol := range s.Volumes { @@ -329,15 +331,14 @@ func prepareVolumes(deployment, name string, s *types.ServiceConfig, container * isConfigMap := false for _, cmVol := range configMapsVolumes { - cmVol = strings.TrimSpace(cmVol) - if volname == cmVol { + if GetRelPath(volname) == cmVol { isConfigMap = true break } } + // local volume cannt be mounted if !isConfigMap && (strings.HasPrefix(volname, ".") || strings.HasPrefix(volname, "/")) { - // local volume cannt be mounted logger.ActivateColors = true logger.Redf("You cannot, at this time, have local volume in %s deployment\n", name) logger.ActivateColors = false @@ -359,11 +360,10 @@ func prepareVolumes(deployment, name string, s *types.ServiceConfig, container * } // the volume is a path and it's explicitally asked to be a configmap in labels - cm := buildCMFromPath(name, volname) - volname = PathToName(volname) - cm.K8sBase.Metadata.Name = helm.ReleaseNameTpl + "-" + name + "-" + volname + cm := buildConfigMapFromPath(name, volname) + cm.K8sBase.Metadata.Name = helm.ReleaseNameTpl + "-" + name + "-" + PathToName(volname) - // build a configmap from the volume path + // build a configmapRef for this volume volumes = append(volumes, map[string]interface{}{ "name": volname, "configMap": map[string]string{ @@ -432,6 +432,7 @@ func prepareVolumes(deployment, name string, s *types.ServiceConfig, container * } } } + // add the volume in the container and return the volume definition to add in Deployment container.VolumeMounts = append(container.VolumeMounts, mountPoints...) return volumes } @@ -889,13 +890,3 @@ func addVolumeFrom(deployment *helm.Deployment, container *helm.Container, s *ty } } } - -// replaceChars replaces some chars in a string. -const replaceChars = `[^a-zA-Z0-9._-]` - -// PathToName transform a path to a yaml name. -func PathToName(path string) string { - path = strings.TrimPrefix(path, "./") - path = regexp.MustCompile(replaceChars).ReplaceAllString(path, "-") - return path -} diff --git a/generator/utils.go b/generator/utils.go new file mode 100644 index 0000000..a862035 --- /dev/null +++ b/generator/utils.go @@ -0,0 +1,22 @@ +package generator + +import ( + "katenary/compose" + "regexp" + "strings" +) + +// replaceChars replaces some chars in a string. +const replaceChars = `[^a-zA-Z0-9._]` + +// GetRelPath return the relative path from the root of the project. +func GetRelPath(path string) string { + return strings.Replace(path, compose.GetCurrentDir(), ".", 1) +} + +// PathToName transform a path to a yaml name. +func PathToName(path string) string { + path = strings.TrimPrefix(GetRelPath(path), "./") + path = regexp.MustCompile(replaceChars).ReplaceAllString(path, "-") + return path +} -- 2.49.1