diff --git a/pkg/instance/build.go b/pkg/instance/build.go index d09f14b..07d823b 100644 --- a/pkg/instance/build.go +++ b/pkg/instance/build.go @@ -3,6 +3,7 @@ package instance import ( "context" "fmt" + "os" "path/filepath" "sync" @@ -14,8 +15,6 @@ import ( "github.com/celestiaorg/knuu/pkg/container" ) -const buildDirBase = "/tmp/knuu" - type build struct { instance *Instance imageName string @@ -25,6 +24,7 @@ type build struct { args []string env map[string]string imageCache *sync.Map + buildDir string } func (i *Instance) Build() *build { @@ -53,10 +53,15 @@ func (b *build) SetImage(ctx context.Context, image string, args ...builder.ArgI return ErrSettingImageNotAllowed.WithParams(b.instance.state.String()) } + buildDir, err := b.getBuildDir() + if err != nil { + return ErrGettingBuildDir.Wrap(err) + } + // Use the builder to build a new image factory, err := container.NewBuilderFactory(container.BuilderFactoryOptions{ ImageName: image, - BuildContext: b.getBuildDir(), + BuildContext: buildDir, ImageBuilder: b.instance.ImageBuilder, Args: args, Logger: b.instance.Logger, @@ -88,9 +93,20 @@ func (b *build) SetGitRepo(ctx context.Context, gitContext builder.GitContext, a } b.imageName = resolvedImage.ToString() + buildDir, err := b.getBuildDir() + if err != nil { + return ErrGettingBuildDir.Wrap(err) + } + b.imageName = resolvedImage.ToString() + + buildDir, err := b.getBuildDir() + if err != nil { + return ErrGettingBuildDir.Wrap(err) + } + factory, err := container.NewBuilderFactory(container.BuilderFactoryOptions{ ImageName: b.imageName, - BuildContext: b.getBuildDir(), + BuildContext: buildDir, ImageBuilder: b.instance.ImageBuilder, Args: args, Logger: b.instance.Logger, @@ -226,8 +242,16 @@ func getImageRegistry(imageName string) (string, error) { } // getBuildDir returns the build directory for the instance -func (b *build) getBuildDir() string { - return filepath.Join(buildDirBase, b.instance.name) +func (b *build) getBuildDir() (string, error) { + if b.buildDir != "" { + return b.buildDir, nil + } + + tmpDir, err := os.MkdirTemp("", "knuu-build-*") + if err != nil { + return "", err + } + return filepath.Join(tmpDir, b.instance.name), nil } // addFileToBuilder adds a file to the builder diff --git a/pkg/instance/errors.go b/pkg/instance/errors.go index a0487ec..6a87395 100644 --- a/pkg/instance/errors.go +++ b/pkg/instance/errors.go @@ -83,6 +83,7 @@ var ( ErrSettingGitRepo = errors.New("SettingGitRepo", "setting git repo is only allowed in state 'None'. Current state is '%s'") ErrGettingBuildContext = errors.New("GettingBuildContext", "error getting build context") ErrGettingImageName = errors.New("GettingImageName", "error getting image name") + ErrGettingBuildDir = errors.New("GettingBuildDir", "error getting build directory") ErrSettingImageNotAllowedForSidecars = errors.New("SettingImageNotAllowedForSidecars", "setting image is not allowed for sidecars") ErrSettingCommand = errors.New("SettingCommand", "setting command is only allowed in state 'Preparing' or 'Committed'. Current state is '%s") ErrSettingArgsNotAllowed = errors.New("SettingArgsNotAllowed", "setting args is only allowed in state 'Preparing' or 'Committed'. Current state is '%s") @@ -102,7 +103,9 @@ var ( ErrCreatingDirectory = errors.New("CreatingDirectory", "error creating directory") ErrFailedToCreateDestFile = errors.New("FailedToCreateDestFile", "failed to create destination file '%s'") ErrFailedToOpenSrcFile = errors.New("FailedToOpenSrcFile", "failed to open source file '%s'") + ErrFailedToGetSrcFileInfo = errors.New("FailedToGetSrcFileInfo", "failed to get source file info for %s") ErrFailedToCopyFile = errors.New("FailedToCopyFile", "failed to copy from source '%s' to destination '%s'") + ErrFailedToSetPermissions = errors.New("FailedToSetPermissions", "failed to set permissions for destination file") ErrSrcDoesNotExistOrIsDirectory = errors.New("SrcDoesNotExistOrIsDirectory", "src '%s' does not exist or is a directory") ErrInvalidFormat = errors.New("InvalidFormat", "invalid format") ErrFailedToConvertToInt64 = errors.New("FailedToConvertToInt64", "failed to convert to int64") @@ -215,6 +218,7 @@ var ( ErrApplyingFunctionToSidecar = errors.New("ApplyingFunctionToSidecar", "error applying function to sidecar '%s'") ErrInitializingSidecar = errors.New("InitializingSidecar", "error initializing sidecar for instance '%s'") ErrSidecarInstanceIsNil = errors.New("SidecarInstanceIsNil", "sidecar instance is nil for instance '%s'") + ErrFailedToCreatePersistentVolumeClaim = errors.New("FailedToCreatePersistentVolumeClaim", "failed to create persistent volume claim") ErrFailedToDeletePersistentVolumeClaim = errors.New("FailedToDeletePersistentVolumeClaim", "failed to delete persistent volume claim") ErrUpgradingImageNotAllowed = errors.New("UpgradingImageNotAllowed", "upgrading image is only allowed in state 'Started'. Current state is '%s'") ErrAddingHostToProxyNotAllowed = errors.New("AddingHostToProxyNotAllowed", "adding host to proxy is only allowed in state 'Started' and 'Preparing'. Current state is '%s'") @@ -224,4 +228,7 @@ var ( ErrCannotCloneInstance = errors.New("CannotCloneInstance", "cannot clone instance '%s' in state '%s'") ErrGettingIPNotAllowed = errors.New("GettingIPNotAllowed", "getting IP is allowed in state 'Started'. Current state is '%s'") ErrPodIPNotReady = errors.New("PodIPNotReady", "pod IP is not ready for pod '%s'") + ErrFailedToGetFileSize = errors.New("FailedToGetFileSize", "failed to get file size") + ErrFileTooLargeCommitted = errors.New("FileTooLargeCommitted", "file '%s' is too large (max 1MiB) to add after instance is committed") + ErrTotalFilesSizeTooLarge = errors.New("TotalFilesSizeTooLarge", "total files size is too large (max 1MiB)") ) diff --git a/pkg/instance/execution.go b/pkg/instance/execution.go index a43ed00..09eceac 100644 --- a/pkg/instance/execution.go +++ b/pkg/instance/execution.go @@ -385,6 +385,8 @@ func (e *execution) prepareReplicaSetConfig() k8s.ReplicaSetConfig { StartupProbe: e.instance.monitoring.startupProbe, Files: e.instance.storage.files, SecurityContext: e.instance.security.prepareSecurityContext(), + TCPPorts: e.instance.network.portsTCP, + UDPPorts: e.instance.network.portsUDP, } sidecarConfigs := make([]k8s.ContainerConfig, 0) @@ -404,6 +406,8 @@ func (e *execution) prepareReplicaSetConfig() k8s.ReplicaSetConfig { StartupProbe: sidecar.Instance().monitoring.startupProbe, Files: sidecar.Instance().storage.files, SecurityContext: sidecar.Instance().security.prepareSecurityContext(), + TCPPorts: sidecar.Instance().network.portsTCP, + UDPPorts: sidecar.Instance().network.portsUDP, }) } @@ -412,7 +416,6 @@ func (e *execution) prepareReplicaSetConfig() k8s.ReplicaSetConfig { Name: e.instance.name, Labels: e.Labels(), ServiceAccountName: e.instance.name, - FsGroup: e.instance.storage.fsGroup, ContainerConfig: containerConfig, SidecarConfigs: sidecarConfigs, ImagePullSecrets: []v1.LocalObjectReference{{Name: "registry-cert-secret"}}, diff --git a/pkg/instance/proxy.go b/pkg/instance/proxy.go index 58342af..18903dc 100644 --- a/pkg/instance/proxy.go +++ b/pkg/instance/proxy.go @@ -27,7 +27,7 @@ func (n *network) AddHost(ctx context.Context, port int) (host string, err error serviceName = n.instance.parentInstance.name } - prefix := fmt.Sprintf("%s-%d", serviceName, port) + prefix := fmt.Sprintf("%s-%s-%d", n.instance.Scope, serviceName, port) if err := n.instance.Proxy.AddHost(ctx, serviceName, prefix, port); err != nil { return "", ErrAddingToProxy.WithParams(serviceName).Wrap(err) } diff --git a/pkg/instance/resources.go b/pkg/instance/resources.go index 9497b1f..bbe99e4 100644 --- a/pkg/instance/resources.go +++ b/pkg/instance/resources.go @@ -89,13 +89,12 @@ func (r *resources) deployStorage(ctx context.Context) error { return ErrDeployingVolumeForInstance.WithParams(r.instance.name).Wrap(err) } } - if len(r.instance.storage.files) == 0 { - return nil + if len(r.instance.storage.files) != 0 { + if err := r.instance.storage.deployFiles(ctx); err != nil { + return ErrDeployingFilesForInstance.WithParams(r.instance.name).Wrap(err) + } } - if err := r.instance.storage.deployFiles(ctx); err != nil { - return ErrDeployingFilesForInstance.WithParams(r.instance.name).Wrap(err) - } return nil } @@ -123,8 +122,7 @@ func (r *resources) destroyResources(ctx context.Context) error { } if len(r.instance.storage.files) != 0 { - err := r.instance.storage.destroyFiles(ctx) - if err != nil { + if err := r.instance.storage.destroyFiles(ctx); err != nil { return ErrDestroyingFilesForInstance.WithParams(r.instance.name).Wrap(err) } } diff --git a/pkg/instance/storage.go b/pkg/instance/storage.go index 4287a4d..b2343ec 100644 --- a/pkg/instance/storage.go +++ b/pkg/instance/storage.go @@ -16,13 +16,16 @@ import ( "github.com/celestiaorg/knuu/pkg/names" ) +const maxTotalFilesBytes = 1024 * 1024 + type storage struct { instance *Instance volumes []*k8s.Volume files []*k8s.File - fsGroup int64 } +const defaultFilePermission = 0644 + func (i *Instance) Storage() *storage { return i.storage } @@ -41,7 +44,7 @@ func (s *storage) AddFile(src string, dest string, chown string) error { return err } - dstPath, err := s.copyFileToBuildDir(src, dest) + buildDirPath, err := s.copyFileToBuildDir(src, dest) if err != nil { return err } @@ -51,14 +54,25 @@ func (s *storage) AddFile(src string, dest string, chown string) error { s.instance.build.addFileToBuilder(src, dest, chown) return nil case StateCommitted, StateStopped: - return s.addFileToInstance(dstPath, dest, chown) + srcInfo, err := os.Stat(src) + if err != nil { + return ErrFailedToGetFileSize.Wrap(err) + } + if srcInfo.Size() > maxTotalFilesBytes { + return ErrFileTooLargeCommitted.WithParams(src) + } + return s.addFileToInstance(buildDirPath, dest, chown) } + buildDir, err := s.instance.build.getBuildDir() + if err != nil { + return ErrGettingBuildDir.Wrap(err) + } s.instance.Logger.WithFields(logrus.Fields{ "file": dest, "instance": s.instance.name, "state": s.instance.state, - "build_dir": s.instance.build.getBuildDir(), + "build_dir": buildDir, }).Debug("added file") return nil } @@ -92,7 +106,11 @@ func (s *storage) AddFolder(src string, dest string, chown string) error { if err != nil { return err } - dstPath := filepath.Join(s.instance.build.getBuildDir(), dest, relPath) + buildDir, err := s.instance.build.getBuildDir() + if err != nil { + return ErrGettingBuildDir.Wrap(err) + } + dstPath := filepath.Join(buildDir, dest, relPath) if info.IsDir() { // create directory at destination path @@ -106,11 +124,15 @@ func (s *storage) AddFolder(src string, dest string, chown string) error { return ErrCopyingFolderToInstance.WithParams(src, s.instance.name).Wrap(err) } + buildDir, err := s.instance.build.getBuildDir() + if err != nil { + return ErrGettingBuildDir.Wrap(err) + } s.instance.Logger.WithFields(logrus.Fields{ "folder": dest, "instance": s.instance.name, "state": s.instance.state, - "build_dir": s.instance.build.getBuildDir(), + "build_dir": buildDir, }).Debug("added folder") return nil } @@ -131,6 +153,9 @@ func (s *storage) AddFileBytes(bytes []byte, dest string, chown string) error { if _, err := tmpfile.Write(bytes); err != nil { return err } + if err := tmpfile.Chmod(defaultFilePermission); err != nil { + return err + } if err := tmpfile.Close(); err != nil { return err } @@ -240,55 +265,83 @@ func (s *storage) validateFileArgs(src, dest, chown string) error { if !strings.Contains(chown, ":") || len(strings.Split(chown, ":")) != 2 { return ErrChownMustBeInFormatUserGroup } + + parts := strings.Split(chown, ":") + for _, part := range parts { + if _, err := strconv.ParseInt(part, 10, 64); err != nil { + return ErrFailedToConvertToInt64.WithParams(part).Wrap(err) + } + } return nil } func (s *storage) copyFileToBuildDir(src, dest string) (string, error) { - dstPath := filepath.Join(s.instance.build.getBuildDir(), dest) + buildDir, err := s.instance.build.getBuildDir() + if err != nil { + return "", ErrGettingBuildDir.Wrap(err) + } + dstPath := filepath.Join(buildDir, dest) if err := os.MkdirAll(filepath.Dir(dstPath), os.ModePerm); err != nil { return "", ErrCreatingDirectory.Wrap(err) } - dst, err := os.Create(dstPath) - if err != nil { - return "", ErrFailedToCreateDestFile.WithParams(dstPath).Wrap(err) - } - defer dst.Close() - srcFile, err := os.Open(src) if err != nil { return "", ErrFailedToOpenSrcFile.WithParams(src).Wrap(err) } defer srcFile.Close() + srcInfo, err := srcFile.Stat() + if err != nil { + return "", ErrFailedToGetSrcFileInfo.WithParams(src).Wrap(err) + } + + dst, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY, srcInfo.Mode().Perm()) + if err != nil { + return "", ErrFailedToCreateDestFile.WithParams(dstPath).Wrap(err) + } + defer dst.Close() + if _, err := io.Copy(dst, srcFile); err != nil { return "", ErrFailedToCopyFile.WithParams(src, dstPath).Wrap(err) } + // Ensure the destination file has the same permissions as the source file + if err := os.Chmod(dstPath, srcInfo.Mode().Perm()); err != nil { + return "", ErrFailedToSetPermissions.WithParams(dstPath).Wrap(err) + } + return dstPath, nil } -func (s *storage) addFileToInstance(dstPath, dest, chown string) error { - srcInfo, err := os.Stat(dstPath) +func (s *storage) addFileToInstance(srcPath, dest, chown string) error { + srcInfo, err := os.Stat(srcPath) if os.IsNotExist(err) || srcInfo.IsDir() { - return ErrSrcDoesNotExistOrIsDirectory.WithParams(dstPath).Wrap(err) + return ErrSrcDoesNotExistOrIsDirectory.WithParams(srcPath).Wrap(err) } - file := s.instance.K8sClient.NewFile(dstPath, dest) - parts := strings.Split(chown, ":") - if len(parts) != 2 { - return ErrInvalidFormat - } + // get the permission of the src file + permission := fmt.Sprintf("%o", srcInfo.Mode().Perm()) - group, err := strconv.ParseInt(parts[1], 10, 64) + size := int64(0) + for _, file := range s.files { + srcInfo, err := os.Stat(file.Source) + if err != nil { + return ErrFailedToGetFileSize.Wrap(err) + } + size += srcInfo.Size() + } + srcInfo, err = os.Stat(srcPath) if err != nil { - return ErrFailedToConvertToInt64.Wrap(err) + return ErrFailedToGetFileSize.Wrap(err) } - - if s.fsGroup != 0 && s.fsGroup != group { - return ErrAllFilesMustHaveSameGroup + size += srcInfo.Size() + if size > maxTotalFilesBytes { + return ErrTotalFilesSizeTooLarge.WithParams(srcPath) } - s.fsGroup = group + + file := s.instance.K8sClient.NewFile(srcPath, dest, chown, permission) + s.files = append(s.files, file) return nil } @@ -307,7 +360,10 @@ func (s *storage) deployVolume(ctx context.Context) error { for _, volume := range s.volumes { totalSize.Add(volume.Size) } - s.instance.K8sClient.CreatePersistentVolumeClaim(ctx, s.instance.name, s.instance.execution.Labels(), totalSize) + err := s.instance.K8sClient.CreatePersistentVolumeClaim(ctx, s.instance.name, s.instance.execution.Labels(), totalSize) + if err != nil { + return ErrFailedToCreatePersistentVolumeClaim.Wrap(err) + } s.instance.Logger.WithFields(logrus.Fields{ "total_size": totalSize.String(), "instance": s.instance.name, @@ -439,6 +495,5 @@ func (s *storage) clone() *storage { instance: nil, volumes: volumesCopy, files: filesCopy, - fsGroup: s.fsGroup, } } diff --git a/pkg/k8s/pod.go b/pkg/k8s/pod.go index f2dc23d..5abf352 100644 --- a/pkg/k8s/pod.go +++ b/pkg/k8s/pod.go @@ -26,12 +26,10 @@ const ( // knuuPath is the path where the knuu volume is mounted knuuPath = "/knuu" - // 0777 is used so that the files are usable by any user in the container without needing to change permissions - defaultFileModeForVolume = 0777 - podFilesConfigmapNameSuffix = "-config" initContainerNameSuffix = "-init" + initContainerImage = "nicolaka/netshoot" defaultContainerUser = 0 ) @@ -51,6 +49,8 @@ type ContainerConfig struct { StartupProbe *v1.Probe // Startup probe for the container Files []*File // Files to add to the Pod SecurityContext *v1.SecurityContext // Security context for the container + TCPPorts []int // TCP ports to expose on the Pod + UDPPorts []int // UDP ports to expose on the Pod } type PodConfig struct { @@ -72,8 +72,10 @@ type Volume struct { } type File struct { - Source string - Dest string + Source string + Dest string + Chown string + Permission string } // DeployPod creates a new pod in the namespace that k8s client is initiate with if it doesn't already exist. @@ -102,10 +104,12 @@ func (c *Client) NewVolume(path string, size resource.Quantity, owner int64) *Vo } } -func (c *Client) NewFile(source, dest string) *File { +func (c *Client) NewFile(source, dest, chown, permission string) *File { return &File{ - Source: source, - Dest: dest, + Source: source, + Dest: dest, + Chown: chown, + Permission: permission, } } @@ -382,6 +386,16 @@ func buildPodVolumes(name string, volumesAmount, filesAmount int) []v1.Volume { podVolumes = append(podVolumes, podVolume) } + if volumesAmount == 0 && filesAmount != 0 { + podVolume := v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + } + podVolumes = append(podVolumes, podVolume) + } + if filesAmount != 0 { podFiles := v1.Volume{ Name: name + podFilesConfigmapNameSuffix, @@ -390,7 +404,7 @@ func buildPodVolumes(name string, volumesAmount, filesAmount int) []v1.Volume { LocalObjectReference: v1.LocalObjectReference{ Name: name, }, - DefaultMode: ptr.To[int32](defaultFileModeForVolume), + DefaultMode: ptr.To[int32](0600), }, }, } @@ -415,25 +429,22 @@ func buildContainerVolumes(name string, volumes []*Volume, files []*File) []v1.V ) } - var containerFiles []v1.VolumeMount - - for n, file := range files { - shouldAddFile := true - for _, volume := range volumes { - if strings.HasPrefix(file.Dest, volume.Path) { - shouldAddFile = false - break - } + if len(volumes) == 0 && len(files) != 0 { + uniquePaths := make(map[string]bool) + for _, file := range files { + uniquePaths[filepath.Dir(file.Dest)] = true } - if shouldAddFile { - containerFiles = append(containerFiles, v1.VolumeMount{ - Name: name + podFilesConfigmapNameSuffix, - MountPath: file.Dest, - SubPath: fmt.Sprintf("%d", n), + for path := range uniquePaths { + containerVolumes = append(containerVolumes, v1.VolumeMount{ + Name: name, + MountPath: path, + SubPath: strings.TrimPrefix(path, "/"), }) } } + var containerFiles []v1.VolumeMount + return append(containerVolumes, containerFiles...) } @@ -443,11 +454,27 @@ func buildInitContainerVolumes(name string, volumes []*Volume, files []*File) [] return []v1.VolumeMount{} // return empty slice if no volumes are specified } - containerVolumes := []v1.VolumeMount{ - { + var containerVolumes []v1.VolumeMount + // if the user want do add volumes, we need to mount the knuu path + if len(volumes) != 0 { + containerVolumes = append(containerVolumes, v1.VolumeMount{ Name: name, - MountPath: knuuPath, // set the path to "/knuu" as per the requirements - }, + MountPath: knuuPath, + }) + } + // if the user don't want to add volumes, but want to add files, we need to mount the knuu path for the init container + if len(volumes) == 0 && len(files) != 0 { + uniquePaths := make(map[string]bool) + for _, file := range files { + uniquePaths[filepath.Dir(file.Dest)] = true + } + for path := range uniquePaths { + containerVolumes = append(containerVolumes, v1.VolumeMount{ + Name: name, + MountPath: knuuPath + path, + SubPath: strings.TrimPrefix(path, "/"), + }) + } } var containerFiles []v1.VolumeMount @@ -484,26 +511,33 @@ func (c *Client) buildInitContainerCommand(volumes []*Volume, files []*File) []s cmds = append(cmds, parentDirCmd) dirsProcessed[folder] = true } - copyFileToKnuu := fmt.Sprintf("cp %s %s && ", file.Dest, filepath.Join(knuuPath, file.Dest)) - cmds = append(cmds, copyFileToKnuu) + chown := file.Chown + permission := file.Permission + addFileToKnuu := fmt.Sprintf("cp %s %s && ", file.Dest, filepath.Join(knuuPath, file.Dest)) + if chown != "" { + addFileToKnuu += fmt.Sprintf("chown %s %s && ", chown, filepath.Join(knuuPath, file.Dest)) + } + if permission != "" { + addFileToKnuu += fmt.Sprintf("chmod %s %s && ", permission, filepath.Join(knuuPath, file.Dest)) + } + cmds = append(cmds, addFileToKnuu) } // for each volume, copy the contents of the volume to the knuu volume - for i, volume := range volumes { + for _, volume := range volumes { knuuVolumePath := fmt.Sprintf("%s%s", knuuPath, volume.Path) cmd := fmt.Sprintf("if [ -d %s ] && [ \"$(ls -A %s)\" ]; then mkdir -p %s && cp -r %s/* %s && chown -R %d:%d %s", volume.Path, volume.Path, knuuVolumePath, volume.Path, knuuVolumePath, volume.Owner, volume.Owner, knuuVolumePath) - if i < len(volumes)-1 { - cmd += " ;fi && " - } else { - cmd += " ;fi" - } + cmd += " ;fi && " cmds = append(cmds, cmd) } fullCommand := strings.Join(cmds, "") commands = append(commands, fullCommand) + if strings.HasSuffix(fullCommand, " && ") { + commands[len(commands)-1] = strings.TrimSuffix(commands[len(commands)-1], " && ") + } c.logger.WithField("command", fullCommand).Debug("init container command") return commands @@ -522,6 +556,25 @@ func buildResources(memoryRequest, memoryLimit, cpuRequest resource.Quantity) v1 } } +func buildPodPorts(tcpPorts, udpPorts []int) []v1.ContainerPort { + ports := make([]v1.ContainerPort, 0, len(tcpPorts)+len(udpPorts)) + for _, port := range tcpPorts { + ports = append(ports, v1.ContainerPort{ + Name: fmt.Sprintf("tcp-%d", port), + Protocol: v1.ProtocolTCP, + ContainerPort: int32(port), + }) + } + for _, port := range udpPorts { + ports = append(ports, v1.ContainerPort{ + Name: fmt.Sprintf("udp-%d", port), + Protocol: v1.ProtocolUDP, + ContainerPort: int32(port), + }) + } + return ports +} + // prepareContainer creates a v1.Container from a given ContainerConfig. func prepareContainer(config ContainerConfig) v1.Container { return v1.Container{ @@ -533,6 +586,7 @@ func prepareContainer(config ContainerConfig) v1.Container { Env: buildEnv(config.Env), VolumeMounts: buildContainerVolumes(config.Name, config.Volumes, config.Files), Resources: buildResources(config.MemoryRequest, config.MemoryLimit, config.CPURequest), + Ports: buildPodPorts(config.TCPPorts, config.UDPPorts), LivenessProbe: config.LivenessProbe, ReadinessProbe: config.ReadinessProbe, StartupProbe: config.StartupProbe, @@ -542,35 +596,14 @@ func prepareContainer(config ContainerConfig) v1.Container { // prepareInitContainers creates a slice of v1.Container as init containers. func (c *Client) prepareInitContainers(config ContainerConfig, init bool) []v1.Container { - if strings.HasPrefix(config.Name, "registry-build-from-git") { - if exists, err := c.ConfigMapExists(context.Background(), "ttlsh-registry-certs"); err == nil && exists { - return []v1.Container{ - { - Name: "init-container", - Image: "debian:bullseye-slim", - // SecurityContext: &v1.SecurityContext{ - // RunAsUser: ptr.To[int64](defaultContainerUser), - // }, - Command: []string{"sh", "-c", "apt-get update && apt-get install -y ca-certificates && cp /certs/cert.pem /usr/local/share/ca-certificates/registry-cert.crt && update-ca-certificates"}, - VolumeMounts: []v1.VolumeMount{ - { - Name: "cert-volume", - MountPath: "/certs", - }, - }, - }, - } - } - } - - if !init || len(config.Volumes) == 0 { + if !init || (len(config.Volumes) == 0 && len(config.Files) == 0) { return nil } return []v1.Container{ { Name: config.Name + initContainerNameSuffix, - Image: config.Image, + Image: initContainerImage, SecurityContext: &v1.SecurityContext{ RunAsUser: ptr.To[int64](defaultContainerUser), }, @@ -588,7 +621,6 @@ func preparePodVolumes(config ContainerConfig) []v1.Volume { func (c *Client) preparePodSpec(spec PodConfig, init bool) v1.PodSpec { podSpec := v1.PodSpec{ ServiceAccountName: spec.ServiceAccountName, - SecurityContext: &v1.PodSecurityContext{FSGroup: &spec.FsGroup}, InitContainers: c.prepareInitContainers(spec.ContainerConfig, init), Containers: []v1.Container{prepareContainer(spec.ContainerConfig)}, Volumes: preparePodVolumes(spec.ContainerConfig), @@ -612,9 +644,11 @@ func (c *Client) preparePodSpec(spec PodConfig, init bool) v1.PodSpec { // Prepare sidecar containers and append to the pod spec for _, sidecarConfig := range spec.SidecarConfigs { + sidecarInitContainer := c.prepareInitContainers(sidecarConfig, true) sidecarContainer := prepareContainer(sidecarConfig) sidecarVolumes := preparePodVolumes(sidecarConfig) + podSpec.InitContainers = append(podSpec.InitContainers, sidecarInitContainer...) podSpec.Containers = append(podSpec.Containers, sidecarContainer) podSpec.Volumes = append(podSpec.Volumes, sidecarVolumes...) } diff --git a/pkg/k8s/types.go b/pkg/k8s/types.go index ae33b6a..9ac06e1 100644 --- a/pkg/k8s/types.go +++ b/pkg/k8s/types.go @@ -66,7 +66,7 @@ type KubeManager interface { Namespace() string NamespaceExists(ctx context.Context, name string) (bool, error) NetworkPolicyExists(ctx context.Context, name string) bool - NewFile(source, dest string) *File + NewFile(source, dest, chown, permission string) *File NewVolume(path string, size resource.Quantity, owner int64) *Volume PatchService(ctx context.Context, name string, opts ServiceOptions) (*corev1.Service, error) PortForwardPod(ctx context.Context, podName string, localPort, remotePort int) error diff --git a/pkg/sidecars/observability/obsy.go b/pkg/sidecars/observability/obsy.go index 413bb86..f96f4dc 100644 --- a/pkg/sidecars/observability/obsy.go +++ b/pkg/sidecars/observability/obsy.go @@ -20,7 +20,7 @@ const ( otelAgentName = "otel-agent" // %s will be replaced with the otelCollectorVersion otelAgentConfigFile = "/etc/otel-agent.yaml" - otelAgentConfigFilePermissions = "0:0" + otelAgentConfigFilePermissions = "10001:10001" otelCollectorCommand = "/otelcol-contrib" otelCollectorConfigArg = "--config=/etc/otel-agent.yaml" diff --git a/pkg/traefik/traefik.go b/pkg/traefik/traefik.go index 381dfe1..e273bdc 100644 --- a/pkg/traefik/traefik.go +++ b/pkg/traefik/traefik.go @@ -17,7 +17,6 @@ import ( "k8s.io/utils/ptr" "github.com/celestiaorg/knuu/pkg/k8s" - "github.com/celestiaorg/knuu/pkg/names" ) const ( @@ -26,6 +25,7 @@ const ( Port = 80 PortSecure = 443 deploymentName = "traefik-deployment" + serviceAccountName = "traefik-service-account" roleName = "traefik-role" containerName = "traefik" image = "traefik:v3.0" @@ -51,22 +51,14 @@ func (t *Traefik) Deploy(ctx context.Context) error { return ErrTraefikClientNotInitialized } - // Create a dedicated service account for Traefik - serviceAccountName, err := names.NewRandomK8("traefik-service-account") - if err != nil { - return err - } if err := t.K8sClient.CreateServiceAccount(ctx, serviceAccountName, nil); err != nil { return ErrFailedToCreateServiceAccount.Wrap(err) } - clusterRoleName, err := names.NewRandomK8(roleName) - if err != nil { - return err - } + clusterRoleName := k8s.SanitizeName(t.K8sClient.Namespace() + "-" + roleName) // Define and create a ClusterRole for Traefik - err = t.K8sClient.CreateClusterRole(ctx, clusterRoleName, nil, []rbacv1.PolicyRule{ + err := t.K8sClient.CreateClusterRole(ctx, clusterRoleName, nil, []rbacv1.PolicyRule{ { APIGroups: []string{""}, // Core group Resources: []string{"pods", "endpoints", "secrets", "services"}, @@ -209,10 +201,7 @@ func (t *Traefik) Endpoint(ctx context.Context) (string, error) { } func (t *Traefik) AddHost(ctx context.Context, serviceName, prefix string, portTCP int) error { - middlewareName, err := names.NewRandomK8("strip-" + prefix) - if err != nil { - return ErrGeneratingRandomK8sName.Wrap(err) - } + middlewareName := k8s.SanitizeName(prefix + "-strip") // middleware is required to strip the prefix from the service name if err := t.createMiddleware(ctx, prefix, middlewareName); err != nil { @@ -303,11 +292,7 @@ func (t *Traefik) createIngressRoute( Resource: "ingressroutes", } - ingressRouteName, err := names.NewRandomK8("ing-route-" + prefix) - if err != nil { - return ErrTraefikIngressRouteCreationFailed.Wrap(err) - } - + ingressRouteName := k8s.SanitizeName(prefix + "-ing-route") ingressRoute := &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "traefik.io/v1alpha1", @@ -341,7 +326,7 @@ func (t *Traefik) createIngressRoute( }, } - _, err = t.K8sClient.DynamicClient().Resource(ingressRouteGVR).Namespace(t.K8sClient.Namespace()). + _, err := t.K8sClient.DynamicClient().Resource(ingressRouteGVR).Namespace(t.K8sClient.Namespace()). Create(ctx, ingressRoute, metav1.CreateOptions{}) if err != nil { return ErrTraefikIngressRouteCreationFailed.Wrap(err)