Skip to content

Commit

Permalink
feat(injector): add ephemeral-storage resource request/limit for side…
Browse files Browse the repository at this point in the history
…cars (#8882)

Signed-off-by: Jay Chen <[email protected]>
  • Loading branch information
jijiechen authored Jan 23, 2024
1 parent 7a68c4c commit 2b1cac3
Show file tree
Hide file tree
Showing 35 changed files with 271 additions and 148 deletions.
11 changes: 7 additions & 4 deletions pkg/plugins/runtime/k8s/containers/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

runtime_k8s "github.com/kumahq/kuma/pkg/config/plugins/runtime/k8s"
"github.com/kumahq/kuma/pkg/plugins/runtime/k8s/metadata"
"github.com/kumahq/kuma/pkg/util/pointer"
)

type EnvVarsByName []kube_core.EnvVar
Expand Down Expand Up @@ -158,12 +159,14 @@ func (i *DataplaneProxyFactory) NewContainer(
},
Resources: kube_core.ResourceRequirements{
Requests: kube_core.ResourceList{
kube_core.ResourceCPU: kube_api.MustParse(i.ContainerConfig.Resources.Requests.CPU),
kube_core.ResourceMemory: kube_api.MustParse(i.ContainerConfig.Resources.Requests.Memory),
kube_core.ResourceCPU: kube_api.MustParse(i.ContainerConfig.Resources.Requests.CPU),
kube_core.ResourceMemory: kube_api.MustParse(i.ContainerConfig.Resources.Requests.Memory),
kube_core.ResourceEphemeralStorage: pointer.Deref(kube_api.NewScaledQuantity(50, kube_api.Mega)),
},
Limits: kube_core.ResourceList{
kube_core.ResourceCPU: kube_api.MustParse(i.ContainerConfig.Resources.Limits.CPU),
kube_core.ResourceMemory: kube_api.MustParse(i.ContainerConfig.Resources.Limits.Memory),
kube_core.ResourceCPU: kube_api.MustParse(i.ContainerConfig.Resources.Limits.CPU),
kube_core.ResourceMemory: kube_api.MustParse(i.ContainerConfig.Resources.Limits.Memory),
kube_core.ResourceEphemeralStorage: pointer.Deref(kube_api.NewScaledQuantity(1, kube_api.Giga)),
},
},
}
Expand Down
119 changes: 7 additions & 112 deletions pkg/plugins/runtime/k8s/webhooks/injector/injector.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,27 +73,10 @@ type KumaInjector struct {
}

func (i *KumaInjector) InjectKuma(ctx context.Context, pod *kube_core.Pod) error {
ns, err := i.namespaceFor(ctx, pod)
if err != nil {
return errors.Wrap(err, "could not retrieve namespace for pod")
}
logger := log.WithValues("pod", pod.GenerateName, "namespace", pod.Namespace)
// Log deprecated annotations
for _, d := range metadata.PodAnnotationDeprecations {
if _, exists := pod.Annotations[d.Key]; exists {
logger.Info("WARNING: using deprecated pod annotation", "key", d.Key, "message", d.Message)
}
}
if inject, err := i.needInject(pod, ns); err != nil {
return err
} else if !inject {
logger.V(1).Info("skip injecting Kuma")
return nil
}
meshName := k8s_util.MeshOfByAnnotation(pod, ns)
logger = logger.WithValues("mesh", meshName)
// Check mesh exists
if err := i.client.Get(ctx, kube_types.NamespacedName{Name: meshName}, &mesh_k8s.Mesh{}); err != nil {

meshName, err := i.preCheck(ctx, pod, logger)
if meshName == "" || err != nil {
return err
}

Expand All @@ -107,48 +90,12 @@ func (i *KumaInjector) InjectKuma(ctx context.Context, pod *kube_core.Pod) error
if err != nil {
return err
}

// Warn if an init container in the pod is using the same UID as the sidecar. This traffic will be exempt from
// redirection and may be unintended behavior.
for _, c := range pod.Spec.InitContainers {
if c.SecurityContext != nil && c.SecurityContext.RunAsUser != nil {
if *c.SecurityContext.RunAsUser == i.cfg.SidecarContainer.UID {
logger.Info(
"WARNING: init container using ignored sidecar UID",
"container",
c.Name,
"uid",
i.cfg.SidecarContainer.UID,
)
}
}
}

var duplicateUidContainers []string
// Error if a container in the pod is using the same UID as the sidecar. This scenario is not supported.
for _, c := range pod.Spec.Containers {
if c.SecurityContext != nil && c.SecurityContext.RunAsUser != nil {
if *c.SecurityContext.RunAsUser == i.cfg.SidecarContainer.UID {
duplicateUidContainers = append(duplicateUidContainers, c.Name)
}
}
}

if len(duplicateUidContainers) > 0 {
err := fmt.Errorf(
"containers using same UID as sidecar is unsupported: %q",
duplicateUidContainers,
)

logger.Error(err, "injection failed")

return err
}

sidecarTmp := kube_core.Volume{
Name: "kuma-sidecar-tmp",
VolumeSource: kube_core.VolumeSource{
EmptyDir: &kube_core.EmptyDirVolumeSource{},
EmptyDir: &kube_core.EmptyDirVolumeSource{
SizeLimit: kube_api.NewScaledQuantity(10, kube_api.Mega),
},
},
}
pod.Spec.Volumes = append(pod.Spec.Volumes, sidecarTmp)
Expand Down Expand Up @@ -232,54 +179,6 @@ func (i *KumaInjector) InjectKuma(ctx context.Context, pod *kube_core.Pod) error
return nil
}

func (i *KumaInjector) needInject(pod *kube_core.Pod, ns *kube_core.Namespace) (bool, error) {
log.WithValues("name", pod.Name, "namespace", pod.Namespace)
if i.isInjectionException(pod) {
log.V(1).Info("pod fulfills exception requirements")
return false, nil
}

for _, container := range pod.Spec.Containers {
if container.Name == k8s_util.KumaSidecarContainerName {
log.V(1).Info("pod already has Kuma sidecar")
return false, nil
}
}

enabled, exist, err := metadata.Annotations(pod.Labels).GetEnabled(metadata.KumaSidecarInjectionAnnotation)
if err != nil {
return false, err
}
if exist {
if !enabled {
log.V(1).Info(`pod has "kuma.io/sidecar-injection: disabled" label`)
}
return enabled, nil
}

enabled, exist, err = metadata.Annotations(ns.Labels).GetEnabled(metadata.KumaSidecarInjectionAnnotation)
if err != nil {
return false, err
}
if exist {
if !enabled {
log.V(1).Info(`namespace has "kuma.io/sidecar-injection: disabled" label`)
}
return enabled, nil
}
return false, err
}

func (i *KumaInjector) isInjectionException(pod *kube_core.Pod) bool {
for key, value := range i.cfg.Exceptions.Labels {
podValue, exist := pod.Labels[key]
if exist && (value == "*" || value == podValue) {
return true
}
}
return false
}

type namedContainerPatches struct {
names []string
patches []mesh_k8s.JsonPatchBlock
Expand Down Expand Up @@ -404,7 +303,6 @@ func (i *KumaInjector) NewSidecarContainer(
}

container.Name = k8s_util.KumaSidecarContainerName

return container, nil
}

Expand Down Expand Up @@ -482,13 +380,10 @@ func (i *KumaInjector) NewInitContainer(pod *kube_core.Pod) (kube_core.Container
}

if i.cfg.EBPF.Enabled {
// container.SecurityContext.Privileged expects to have a reference
// to the bool value
tru := true
bidirectional := kube_core.MountPropagationBidirectional

container.SecurityContext.Capabilities = &kube_core.Capabilities{}
container.SecurityContext.Privileged = &tru
container.SecurityContext.Privileged = pointer.To(true)

container.Env = []kube_core.EnvVar{
{
Expand Down
129 changes: 129 additions & 0 deletions pkg/plugins/runtime/k8s/webhooks/injector/precheck.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
package injector

import (
"context"
"fmt"

"github.com/go-logr/logr"
"github.com/pkg/errors"
kube_core "k8s.io/api/core/v1"
kube_types "k8s.io/apimachinery/pkg/types"

mesh_k8s "github.com/kumahq/kuma/pkg/plugins/resources/k8s/native/api/v1alpha1"
"github.com/kumahq/kuma/pkg/plugins/runtime/k8s/metadata"
k8s_util "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/util"
)

func (i *KumaInjector) preCheck(ctx context.Context, pod *kube_core.Pod, logger logr.Logger) (string, error) {
ns, err := i.namespaceFor(ctx, pod)
if err != nil {
return "", errors.Wrap(err, "could not retrieve namespace for pod")
}

// Log deprecated annotations
for _, d := range metadata.PodAnnotationDeprecations {
if _, exists := pod.Annotations[d.Key]; exists {
logger.Info("WARNING: using deprecated pod annotation", "key", d.Key, "message", d.Message)
}
}

if inject, err := i.needToInject(pod, ns); err != nil {
return "", err
} else if !inject {
logger.V(1).Info("skipping Kuma injection")
return "", nil
}

meshName := k8s_util.MeshOfByAnnotation(pod, ns)
logger = logger.WithValues("mesh", meshName)
// Check mesh exists
if err := i.client.Get(ctx, kube_types.NamespacedName{Name: meshName}, &mesh_k8s.Mesh{}); err != nil {
return "", err
}

// Warn if an init container in the pod is using the same UID as the sidecar. This traffic will be exempt from
// redirection and may be unintended behavior.
for _, c := range pod.Spec.InitContainers {
if c.SecurityContext != nil && c.SecurityContext.RunAsUser != nil {
if *c.SecurityContext.RunAsUser == i.cfg.SidecarContainer.UID {
logger.Info(
"WARNING: init container using ignored sidecar UID",
"container",
c.Name,
"uid",
i.cfg.SidecarContainer.UID,
)
}
}
}

var duplicateUidContainers []string
// Error if a container in the pod is using the same UID as the sidecar. This scenario is not supported.
for _, c := range pod.Spec.Containers {
if c.SecurityContext != nil && c.SecurityContext.RunAsUser != nil {
if *c.SecurityContext.RunAsUser == i.cfg.SidecarContainer.UID {
duplicateUidContainers = append(duplicateUidContainers, c.Name)
}
}
}

if len(duplicateUidContainers) > 0 {
err := fmt.Errorf(
"containers using same UID as sidecar is unsupported: %q",
duplicateUidContainers,
)

logger.Error(err, "injection failed")

return "", err
}
return meshName, nil
}

func (i *KumaInjector) needToInject(pod *kube_core.Pod, ns *kube_core.Namespace) (bool, error) {
log.WithValues("name", pod.Name, "namespace", pod.Namespace)
if i.isInjectionException(pod) {
log.V(1).Info("pod fulfills exception requirements")
return false, nil
}

for _, container := range pod.Spec.Containers {
if container.Name == k8s_util.KumaSidecarContainerName {
log.V(1).Info("pod already has Kuma sidecar")
return false, nil
}
}

enabled, exist, err := metadata.Annotations(pod.Labels).GetEnabled(metadata.KumaSidecarInjectionAnnotation)
if err != nil {
return false, err
}
if exist {
if !enabled {
log.V(1).Info(`pod has "kuma.io/sidecar-injection: disabled" label`)
}
return enabled, nil
}

enabled, exist, err = metadata.Annotations(ns.Labels).GetEnabled(metadata.KumaSidecarInjectionAnnotation)
if err != nil {
return false, err
}
if exist {
if !enabled {
log.V(1).Info(`namespace has "kuma.io/sidecar-injection: disabled" label`)
}
return enabled, nil
}
return false, err
}

func (i *KumaInjector) isInjectionException(pod *kube_core.Pod) bool {
for key, value := range i.cfg.Exceptions.Labels {
podValue, exist := pod.Labels[key]
if exist && (value == "*" || value == podValue) {
return true
}
}
return false
}
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,11 @@ spec:
limits:
cpu: 1100m
memory: 1512Mi
ephemeral-storage: 1G
requests:
cpu: 150m
memory: 164Mi
ephemeral-storage: 50M
securityContext:
readOnlyRootFilesystem: true
runAsGroup: 5678
Expand Down Expand Up @@ -158,6 +160,7 @@ spec:
- name: default-token-w7dxf
secret:
secretName: default-token-w7dxf
- emptyDir: {}
- emptyDir:
sizeLimit: 10M
name: kuma-sidecar-tmp
status: {}
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,11 @@ spec:
limits:
cpu: 1100m
memory: 1512Mi
ephemeral-storage: 1G
requests:
cpu: 150m
memory: 164Mi
ephemeral-storage: 50M
securityContext:
readOnlyRootFilesystem: true
runAsGroup: 5678
Expand Down Expand Up @@ -166,6 +168,7 @@ spec:
- name: default-token-w7dxf
secret:
secretName: default-token-w7dxf
- emptyDir: {}
- emptyDir:
sizeLimit: 10M
name: kuma-sidecar-tmp
status: {}
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,11 @@ spec:
limits:
cpu: 1100m
memory: 1512Mi
ephemeral-storage: 1G
requests:
cpu: 150m
memory: 164Mi
ephemeral-storage: 50M
securityContext:
readOnlyRootFilesystem: true
runAsGroup: 5678
Expand Down Expand Up @@ -250,6 +252,7 @@ spec:
- name: coredns-token-9gmrh
secret:
secretName: coredns-token-9gmrh
- emptyDir: {}
- emptyDir:
sizeLimit: 10M
name: kuma-sidecar-tmp
status: {}
Loading

0 comments on commit 2b1cac3

Please sign in to comment.