From f4ea216fbd9ef8b5ff109522c59ee25037ab3405 Mon Sep 17 00:00:00 2001 From: Deshi Xiao Date: Sat, 9 Mar 2024 13:29:57 +0800 Subject: [PATCH 1/5] Don't request metadata when listing objects While some implementations may support it, it appears that most don't, and some may in fact return an error if it is requested. We already stat the object to get the metadata anyway, so this was unnecessary if harmless on most implementations. Signed-off-by: Brad Davidson porting by Signed-off-by: Deshi Xiao --- pkg/etcd/s3.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index bc3c24e3..2720fdff 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -358,9 +358,8 @@ func (s *S3) listSnapshots(ctx context.Context) (map[string]snapshotFile, error) defer cancel() opts := minio.ListObjectsOptions{ - Prefix: s.config.EtcdS3Folder, - Recursive: true, - WithMetadata: true, + Prefix: s.config.EtcdS3Folder, + Recursive: true, } objects := s.client.ListObjects(ctx, s.config.EtcdS3BucketName, opts) From 80ab27d4e17eab17f5a7b7acecce96fdbc8f7b59 Mon Sep 17 00:00:00 2001 From: Deshi Xiao Date: Sat, 9 Mar 2024 13:33:48 +0800 Subject: [PATCH 2/5] Don't apply s3 retention if S3 client failed to initialize Signed-off-by: Brad Davidson porting by Signed-off-by: Deshi Xiao --- pkg/etcd/snapshot.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/etcd/snapshot.go b/pkg/etcd/snapshot.go index 98f3bf8f..a99dba48 100644 --- a/pkg/etcd/snapshot.go +++ b/pkg/etcd/snapshot.go @@ -368,14 +368,16 @@ func (e *ETCD) Snapshot(ctx context.Context) error { } else { logrus.Infof("S3 upload complete for %s", snapshotName) } + // Attempt to apply retention even if the upload failed; failure may be due to bucket + // being full or some other condition that retention policy would resolve. + if err := e.s3.snapshotRetention(ctx); err != nil { + logrus.Errorf("Failed to apply s3 snapshot retention policy: %v", err) + } } + // sf is either s3 snapshot metadata, or s3 failure record if err := e.addSnapshotData(*sf); err != nil { return errors.Wrap(err, "failed to sync ETCDSnapshotFile") } - if err := e.s3.snapshotRetention(ctx); err != nil { - logrus.Errorf("Failed to apply s3 snapshot retention policy: %v", err) - } - } } @@ -547,6 +549,7 @@ func (e *ETCD) DeleteSnapshots(ctx context.Context, snapshots []string) error { } if e.config.EtcdS3 { if err := e.initS3IfNil(ctx); err != nil { + logrus.Warnf("Unable to initialize S3 client: %v", err) return err } } @@ -740,6 +743,7 @@ func (e *ETCD) ReconcileSnapshotData(ctx context.Context) error { // Get snapshots from S3 if e.config.EtcdS3 { if err := e.initS3IfNil(ctx); err != nil { + logrus.Warnf("Unable to initialize S3 client: %v", err) return err } From bfdee0a6ecdf857968e7776ad2875c22bb0d2e12 Mon Sep 17 00:00:00 2001 From: Deshi Xiao Date: Sat, 9 Mar 2024 13:36:10 +0800 Subject: [PATCH 3/5] Print key instead of file path in snapshot metadata log message Signed-off-by: Brad Davidson porting by Signed-off-by: Deshi Xiao --- pkg/etcd/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index 2720fdff..6cd8ddbf 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -178,7 +178,7 @@ func (s *S3) upload(ctx context.Context, snapshot string, extraMetadata *v1.Conf if _, err := s.uploadSnapshotMetadata(ctx, metadataKey, metadata); err != nil { logrus.Warnf("Failed to upload snapshot metadata to S3: %v", err) } else { - logrus.Infof("Uploaded snapshot metadata s3://%s/%s", s.config.EtcdS3BucketName, metadata) + logrus.Infof("Uploaded snapshot metadata s3://%s/%s", s.config.EtcdS3BucketName, metadataKey) } return sf, err } From 08e1c70ecf56a808fa5b4d4cf8d7e882a82edd5a Mon Sep 17 00:00:00 2001 From: Deshi Xiao Date: Sat, 9 Mar 2024 13:39:59 +0800 Subject: [PATCH 4/5] Update to v1.25.16 * Update to v1.25.16 Signed-off-by: Johnatas porting by Signed-off-by: Deshi Xiao --- go.mod | 64 ++++++++++++++++++------------------ go.sum | 100 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 82 insertions(+), 82 deletions(-) diff --git a/go.mod b/go.mod index 79f9c9d2..c38a4faf 100644 --- a/go.mod +++ b/go.mod @@ -49,38 +49,38 @@ replace ( google.golang.org/genproto => google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 google.golang.org/grpc => google.golang.org/grpc v1.58.3 gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.2.2 - k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.25.15-k3s1 - k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.25.15-k3s1 - k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.25.15-k3s1 - k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.25.15-k3s1 - k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.25.15-k3s1 - k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.25.15-k3s1 - k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.25.15-k3s1 - k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.25.15-k3s1 - k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.25.15-k3s1 - k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.25.15-k3s1 - k8s.io/component-helpers => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.25.15-k3s1 - k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.25.15-k3s1 - k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.25.15-k3s1 - k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.25.15-k3s1 + k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.25.16-k3s1 + k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.25.16-k3s1 + k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.25.16-k3s1 + k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.25.16-k3s1 + k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.25.16-k3s1 + k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.25.16-k3s1 + k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.25.16-k3s1 + k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.25.16-k3s1 + k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.25.16-k3s1 + k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.25.16-k3s1 + k8s.io/component-helpers => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.25.16-k3s1 + k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.25.16-k3s1 + k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.25.16-k3s1 + k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.25.16-k3s1 k8s.io/klog => github.com/k3s-io/klog v1.0.0-k3s2 // k3s-release-1.x k8s.io/klog/v2 => github.com/k3s-io/klog/v2 v2.60.1-k3s1 // k3s-main - k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.25.15-k3s1 - k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.25.15-k3s1 + k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.25.16-k3s1 + k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.25.16-k3s1 k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 - k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.25.15-k3s1 - k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.25.15-k3s1 - k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.25.15-k3s1 - k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.25.15-k3s1 - k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.25.15-k3s1 - k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.25.15-k3s1 - k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.25.15-k3s1 - k8s.io/mount-utils => github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.25.15-k3s1 - k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.25.15-k3s1 - k8s.io/pod-security-admission => github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.25.15-k3s1 - k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.25.15-k3s1 - k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.25.15-k3s1 - k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.25.15-k3s1 + k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.25.16-k3s1 + k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.25.16-k3s1 + k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.25.16-k3s1 + k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.25.16-k3s1 + k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.25.16-k3s1 + k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.25.16-k3s1 + k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.25.16-k3s1 + k8s.io/mount-utils => github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.25.16-k3s1 + k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.25.16-k3s1 + k8s.io/pod-security-admission => github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.25.16-k3s1 + k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.25.16-k3s1 + k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.25.16-k3s1 + k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.25.16-k3s1 mvdan.cc/unparam => mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 ) @@ -143,15 +143,15 @@ require ( k8s.io/apimachinery v0.27.5 k8s.io/apiserver v0.27.0 k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible - k8s.io/cloud-provider v0.25.15 + k8s.io/cloud-provider v0.25.16 k8s.io/cluster-bootstrap v0.0.0 k8s.io/component-base v0.27.0 k8s.io/component-helpers v0.27.0 k8s.io/cri-api v0.27.5 k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.100.1 - k8s.io/kubectl v0.25.15 - k8s.io/kubernetes v1.25.15 + k8s.io/kubectl v0.25.16 + k8s.io/kubernetes v1.25.16 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 sigs.k8s.io/yaml v1.3.0 ) diff --git a/go.sum b/go.sum index a149aef9..9f76245d 100644 --- a/go.sum +++ b/go.sum @@ -644,56 +644,56 @@ github.com/k3s-io/klog v1.0.0-k3s2 h1:yyvD2bQbxG7m85/pvNctLX2bUDmva5kOBvuZ77tTGB github.com/k3s-io/klog v1.0.0-k3s2/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= github.com/k3s-io/klog/v2 v2.60.1-k3s1 h1:C1hsMF1Eo6heGVQzts6cZ+rDZAReSiOBUxsYMuUkkZI= github.com/k3s-io/klog/v2 v2.60.1-k3s1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -github.com/k3s-io/kubernetes v1.25.15-k3s1 h1:IkNiJleJ8O7b0uhUcLSo6TD4wcTVeWqBCks+sWH8N5g= -github.com/k3s-io/kubernetes v1.25.15-k3s1/go.mod h1:ZrRRMmY0nDNxlnPnJopZRhF2i0tzfB3+DnitJcDe8Vw= -github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.25.15-k3s1 h1:ZUvIOQa0G9h6dDf4ENxwbuXhatXgtmNV5INzbXfBAI0= -github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.25.15-k3s1/go.mod h1:/N8bcrUz/rMP42kr63tx/hw8vm8Y7p56XFzZgWfhvMo= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.25.15-k3s1 h1:0jenTH+slgNz+as9tTE3ElD+aFZU3LSYa43X3Nc6kcQ= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.25.15-k3s1/go.mod h1:d/I3BjIbV80Dkus7MugFoUk2WlRSzsfaBGSC+ntsWZg= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.25.15-k3s1 h1:rwGdQ3loCyWe0Xp9m5khQhzzKD5QcUJR2eQJgWGbSA4= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.25.15-k3s1/go.mod h1:I5/T97TnTcndA0hLRTsTqeRRQG4RGDk8uZcbs7lMwhE= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.25.15-k3s1 h1:Lg6Z2mzLWj/mf9DjUir1SabUhLaUWuMkmr9n2DywghA= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.25.15-k3s1/go.mod h1:+ehVFHjV8ewBesDyBWMn21gGuGaP7jms2jvf+aN+B34= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.25.15-k3s1 h1:xNU2NZrYWe7Y0oGjShSFqZRNlG32P2GlFwfwVYYq7uU= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.25.15-k3s1/go.mod h1:53ICk6yby1KL6nFqjqxKEBxuGz22fTM8kN50saMpUJs= -github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.25.15-k3s1 h1:W8WwWLE6FA9ldzyO+zdaXJCkOQw4TSS6mCTRWt4bgz4= -github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.25.15-k3s1/go.mod h1:iCBSdKXcLFqYDOLMgaZWK0cB3kVp7TCR+eSs56BDUnY= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.25.15-k3s1 h1:+kBJ6royZr9B6T0KFMdh6eNCNTsKzmBSNKqbsOG7IOw= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.25.15-k3s1/go.mod h1:WKmYlger6OXeolhTZBWytmvy+85J7fdwqFPv2Rfvof8= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.25.15-k3s1 h1:vKYNfo+d2Ut8zbiJpy9YjQTDrhDZ+LeTl9JLQcmyajc= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.25.15-k3s1/go.mod h1:zrULy04n3XUbx60Nt0SezEPCUCac5azDUAyEBfxZEAQ= -github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.25.15-k3s1 h1:/FjieemXEjHFuEZI3hxmx0yWjmd7WYRNhPgqW6giIAM= -github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.25.15-k3s1/go.mod h1:HYaqm/S966PX6yJtewIFP0Bd7QQO8UnDGgVu2gT1glM= -github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.25.15-k3s1 h1:amEe1XBQnQQcbo8miDMrrS2g4+iUgSAdqZLZh2XrQXQ= -github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.25.15-k3s1/go.mod h1:MBxBJEjOchffxgihy+O3qQD3OT9P7mnatA85x5dxO/Q= -github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.25.15-k3s1 h1:s4ryne2TPpClKT5CSKcmpg9T+RR7aNu1h0qwtInBeUM= -github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.25.15-k3s1/go.mod h1:fo9V6qV3s0BXcgdJV3wmisFS3H3DnsSfpjWrfGvPUaE= -github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.25.15-k3s1 h1:zVU2maYLxOmz++Z2gvD3W8wsuiqFUaRkUB4pVrF3Dy0= -github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.25.15-k3s1/go.mod h1:PeQzOnq8NDoaP/UG7OTmVW5xJabzhUCeAipkAsH0rfI= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.25.15-k3s1 h1:OvxK4fgBomcSNuXptJRAuxogeiMJPiK2Ki19xcvBi4k= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.25.15-k3s1/go.mod h1:fjRHAkEDvhp4CYP+uH2fwDaVphp7CvNqBwMK4W6u3CI= -github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.25.15-k3s1 h1:Wcb1DDfe+23rRpT+ZKtILBsnCPTSLwjYqqqooodNJLk= -github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.25.15-k3s1/go.mod h1:IEPPKueX822CfmQMaPJIdZkIN3sVmJMseMV5qee067I= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.25.15-k3s1 h1:i9Qlg9y9YPHMStEe2GrYuE9qU3FcLc1/lKQ2gXOWFW8= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.25.15-k3s1/go.mod h1:M+yHpazu/Q7h/sDtYnTd0tPlx/oIooBYcUWLOlSjylE= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.25.15-k3s1 h1:eIDGoamtKtWiUhf/xIHOJofxvmrqcFJBbNHUjzzUfHg= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.25.15-k3s1/go.mod h1:fw3oJiD0qKJSk5X3CYET9Y0ca5/2kaNhzcyWAC/x1YE= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.25.15-k3s1/go.mod h1:2c4F3OIIqIDXcNR/DiI2Wz56nlLh7lKd9ofA0m4BZ9c= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.25.15-k3s1 h1:cy80/faKaLvdeJkQeQ6ZaJJCWWWX2ZgiSkZcUQ2CqNM= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.25.15-k3s1/go.mod h1:Hlpchmj7z0fZqDm4H391IfBtQ0gqJTp2aaDlV6CIf8s= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.25.15-k3s1 h1:MRvVWMgDH7IxYAjYOCu99by3Lq0qs+zYSj1tcITvEEw= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.25.15-k3s1/go.mod h1:i6Ffq/3boXLMwfDKYNBKRvom1HRthHjSskxV61bFuPE= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.25.15-k3s1 h1:Jka+LYT1P6PWE9SC1UO4j2o6LUaDDsSWV9Z30d5gkQw= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.25.15-k3s1/go.mod h1:49Qgz77HFyZkojPrPgKF3QcCpDEk88NeCRwi/0i+QhQ= -github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.25.15-k3s1 h1:Rf/TEWFEBZB9OjpiVHyg62W3ugGFrJ+pMnkSMcg+p8A= -github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.25.15-k3s1/go.mod h1:fB49PcIyv3A8Bg+xOQqTf7FOxCUA8C6BGGcQIwAoiu4= -github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.25.15-k3s1 h1:7kD7mSuczGyVurGP2gw06IIf08bQ2hYhvUo/aF3zVTA= -github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.25.15-k3s1/go.mod h1:tELkwF9xCmwJ3l3siCo2niUaSIyjTpZ+64S+Z8fTcUw= -github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.25.15-k3s1 h1:RHmLHwd2CJWCalJH24zRVcvSQakb/UjWe0LkzfdXTH8= -github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.25.15-k3s1/go.mod h1:Mjv5B8KCT278Nm2okrd1rijnIpNP9Oys7TIkqGF1qHY= -github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.25.15-k3s1 h1:qJTPqJTSY3qZ6XrueC7w1A4J+QcPeQMNEmBLANppMB8= -github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.25.15-k3s1/go.mod h1:Sa550rNU+kF1UB09tYc4SyKC73NCcD0vEYR+JCt6IO0= -github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.25.15-k3s1/go.mod h1:wgON9dzQlkboLpD5iQw/jJhYh4tyI9RLBo/XHZlGlKk= +github.com/k3s-io/kubernetes v1.25.16-k3s1 h1:PFCNVUmSSH1qyUdGeBCMd8LkS7fgVIc0iCcM/8hjt5o= +github.com/k3s-io/kubernetes v1.25.16-k3s1/go.mod h1:ZrRRMmY0nDNxlnPnJopZRhF2i0tzfB3+DnitJcDe8Vw= +github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.25.16-k3s1 h1:9aafKwK0KOzOvBabLvHlzS39AfVIMQZXHSIzpGLtg14= +github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.25.16-k3s1/go.mod h1:/N8bcrUz/rMP42kr63tx/hw8vm8Y7p56XFzZgWfhvMo= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.25.16-k3s1 h1:mtA75stK9PBwwXpDYPiwog7/3h2AgbhNUGbRFEJGFEM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.25.16-k3s1/go.mod h1:d/I3BjIbV80Dkus7MugFoUk2WlRSzsfaBGSC+ntsWZg= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.25.16-k3s1 h1:cdDj1YvlcOXsuEa7+O58KxOUT6RAW5il+iLJHF1pTKg= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.25.16-k3s1/go.mod h1:I5/T97TnTcndA0hLRTsTqeRRQG4RGDk8uZcbs7lMwhE= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.25.16-k3s1 h1:IoGv7iGxAqeEAdmLSjPqOvhnMrhdbnzgmxUxczQebRs= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.25.16-k3s1/go.mod h1:+ehVFHjV8ewBesDyBWMn21gGuGaP7jms2jvf+aN+B34= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.25.16-k3s1 h1:hmjTkWREnkmhrczbT9KTkorDvufWdYuAAHYklLaDnCA= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.25.16-k3s1/go.mod h1:53ICk6yby1KL6nFqjqxKEBxuGz22fTM8kN50saMpUJs= +github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.25.16-k3s1 h1:VSXpJrPdOZmNKl4fLid69o/sSeet+wQdEZvgB+MOFrM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.25.16-k3s1/go.mod h1:iCBSdKXcLFqYDOLMgaZWK0cB3kVp7TCR+eSs56BDUnY= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.25.16-k3s1 h1:TkhcdxWmseTVLEZ4NvDLBAVgerjdHzj1GnvSPxv6GIA= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.25.16-k3s1/go.mod h1:WKmYlger6OXeolhTZBWytmvy+85J7fdwqFPv2Rfvof8= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.25.16-k3s1 h1:p0x2Hh8CAblmMLGTirLAGfTNkQCGTiZDI/Sykvu+wK4= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.25.16-k3s1/go.mod h1:zrULy04n3XUbx60Nt0SezEPCUCac5azDUAyEBfxZEAQ= +github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.25.16-k3s1 h1:fGV5gLJwSUCD6+Y/pnbSjniho6bLMKSNeViqEYqX8Hs= +github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.25.16-k3s1/go.mod h1:HYaqm/S966PX6yJtewIFP0Bd7QQO8UnDGgVu2gT1glM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.25.16-k3s1 h1:o8RWF/WKEByECD023IGeeS/yT0ifeThQjYJVEhQZvB8= +github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.25.16-k3s1/go.mod h1:MBxBJEjOchffxgihy+O3qQD3OT9P7mnatA85x5dxO/Q= +github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.25.16-k3s1 h1:MhiL58ayWui5dLxz2laRm00MEEUEOxfMKgGKtFcIWKc= +github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.25.16-k3s1/go.mod h1:fo9V6qV3s0BXcgdJV3wmisFS3H3DnsSfpjWrfGvPUaE= +github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.25.16-k3s1 h1:i7X/tZQQzr7Jv5G8m5GbTQm7QmstPQzXbW5FzHVJNOI= +github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.25.16-k3s1/go.mod h1:PeQzOnq8NDoaP/UG7OTmVW5xJabzhUCeAipkAsH0rfI= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.25.16-k3s1 h1:otmcf5mYOGXMoBTIyL9QcPToYSDtp7WYvbRHCp2k3MM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.25.16-k3s1/go.mod h1:fjRHAkEDvhp4CYP+uH2fwDaVphp7CvNqBwMK4W6u3CI= +github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.25.16-k3s1 h1:eqzYjx6OoJuu3nbZC0JDo/dTtUDIMZdyWFquudRPG9s= +github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.25.16-k3s1/go.mod h1:IEPPKueX822CfmQMaPJIdZkIN3sVmJMseMV5qee067I= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.25.16-k3s1 h1:bV5OwxCITt/5kpVx7dfyn7Vq+/9MSCScgxZCKR5PiOY= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.25.16-k3s1/go.mod h1:M+yHpazu/Q7h/sDtYnTd0tPlx/oIooBYcUWLOlSjylE= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.25.16-k3s1 h1:Vcj7swoM36pf45NGBkEIUxdLcGaiMVTZTqAYCzrtVD4= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.25.16-k3s1/go.mod h1:fw3oJiD0qKJSk5X3CYET9Y0ca5/2kaNhzcyWAC/x1YE= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.25.16-k3s1/go.mod h1:2c4F3OIIqIDXcNR/DiI2Wz56nlLh7lKd9ofA0m4BZ9c= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.25.16-k3s1 h1:uxr/r3pHs0spLIMT5MKyKhuVMJioCiLI4TPGk8qxhbQ= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.25.16-k3s1/go.mod h1:Hlpchmj7z0fZqDm4H391IfBtQ0gqJTp2aaDlV6CIf8s= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.25.16-k3s1 h1:ag149duTKGYxdPUe/NbdSzE6qDfkBLGTGSI6UEWzOpA= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.25.16-k3s1/go.mod h1:i6Ffq/3boXLMwfDKYNBKRvom1HRthHjSskxV61bFuPE= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.25.16-k3s1 h1:EXvl/CSg3s6kKyBwJQOY6qLKuuAAoODoVeWFhAIWfCM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.25.16-k3s1/go.mod h1:49Qgz77HFyZkojPrPgKF3QcCpDEk88NeCRwi/0i+QhQ= +github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.25.16-k3s1 h1:eM1M/I0/L8zXidLag6s2oHuSk8duaATXQ0mvJc5MMe0= +github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.25.16-k3s1/go.mod h1:fB49PcIyv3A8Bg+xOQqTf7FOxCUA8C6BGGcQIwAoiu4= +github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.25.16-k3s1 h1:DSfgxUnWyEZFWrbvY1vMX9JxI8PB5CHLdXCP2O8BM7A= +github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.25.16-k3s1/go.mod h1:tELkwF9xCmwJ3l3siCo2niUaSIyjTpZ+64S+Z8fTcUw= +github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.25.16-k3s1 h1:GTfnOebp9SqM3tgjgIulPYKUtx+3JwzXW+3BlDnAAPQ= +github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.25.16-k3s1/go.mod h1:Mjv5B8KCT278Nm2okrd1rijnIpNP9Oys7TIkqGF1qHY= +github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.25.16-k3s1 h1:56yP2tf+ckLaou4sHZcDrrAcNCRJk271vftvl6Lanto= +github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.25.16-k3s1/go.mod h1:Sa550rNU+kF1UB09tYc4SyKC73NCcD0vEYR+JCt6IO0= +github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.25.16-k3s1/go.mod h1:wgON9dzQlkboLpD5iQw/jJhYh4tyI9RLBo/XHZlGlKk= github.com/k3s-io/stargz-snapshotter v0.14.3-k3s1 h1:29UZBqL+9Kcwl7ES7oUdN9UtS0oVZY6AL+IujwGNwPk= github.com/k3s-io/stargz-snapshotter v0.14.3-k3s1/go.mod h1:j2Ya4JeA5gMZJr8BchSkPjlcCEh++auAxp4nidPI6N0= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= From f049fe4fc8d1e807f909583c8d0d6e9f78c196ce Mon Sep 17 00:00:00 2001 From: Deshi Xiao Date: Sat, 9 Mar 2024 13:55:20 +0800 Subject: [PATCH 5/5] Added runtimes for wasm/crun/nvidia Create a generic helper function that finds extra containerd runtimes. The code was originally inside of the nvidia container discovery file. Signed-off-by: Flavio Castelli Discover the containerd shims based on runwasi that are already available on the node. The runtimes could have been installed either by a package manager or by the kwasm operator. Signed-off-by: Flavio Castelli The containerd configuration on a Linux system now handles the nvidia and the WebAssembly runtimes. Signed-off-by: Flavio Castelli Added runtime classes for crun/wasm/nvidia Signed-off-by: Vitor Savian Added default runtime flag Signed-off-by: Vitor Savian porting by Signed-off-by: Deshi Xiao --- manifests/runtimes.yaml | 59 ++ pkg/agent/containerd/config_linux.go | 9 +- pkg/agent/containerd/nvidia.go | 67 --- pkg/agent/containerd/nvidia_test.go | 219 ------- pkg/agent/containerd/runtimes.go | 161 +++++ pkg/agent/containerd/runtimes_test.go | 796 +++++++++++++++++++++++++ pkg/agent/run_linux.go | 5 + pkg/agent/templates/templates_linux.go | 1 + pkg/cli/cmds/agent.go | 7 + pkg/cli/cmds/stage.go | 2 +- pkg/daemons/config/types.go | 1 + pkg/deploy/zz_generated_bindata.go | 23 + 12 files changed, 1062 insertions(+), 288 deletions(-) create mode 100644 manifests/runtimes.yaml delete mode 100644 pkg/agent/containerd/nvidia.go delete mode 100644 pkg/agent/containerd/nvidia_test.go create mode 100644 pkg/agent/containerd/runtimes.go create mode 100644 pkg/agent/containerd/runtimes_test.go diff --git a/manifests/runtimes.yaml b/manifests/runtimes.yaml new file mode 100644 index 00000000..fdbc9512 --- /dev/null +++ b/manifests/runtimes.yaml @@ -0,0 +1,59 @@ +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia +handler: nvidia +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia-experimental +handler: nvidia-experimental +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: crun +handler: crun +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: lunatic +handler: lunatic +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: slight +handler: slight +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: spin +handler: spin +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wws +handler: wws +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wasmedge +handler: wasmedge +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wasmer +handler: wasmer +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wasmtime +handler: wasmtime \ No newline at end of file diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go index 81666437..96f6f5e2 100644 --- a/pkg/agent/containerd/config_linux.go +++ b/pkg/agent/containerd/config_linux.go @@ -57,6 +57,13 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { cfg.AgentConfig.Systemd = !isRunningInUserNS && controllers["cpuset"] && os.Getenv("INVOCATION_ID") != "" } + extraRuntimes := findContainerRuntimes(os.DirFS(string(os.PathSeparator))) + + // Verifies if the DefaultRuntime can be found + if _, ok := extraRuntimes[cfg.DefaultRuntime]; !ok && cfg.DefaultRuntime != "" { + return errors.Errorf("default runtime %s was not found", cfg.DefaultRuntime) + } + var containerdTemplate string containerdConfig := templates.ContainerdConfig{ NodeConfig: cfg, @@ -65,7 +72,7 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { IsRunningInUserNS: isRunningInUserNS, EnableUnprivileged: kernel.CheckKernelVersion(4, 11, 0), PrivateRegistryConfig: privRegistries.Registry, - ExtraRuntimes: findNvidiaContainerRuntimes(os.DirFS(string(os.PathSeparator))), + ExtraRuntimes: extraRuntimes, Program: version.Program, } diff --git a/pkg/agent/containerd/nvidia.go b/pkg/agent/containerd/nvidia.go deleted file mode 100644 index 89a2f06b..00000000 --- a/pkg/agent/containerd/nvidia.go +++ /dev/null @@ -1,67 +0,0 @@ -//go:build linux -// +build linux - -package containerd - -import ( - "errors" - "io/fs" - "path/filepath" - - "github.com/sirupsen/logrus" - "github.com/xiaods/k8e/pkg/agent/templates" -) - -// findNvidiaContainerRuntimes returns a list of nvidia container runtimes that -// are available on the system. It checks install locations used by the nvidia -// gpu operator and by system package managers. The gpu operator installation -// takes precedence over the system package manager installation. -// The given fs.FS should represent the filesystem root directory to search in. -func findNvidiaContainerRuntimes(root fs.FS) map[string]templates.ContainerdRuntimeConfig { - // Check these locations in order. The GPU operator's installation should - // take precedence over the package manager's installation. - locationsToCheck := []string{ - "usr/local/nvidia/toolkit", // Path when installing via GPU Operator - "usr/bin", // Path when installing via package manager - } - - // Fill in the binary location with just the name of the binary, - // and check against each of the possible locations. If a match is found, - // set the location to the full path. - potentialRuntimes := map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "nvidia-container-runtime-experimental", - }, - } - foundRuntimes := map[string]templates.ContainerdRuntimeConfig{} -RUNTIME: - for runtimeName, runtimeConfig := range potentialRuntimes { - for _, location := range locationsToCheck { - binaryPath := filepath.Join(location, runtimeConfig.BinaryName) - logrus.Debugf("Searching for %s container runtime at /%s", runtimeName, binaryPath) - if info, err := fs.Stat(root, binaryPath); err == nil { - if info.IsDir() { - logrus.Debugf("Found %s container runtime at /%s, but it is a directory. Skipping.", runtimeName, binaryPath) - continue - } - runtimeConfig.BinaryName = filepath.Join("/", binaryPath) - logrus.Infof("Found %s container runtime at %s", runtimeName, runtimeConfig.BinaryName) - foundRuntimes[runtimeName] = runtimeConfig - // Skip to the next runtime to enforce precedence. - continue RUNTIME - } else { - if errors.Is(err, fs.ErrNotExist) { - logrus.Debugf("%s container runtime not found at /%s", runtimeName, binaryPath) - } else { - logrus.Errorf("Error searching for %s container runtime at /%s: %v", runtimeName, binaryPath, err) - } - } - } - } - return foundRuntimes -} diff --git a/pkg/agent/containerd/nvidia_test.go b/pkg/agent/containerd/nvidia_test.go deleted file mode 100644 index fc0a9900..00000000 --- a/pkg/agent/containerd/nvidia_test.go +++ /dev/null @@ -1,219 +0,0 @@ -//go:build linux -// +build linux - -package containerd - -import ( - "io/fs" - "reflect" - "testing" - "testing/fstest" - - "github.com/xiaods/k8e/pkg/agent/templates" -) - -func Test_UnitFindNvidiaContainerRuntimes(t *testing.T) { - executable := &fstest.MapFile{Mode: 0755} - type args struct { - root fs.FS - } - tests := []struct { - name string - args args - want map[string]templates.ContainerdRuntimeConfig - }{ - { - name: "No runtimes", - args: args{ - root: fstest.MapFS{}, - }, - want: map[string]templates.ContainerdRuntimeConfig{}, - }, - { - name: "Nvidia runtime in /usr/bin", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - }, - }, - { - name: "Experimental runtime in /usr/local/nvidia/toolkit", - args: args{ - root: fstest.MapFS{ - "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", - }, - }, - }, - { - name: "Two runtimes in separate directories", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", - }, - }, - }, - { - name: "Experimental runtime in /usr/bin", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime-experimental": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Same runtime in two directories", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime-experimental": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Both runtimes in /usr/bin", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime-experimental": executable, - "usr/bin/nvidia-container-runtime": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Both runtimes in both directories", - args: args{ - root: fstest.MapFS{ - "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, - "usr/bin/nvidia-container-runtime": executable, - "usr/bin/nvidia-container-runtime-experimental": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Both runtimes in /usr/local/nvidia/toolkit", - args: args{ - root: fstest.MapFS{ - "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Both runtimes in /usr/bin and one duplicate in /usr/local/nvidia/toolkit", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - "usr/bin/nvidia-container-runtime-experimental": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Runtime is a directory", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": &fstest.MapFile{ - Mode: fs.ModeDir, - }, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{}, - }, - { - name: "Runtime in both directories, but one is a directory", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime": &fstest.MapFile{ - Mode: fs.ModeDir, - }, - }, - }, - want: map[string]templates.ContainerdRuntimeConfig{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := findNvidiaContainerRuntimes(tt.args.root); !reflect.DeepEqual(got, tt.want) { - t.Errorf("findNvidiaContainerRuntimes() = %+v\nWant = %+v", got, tt.want) - } - }) - } -} diff --git a/pkg/agent/containerd/runtimes.go b/pkg/agent/containerd/runtimes.go new file mode 100644 index 00000000..df056c82 --- /dev/null +++ b/pkg/agent/containerd/runtimes.go @@ -0,0 +1,161 @@ +//go:build linux +// +build linux + +package containerd + +import ( + "errors" + "io/fs" + "path/filepath" + + "github.com/sirupsen/logrus" + "github.com/xiaods/k8e/pkg/agent/templates" +) + +// A map with string as value and `templates.ContainerdRuntimeConfig` as values. +// The key holds the name of the runtime +type runtimeConfigs map[string]templates.ContainerdRuntimeConfig + +// searchForRuntimes searches for runtimes and add into foundRuntimes +// It checks install locations provided via potentitalRuntimes variable. +// The binaries are searched at the locations specivied by locationsToCheck. +// The given fs.FS should represent the filesystem root directory to search in. +func searchForRuntimes(root fs.FS, potentialRuntimes runtimeConfigs, locationsToCheck []string, foundRuntimes runtimeConfigs) { + // Check these locations in order. The GPU operator's installation should + // take precedence over the package manager's installation. + + // Fill in the binary location with just the name of the binary, + // and check against each of the possible locations. If a match is found, + // set the location to the full path. + for runtimeName, runtimeConfig := range potentialRuntimes { + for _, location := range locationsToCheck { + binaryPath := filepath.Join(location, runtimeConfig.BinaryName) + logrus.Debugf("Searching for %s container runtime at /%s", runtimeName, binaryPath) + if info, err := fs.Stat(root, binaryPath); err == nil { + if info.IsDir() { + logrus.Debugf("Found %s container runtime at /%s, but it is a directory. Skipping.", runtimeName, binaryPath) + continue + } + runtimeConfig.BinaryName = filepath.Join("/", binaryPath) + logrus.Infof("Found %s container runtime at %s", runtimeName, runtimeConfig.BinaryName) + foundRuntimes[runtimeName] = runtimeConfig + break + } else { + if errors.Is(err, fs.ErrNotExist) { + logrus.Debugf("%s container runtime not found at /%s", runtimeName, binaryPath) + } else { + logrus.Errorf("Error searching for %s container runtime at /%s: %v", runtimeName, binaryPath, err) + } + } + } + } +} + +// findContainerRuntimes is a function that searches for all the runtimes and +// return a list with all the runtimes that have been found +func findContainerRuntimes(root fs.FS) runtimeConfigs { + foundRuntimes := runtimeConfigs{} + findCRunContainerRuntime(root, foundRuntimes) + findNvidiaContainerRuntimes(root, foundRuntimes) + findWasiRuntimes(root, foundRuntimes) + return foundRuntimes +} + +// findCRunContainerRuntime finds if crun is available in the system and adds to foundRuntimes +func findCRunContainerRuntime(root fs.FS, foundRuntimes runtimeConfigs) { + // Check these locations in order. + locationsToCheck := []string{ + "usr/sbin", // Path when installing via package manager + "usr/bin", // Path when installing via package manager + } + + // Fill in the binary location with just the name of the binary, + // and check against each of the possible locations. If a match is found, + // set the location to the full path. + potentialRuntimes := runtimeConfigs{ + "crun": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "crun", + }, + } + + searchForRuntimes(root, potentialRuntimes, locationsToCheck, foundRuntimes) +} + +// findNvidiaContainerRuntimes finds the nvidia runtimes that are are available on the system +// and adds to foundRuntimes. It checks install locations used by the nvidia +// gpu operator and by system package managers. The gpu operator installation +// takes precedence over the system package manager installation. +// The given fs.FS should represent the filesystem root directory to search in. +func findNvidiaContainerRuntimes(root fs.FS, foundRuntimes runtimeConfigs) { + // Check these locations in order. The GPU operator's installation should + // take precedence over the package manager's installation. + locationsToCheck := []string{ + "usr/local/nvidia/toolkit", // Path when installing via GPU Operator + "usr/bin", // Path when installing via package manager + } + + // Fill in the binary location with just the name of the binary, + // and check against each of the possible locations. If a match is found, + // set the location to the full path. + potentialRuntimes := runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime-experimental", + }, + } + searchForRuntimes(root, potentialRuntimes, locationsToCheck, foundRuntimes) +} + +// findWasiRuntimes finds the WebAssembly (WASI) container runtimes that +// are available on the system and adds to foundRuntimes. It checks install locations used by the kwasm +// operator and by system package managers. The kwasm operator installation +// takes precedence over the system package manager installation. +// The given fs.FS should represent the filesystem root directory to search in. +func findWasiRuntimes(root fs.FS, foundRuntimes runtimeConfigs) { + // Check these locations in order. + locationsToCheck := []string{ + "opt/kwasm/bin", // Path when installing via kwasm Operator + "usr/bin", // Path when installing via package manager + "usr/sbin", // Path when installing via package manager + } + + // Fill in the binary location with just the name of the binary, + // and check against each of the possible locations. If a match is found, + // set the location to the full path. + potentialRuntimes := runtimeConfigs{ + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "containerd-shim-lunatic-v1", + }, + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "containerd-shim-slight-v1", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "containerd-shim-spin-v1", + }, + "wws": { + RuntimeType: "io.containerd.wws.v2", + BinaryName: "containerd-shim-wws-v1", + }, + "wasmedge": { + RuntimeType: "io.containerd.wasmedge.v2", + BinaryName: "containerd-shim-wasmedge-v1", + }, + "wasmer": { + RuntimeType: "io.containerd.wasmer.v2", + BinaryName: "containerd-shim-wasmer-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "containerd-shim-wasmtime-v1", + }, + } + searchForRuntimes(root, potentialRuntimes, locationsToCheck, foundRuntimes) +} diff --git a/pkg/agent/containerd/runtimes_test.go b/pkg/agent/containerd/runtimes_test.go new file mode 100644 index 00000000..b209b495 --- /dev/null +++ b/pkg/agent/containerd/runtimes_test.go @@ -0,0 +1,796 @@ +//go:build linux +// +build linux + +package containerd + +import ( + "io/fs" + "reflect" + "testing" + "testing/fstest" +) + +func Test_UnitFindContainerRuntimes(t *testing.T) { + executable := &fstest.MapFile{Mode: 0755} + + type args struct { + root fs.FS + } + + tests := []struct { + name string + args args + want runtimeConfigs + }{ + { + name: "No runtimes", + args: args{ + root: fstest.MapFS{}, + }, + want: runtimeConfigs{}, + }, + { + name: "Found crun, nvidia and wasm", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/crun": executable, + "opt/kwasm/bin/containerd-shim-lunatic-v1": executable, + }, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "crun": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/crun", + }, + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-lunatic-v1", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + foundRuntimes := findContainerRuntimes(tt.args.root) + if !reflect.DeepEqual(foundRuntimes, tt.want) { + t.Errorf("findContainerRuntimes = %+v\nWant = %+v", foundRuntimes, tt.want) + } + }) + } +} + +func Test_UnitSearchContainerRuntimes(t *testing.T) { + executable := &fstest.MapFile{Mode: 0755} + locationsToCheck := []string{ + "usr/local/nvidia/toolkit", // Path for nvidia shim when installing via GPU Operator + "opt/kwasm/bin", // Path for wasm shim when installing via the kwasm operator + "usr/bin", // Path when installing via package manager + "usr/sbin", // Path when installing via package manager + } + + potentialRuntimes := runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "containerd-shim-spin-v1", + }, + } + + type args struct { + root fs.FS + potentialRuntimes runtimeConfigs + locationsToCheck []string + } + tests := []struct { + name string + args args + want runtimeConfigs + }{ + { + name: "No runtimes", + args: args{ + root: fstest.MapFS{}, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{}, + }, + { + name: "Nvidia runtime in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + { + name: "Two runtimes in separate directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "opt/kwasm/bin/containerd-shim-spin-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-spin-v1", + }, + }, + }, + { + name: "Same runtime in two directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-spin-v1": executable, + "opt/kwasm/bin/containerd-shim-spin-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-spin-v1", + }, + }, + }, + { + name: "Both runtimes in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-spin-v1": executable, + "usr/bin/nvidia-container-runtime": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/usr/bin/containerd-shim-spin-v1", + }, + }, + }, + { + name: "Both runtimes in both directories", + args: args{ + root: fstest.MapFS{ + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/containerd-shim-spin-v1": executable, + "opt/kwasm/bin/containerd-shim-spin-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-spin-v1", + }, + }, + }, + { + name: "Both runtimes in /usr/bin and one duplicate in /usr/local/nvidia/toolkit", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/containerd-shim-spin-v1": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/usr/bin/containerd-shim-spin-v1", + }, + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + }, + }, + { + name: "Runtime is a directory", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{}, + }, + { + name: "Runtime in both directories, but one is a directory", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + foundRuntimes := runtimeConfigs{} + searchForRuntimes(tt.args.root, tt.args.potentialRuntimes, tt.args.locationsToCheck, foundRuntimes) + if !reflect.DeepEqual(foundRuntimes, tt.want) { + t.Errorf("findContainerRuntimes() = %+v\nWant = %+v", foundRuntimes, tt.want) + } + }) + } +} + +func Test_UnitSearchWasiRuntimes(t *testing.T) { + executable := &fstest.MapFile{Mode: 0755} + + locationsToCheck := []string{ + "usr/local/nvidia/toolkit", // Path for nvidia shim when installing via GPU Operator + "opt/kwasm/bin", // Path for wasm shim when installing via the kwasm operator + "usr/bin", // Path when installing via package manager + "usr/sbin", // Path when installing via package manager + } + + potentialRuntimes := runtimeConfigs{ + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "containerd-shim-wasmtime-v1", + }, + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "containerd-shim-lunatic-v1", + }, + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "containerd-shim-slight-v1", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "containerd-shim-spin-v1", + }, + "wws": { + RuntimeType: "io.containerd.wws.v2", + BinaryName: "containerd-shim-wws-v1", + }, + "wasmedge": { + RuntimeType: "io.containerd.wasmedge.v2", + BinaryName: "containerd-shim-wasmedge-v1", + }, + "wasmer": { + RuntimeType: "io.containerd.wasmer.v2", + BinaryName: "containerd-shim-wasmer-v1", + }, + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime", + }, + } + + type args struct { + root fs.FS + potentialRuntimes runtimeConfigs + locationsToCheck []string + } + + tests := []struct { + name string + args args + want runtimeConfigs + }{ + { + name: "No runtimes", + args: args{ + root: fstest.MapFS{}, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{}, + }, + { + name: "wasmtime runtime in /usr/sbin", + args: args{ + root: fstest.MapFS{ + "usr/sbin/containerd-shim-wasmtime-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/usr/sbin/containerd-shim-wasmtime-v1", + }, + }, + }, + { + name: "lunatic runtime in /opt/kwasm/bin/", + args: args{ + root: fstest.MapFS{ + "opt/kwasm/bin/containerd-shim-lunatic-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-lunatic-v1", + }, + }, + }, + { + name: "Two runtimes in separate directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-wasmer-v1": executable, + "opt/kwasm/bin/containerd-shim-slight-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", + }, + "wasmer": { + RuntimeType: "io.containerd.wasmer.v2", + BinaryName: "/usr/bin/containerd-shim-wasmer-v1", + }, + }, + }, + { + name: "Same runtime in two directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-wasmedge-v1": executable, + "opt/kwasm/bin/containerd-shim-wasmedge-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "wasmedge": { + RuntimeType: "io.containerd.wasmedge.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-wasmedge-v1", + }, + }, + }, + { + name: "All runtimes in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-lunatic-v1": executable, + "usr/bin/containerd-shim-slight-v1": executable, + "usr/bin/containerd-shim-spin-v1": executable, + "usr/bin/containerd-shim-wws-v1": executable, + "usr/bin/containerd-shim-wasmedge-v1": executable, + "usr/bin/containerd-shim-wasmer-v1": executable, + "usr/bin/containerd-shim-wasmtime-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "/usr/bin/containerd-shim-lunatic-v1", + }, + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "/usr/bin/containerd-shim-slight-v1", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/usr/bin/containerd-shim-spin-v1", + }, + "wws": { + RuntimeType: "io.containerd.wws.v2", + BinaryName: "/usr/bin/containerd-shim-wws-v1", + }, + "wasmedge": { + RuntimeType: "io.containerd.wasmedge.v2", + BinaryName: "/usr/bin/containerd-shim-wasmedge-v1", + }, + "wasmer": { + RuntimeType: "io.containerd.wasmer.v2", + BinaryName: "/usr/bin/containerd-shim-wasmer-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/usr/bin/containerd-shim-wasmtime-v1", + }, + }, + }, + { + name: "Both runtimes in both directories", + args: args{ + root: fstest.MapFS{ + "opt/kwasm/bin/containerd-shim-slight-v1": executable, + "opt/kwasm/bin/containerd-shim-wasmtime-v1": executable, + "usr/bin/containerd-shim-slight-v1": executable, + "usr/bin/containerd-shim-wasmtime-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", + }, + }, + }, + { + name: "Preserve already found runtimes", + args: args{ + root: fstest.MapFS{ + "opt/kwasm/bin/containerd-shim-wasmtime-v1": executable, + "usr/bin/nvidia-container-runtime": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + foundRuntimes := runtimeConfigs{} + searchForRuntimes(tt.args.root, tt.args.potentialRuntimes, tt.args.locationsToCheck, foundRuntimes) + if !reflect.DeepEqual(foundRuntimes, tt.want) { + t.Errorf("searchForRuntimes = %+v\nWant = %+v", foundRuntimes, tt.want) + } + }) + } +} + +func Test_UnitSearchNvidiaContainerRuntimes(t *testing.T) { + executable := &fstest.MapFile{Mode: 0755} + + locationsToCheck := []string{ + "usr/local/nvidia/toolkit", // Path for nvidia shim when installing via GPU Operator + "opt/kwasm/bin", // Path for wasm shim when installing via the kwasm operator + "usr/bin", // Path when installing via package manager + "usr/sbin", // Path when installing via package manager + } + + potentialRuntimes := runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime-experimental", + }, + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "containerd-shim-slight-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "containerd-shim-wasmtime-v1", + }, + } + + type args struct { + root fs.FS + potentialRuntimes runtimeConfigs + locationsToCheck []string + } + + tests := []struct { + name string + args args + want runtimeConfigs + }{ + { + name: "No runtimes", + args: args{ + root: fstest.MapFS{}, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{}, + }, + { + name: "Nvidia runtime in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + { + name: "Experimental runtime in /usr/local/nvidia/toolkit", + args: args{ + root: fstest.MapFS{ + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + }, + }, + { + name: "Two runtimes in separate directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + }, + }, + { + name: "Experimental runtime in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Same runtime in two directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime-experimental": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Both runtimes in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime-experimental": executable, + "usr/bin/nvidia-container-runtime": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Both runtimes in both directories", + args: args{ + root: fstest.MapFS{ + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Both runtimes in /usr/local/nvidia/toolkit", + args: args{ + root: fstest.MapFS{ + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Both runtimes in /usr/bin and one duplicate in /usr/local/nvidia/toolkit", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/nvidia-container-runtime-experimental": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Runtime is a directory", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{}, + }, + { + name: "Runtime in both directories, but one is a directory", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + { + name: "Preserve already found runtimes", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "opt/kwasm/bin/containerd-shim-wasmtime-v1": executable, + "opt/kwasm/bin/containerd-shim-slight-v1": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", + }, + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + foundRuntimes := runtimeConfigs{} + searchForRuntimes(tt.args.root, tt.args.potentialRuntimes, tt.args.locationsToCheck, foundRuntimes) + if !reflect.DeepEqual(foundRuntimes, tt.want) { + t.Errorf("searchForRuntimes() = %+v\nWant = %+v", foundRuntimes, tt.want) + } + }) + } +} diff --git a/pkg/agent/run_linux.go b/pkg/agent/run_linux.go index 3dc8016e..fa203dc2 100644 --- a/pkg/agent/run_linux.go +++ b/pkg/agent/run_linux.go @@ -36,6 +36,11 @@ func setupCriCtlConfig(cfg cmds.Agent, nodeConfig *config.Node) error { } } + // Send to node struct the value from cli/config default runtime + if cfg.DefaultRuntime != "" { + nodeConfig.DefaultRuntime = cfg.DefaultRuntime + } + crp := "runtime-endpoint: " + cre + "\n" ise := nodeConfig.ImageServiceEndpoint if ise != "" && ise != cre { diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index 43cf65f2..2a8e51df 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -32,6 +32,7 @@ version = 2 [plugins."io.containerd.grpc.v1.cri".containerd] snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}" disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}} + {{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}} {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} {{ if .NodeConfig.AgentConfig.ImageServiceSocket }} [plugins."io.containerd.snapshotter.v1.stargz"] diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index eb8f5b4a..c9bc7a1c 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -27,6 +27,7 @@ type Agent struct { Snapshotter string Docker bool ContainerRuntimeEndpoint string + DefaultRuntime string ImageServiceEndpoint string Debug bool Rootless bool @@ -109,6 +110,11 @@ var ( Usage: "(agent/runtime) Disable embedded containerd and use the CRI socket at the given path; when used with --docker this sets the docker socket path", Destination: &AgentConfig.ContainerRuntimeEndpoint, } + DefaultRuntimeFlag = &cli.StringFlag{ + Name: "default-runtime", + Usage: "(agent/runtime) Set the default runtime in containerd", + Destination: &AgentConfig.DefaultRuntime, + } ImageServiceEndpointFlag = &cli.StringFlag{ Name: "image-service-endpoint", Usage: "(agent/runtime) Disable embedded containerd image service and use remote image service socket at the given path. If not specified, defaults to --container-runtime-endpoint.", @@ -236,6 +242,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { LBServerPortFlag, ProtectKernelDefaultsFlag, CRIEndpointFlag, + DefaultRuntimeFlag, ImageServiceEndpointFlag, PauseImageFlag, SnapshotterFlag, diff --git a/pkg/cli/cmds/stage.go b/pkg/cli/cmds/stage.go index a5172f65..daef2e75 100644 --- a/pkg/cli/cmds/stage.go +++ b/pkg/cli/cmds/stage.go @@ -7,5 +7,5 @@ const ( // coredns run controllers that are turned off when their manifests are disabled. // The k8e CloudController also has a bundled manifest and can be disabled via the // --disable-cloud-controller flag or --disable=ccm, but the latter method is not documented. - DisableItems = "coredns, local-storage, metrics-server" + DisableItems = "coredns, local-storage, metrics-server, runtimes" ) diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 457ce710..63558c37 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -42,6 +42,7 @@ type Node struct { Token string Certificate *tls.Certificate ServerHTTPSPort int + DefaultRuntime string } type Containerd struct { diff --git a/pkg/deploy/zz_generated_bindata.go b/pkg/deploy/zz_generated_bindata.go index 3a4ae6f3..77881f9e 100644 --- a/pkg/deploy/zz_generated_bindata.go +++ b/pkg/deploy/zz_generated_bindata.go @@ -12,6 +12,7 @@ // manifests/metrics-server/metrics-server-service.yaml // manifests/metrics-server/resource-reader.yaml // manifests/rolebindings.yaml +// manifests/runtimes.yaml //go:build !no_stage // +build !no_stage @@ -331,6 +332,26 @@ func rolebindingsYaml() (*asset, error) { return a, nil } +var _runtimesYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\xd0\x31\x8e\x84\x30\x0c\x05\xd0\x3e\xa7\xc8\x05\xc2\x6a\xbb\x55\xda\xbd\xc1\x14\xd3\x5b\xc4\x02\x8b\xc4\xa0\x38\xc0\x1c\x7f\x04\x1a\x26\x40\xed\xf2\x7f\x4b\xef\x4b\x86\x89\x9e\x98\x85\x46\xf6\x96\xc7\x80\xcd\xf0\x27\x0d\x8d\x3f\xcb\xaf\x19\x88\x83\xb7\x8f\x99\x0b\x25\xfc\x8f\x20\x62\x12\x16\x08\x50\xc0\x1b\x6b\x19\x12\x7a\xcb\x0b\x05\x02\xd3\x03\x87\x88\xf9\x9b\x9d\x73\x46\x87\x76\xf8\x9a\x30\x53\x42\x2e\x10\xef\x3b\xd7\xa3\xc6\x68\x9b\x67\xae\x2b\x7b\xd2\x60\xe3\xcc\x50\xa8\xad\xf2\x51\x68\xe0\x12\xa9\xeb\x4b\xb5\x3f\x59\x85\x9e\xe8\xf4\x8e\x3d\x69\xb0\xeb\x2a\x55\xdd\x82\x0a\x0a\x92\x30\x74\x78\x92\x8f\x46\x8d\xcf\x37\x3c\xab\xd1\xdb\xfd\x8a\x6f\xcd\x3b\x00\x00\xff\xff\xc7\xad\x48\x21\x9f\x03\x00\x00") + +func runtimesYamlBytes() ([]byte, error) { + return bindataRead( + _runtimesYaml, + "runtimes.yaml", + ) +} + +func runtimesYaml() (*asset, error) { + bytes, err := runtimesYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "runtimes.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + // Asset loads and returns the asset for the given name. // It returns an error if the asset could not be found or // could not be loaded. @@ -395,6 +416,7 @@ var _bindata = map[string]func() (*asset, error){ "metrics-server/metrics-server-service.yaml": metricsServerMetricsServerServiceYaml, "metrics-server/resource-reader.yaml": metricsServerResourceReaderYaml, "rolebindings.yaml": rolebindingsYaml, + "runtimes.yaml": runtimesYaml, } // AssetDir returns the file names below a certain @@ -454,6 +476,7 @@ var _bintree = &bintree{nil, map[string]*bintree{ "resource-reader.yaml": &bintree{metricsServerResourceReaderYaml, map[string]*bintree{}}, }}, "rolebindings.yaml": &bintree{rolebindingsYaml, map[string]*bintree{}}, + "runtimes.yaml": &bintree{runtimesYaml, map[string]*bintree{}}, }} // RestoreAsset restores an asset under the given directory