From 0dca795ff0b64ab7b69717cb9cf209420ae5ff78 Mon Sep 17 00:00:00 2001 From: Ricardo R Date: Wed, 13 Nov 2024 23:43:20 +1300 Subject: [PATCH 1/3] Add server provisioner --- .../region.unikorn-cloud.org_servers.yaml | 3 + pkg/apis/unikorn/v1alpha1/types.go | 1 + pkg/providers/interfaces.go | 4 + pkg/providers/openstack/compute.go | 52 +++ pkg/providers/openstack/errors.go | 4 + pkg/providers/openstack/network.go | 61 +++ pkg/providers/openstack/provider.go | 411 ++++++++++++++++++ .../managers/server/provisioner.go | 77 +++- 8 files changed, 611 insertions(+), 2 deletions(-) diff --git a/charts/region/crds/region.unikorn-cloud.org_servers.yaml b/charts/region/crds/region.unikorn-cloud.org_servers.yaml index 8666298..18d0289 100644 --- a/charts/region/crds/region.unikorn-cloud.org_servers.yaml +++ b/charts/region/crds/region.unikorn-cloud.org_servers.yaml @@ -23,6 +23,9 @@ spec: - jsonPath: .metadata.creationTimestamp name: age type: date + - jsonPath: .status.privateIP + name: privateIP + type: string - jsonPath: .status.publicIP name: publicIP type: string diff --git a/pkg/apis/unikorn/v1alpha1/types.go b/pkg/apis/unikorn/v1alpha1/types.go index 68dcdfc..c17be74 100644 --- a/pkg/apis/unikorn/v1alpha1/types.go +++ b/pkg/apis/unikorn/v1alpha1/types.go @@ -696,6 +696,7 @@ type ServerList struct { // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="status",type="string",JSONPath=".status.conditions[?(@.type==\"Available\")].reason" // +kubebuilder:printcolumn:name="age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="privateIP",type="string",JSONPath=".status.privateIP" // +kubebuilder:printcolumn:name="publicIP",type="string",JSONPath=".status.publicIP" type Server struct { metav1.TypeMeta `json:",inline"` diff --git a/pkg/providers/interfaces.go b/pkg/providers/interfaces.go index 1e831f2..3700b27 100644 --- a/pkg/providers/interfaces.go +++ b/pkg/providers/interfaces.go @@ -51,4 +51,8 @@ type Provider interface { CreateSecurityGroupRule(ctx context.Context, identity *unikornv1.Identity, securityGroup *unikornv1.SecurityGroup, rule *unikornv1.SecurityGroupRule) error // DeleteSecurityGroupRule deletes a security group rule. DeleteSecurityGroupRule(ctx context.Context, identity *unikornv1.Identity, securityGroup *unikornv1.SecurityGroup, rule *unikornv1.SecurityGroupRule) error + // CreateServer creates a new server. + CreateServer(ctx context.Context, identity *unikornv1.Identity, server *unikornv1.Server) error + // DeleteServer deletes a server. + DeleteServer(ctx context.Context, identity *unikornv1.Identity, server *unikornv1.Server) error } diff --git a/pkg/providers/openstack/compute.go b/pkg/providers/openstack/compute.go index aef6e33..a88d832 100644 --- a/pkg/providers/openstack/compute.go +++ b/pkg/providers/openstack/compute.go @@ -30,6 +30,7 @@ import ( "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/keypairs" "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/quotasets" "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servergroups" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" @@ -267,3 +268,54 @@ func (c *ComputeClient) UpdateQuotas(ctx context.Context, projectID string) erro return quotasets.Update(ctx, c.client, projectID, opts).Err } + +func (c *ComputeClient) CreateServer(ctx context.Context, name, imageID, flavorID, keyName string, networkIDs []string, serverGroupID *string, metadata map[string]string) (*servers.Server, error) { + tracer := otel.GetTracerProvider().Tracer(constants.Application) + + _, span := tracer.Start(ctx, "POST /compute/v2/servers/") + defer span.End() + + schedulerHintOpts := servers.SchedulerHintOpts{} + + if serverGroupID != nil { + schedulerHintOpts.Group = *serverGroupID + } + + networks := make([]servers.Network, len(networkIDs)) + for i, id := range networkIDs { + networks[i] = servers.Network{UUID: id} + } + + serverCreateOpts := servers.CreateOpts{ + Name: name, + ImageRef: imageID, + FlavorRef: flavorID, + Networks: networks, + Metadata: metadata, + } + + createOpts := keypairs.CreateOptsExt{ + CreateOptsBuilder: serverCreateOpts, + KeyName: keyName, + } + + return servers.Create(ctx, c.client, createOpts, schedulerHintOpts).Extract() +} + +func (c *ComputeClient) DeleteServer(ctx context.Context, id string) error { + tracer := otel.GetTracerProvider().Tracer(constants.Application) + + _, span := tracer.Start(ctx, fmt.Sprintf("DELETE /compute/v2/servers/%s", id)) + defer span.End() + + return servers.Delete(ctx, c.client, id).ExtractErr() +} + +func (c *ComputeClient) GetServer(ctx context.Context, id string) (*servers.Server, error) { + tracer := otel.GetTracerProvider().Tracer(constants.Application) + + _, span := tracer.Start(ctx, fmt.Sprintf("GET /compute/v2/servers/%s", id)) + defer span.End() + + return servers.Get(ctx, c.client, id).Extract() +} diff --git a/pkg/providers/openstack/errors.go b/pkg/providers/openstack/errors.go index 0d294f5..4cee870 100644 --- a/pkg/providers/openstack/errors.go +++ b/pkg/providers/openstack/errors.go @@ -25,4 +25,8 @@ var ( // ErrResourceNotFound is returned when a named resource cannot // be looked up (we have to do it ourselves) and it cannot be found. ErrResourceNotFound = errors.New("requested resource not found") + + // ErrResourceDependency is returned when a resource is in unexpected + // state or condition. + ErrResouceDependency = errors.New("resource dependency error") ) diff --git a/pkg/providers/openstack/network.go b/pkg/providers/openstack/network.go index c315313..6c8ee81 100644 --- a/pkg/providers/openstack/network.go +++ b/pkg/providers/openstack/network.go @@ -27,11 +27,13 @@ import ( gophercloud "github.com/gophercloud/gophercloud/v2" "github.com/gophercloud/gophercloud/v2/openstack" "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/external" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/layer3/floatingips" "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/layer3/routers" "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/provider" "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/security/groups" "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/security/rules" "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/networks" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/ports" "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/subnets" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" @@ -363,3 +365,62 @@ func (c *NetworkClient) DeleteSecurityGroupRule(ctx context.Context, securityGro return rules.Delete(ctx, c.client, ruleID).Err } + +// CreateFloatingIP creates a floating IP. +func (c *NetworkClient) CreateFloatingIP(ctx context.Context, portID string) (*floatingips.FloatingIP, error) { + externalNetworks, err := c.ExternalNetworks(ctx) + if err != nil { + return nil, err + } + + tracer := otel.GetTracerProvider().Tracer(constants.Application) + + _, span := tracer.Start(ctx, "POST /network/v2.0/floatingips", trace.WithSpanKind(trace.SpanKindClient)) + defer span.End() + + opts := &floatingips.CreateOpts{ + FloatingNetworkID: externalNetworks[0].ID, + PortID: portID, + } + + floatingIP, err := floatingips.Create(ctx, c.client, opts).Extract() + if err != nil { + return nil, err + } + + return floatingIP, nil +} + +// DeleteFloatingIP deletes a floating IP. +func (c *NetworkClient) DeleteFloatingIP(ctx context.Context, id string) error { + tracer := otel.GetTracerProvider().Tracer(constants.Application) + + _, span := tracer.Start(ctx, fmt.Sprintf("DELETE /network/v2.0/floatingips/%s", id), trace.WithSpanKind(trace.SpanKindClient)) + defer span.End() + + return floatingips.Delete(ctx, c.client, id).Err +} + +// ListServerPorts returns a list of ports for a server. +func (c *NetworkClient) ListServerPorts(ctx context.Context, serverID string) ([]ports.Port, error) { + tracer := otel.GetTracerProvider().Tracer(constants.Application) + + _, span := tracer.Start(ctx, "GET /network/v2.0/ports", trace.WithSpanKind(trace.SpanKindClient)) + defer span.End() + + listOpts := ports.ListOpts{ + DeviceID: serverID, + } + + allPages, err := ports.List(c.client, listOpts).AllPages(ctx) + if err != nil { + return nil, err + } + + allPorts, err := ports.ExtractPorts(allPages) + if err != nil { + return nil, err + } + + return allPorts, nil +} diff --git a/pkg/providers/openstack/provider.go b/pkg/providers/openstack/provider.go index 76153c3..9895c2c 100644 --- a/pkg/providers/openstack/provider.go +++ b/pkg/providers/openstack/provider.go @@ -20,16 +20,20 @@ import ( "context" "errors" "fmt" + "net/http" "reflect" "slices" "strings" "sync" + "github.com/gophercloud/gophercloud/v2" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers" "github.com/gophercloud/gophercloud/v2/openstack/identity/v3/roles" "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/security/rules" "github.com/gophercloud/utils/openstack/clientconfig" coreconstants "github.com/unikorn-cloud/core/pkg/constants" + "github.com/unikorn-cloud/core/pkg/provisioners" unikornv1 "github.com/unikorn-cloud/region/pkg/apis/unikorn/v1alpha1" "github.com/unikorn-cloud/region/pkg/constants" "github.com/unikorn-cloud/region/pkg/providers" @@ -39,9 +43,11 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/uuid" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/yaml" ) @@ -1495,3 +1501,408 @@ func (p *Provider) DeleteSecurityGroupRule(ctx context.Context, identity *unikor return nil } + +func (p *Provider) GetOpenstackServer(ctx context.Context, server *unikornv1.Server) (*unikornv1.OpenstackServer, error) { + var result unikornv1.OpenstackServer + + if err := p.client.Get(ctx, client.ObjectKey{Namespace: server.Namespace, Name: server.Name}, &result); err != nil { + return nil, err + } + + return &result, nil +} + +func (p *Provider) GetOrCreateOpenstackServer(ctx context.Context, identity *unikornv1.Identity, server *unikornv1.Server) (*unikornv1.OpenstackServer, bool, error) { + create := false + + openstackServer, err := p.GetOpenstackServer(ctx, server) + if err != nil { + if !kerrors.IsNotFound(err) { + return nil, false, err + } + + openstackServer = &unikornv1.OpenstackServer{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: server.Namespace, + Name: server.Name, + Labels: map[string]string{ + constants.IdentityLabel: identity.Name, + constants.ServerLabel: server.Name, + }, + Annotations: server.Annotations, + }, + } + + for k, v := range server.Labels { + openstackServer.Labels[k] = v + } + + create = true + } + + return openstackServer, create, nil +} + +func (p *Provider) getServerFlavor(ctx context.Context, server *unikornv1.Server) (*providers.Flavor, error) { + flavors, err := p.Flavors(ctx) + if err != nil { + return nil, err + } + + i := slices.IndexFunc(flavors, func(f providers.Flavor) bool { + return server.Spec.FlavorID == f.ID + }) + + if i < 0 { + return nil, fmt.Errorf("%w: flavor %s", ErrResourceNotFound, server.Spec.FlavorID) + } + + return &flavors[i], nil +} + +func (p *Provider) getServerImage(ctx context.Context, server *unikornv1.Server) (*providers.Image, error) { + images, err := p.Images(ctx) + if err != nil { + return nil, err + } + + match := func(serverImage *unikornv1.ServerImage, i providers.Image) bool { + // If the image ID is set, use it to find the image. + if serverImage.ID != nil { + return *serverImage.ID == i.ID + } + + // Otherwise, use the image selector to find the image by name. + name := fmt.Sprintf("%s-%s", serverImage.Selector.OS, serverImage.Selector.Version) + return name == i.Name + } + + i := slices.IndexFunc(images, func(i providers.Image) bool { + return match(server.Spec.Image, i) + }) + + if i < 0 { + return nil, fmt.Errorf("%w: image %v", ErrResourceNotFound, server.Spec.Image) + } + + return &images[i], nil +} + +func (p *Provider) serverNetworksToIDs(ctx context.Context, identity *unikornv1.OpenstackIdentity, networks []unikornv1.ServerNetworkSpec) ([]string, error) { + options := &client.ListOptions{ + Namespace: identity.Namespace, + LabelSelector: labels.SelectorFromSet(map[string]string{ + constants.IdentityLabel: identity.Name, + }), + } + + resources := &unikornv1.OpenstackPhysicalNetworkList{} + if err := p.client.List(ctx, resources, options); err != nil { + return nil, err + } + + physicalNetworkMap := make(map[string]*unikornv1.OpenstackPhysicalNetwork) + for _, physNet := range resources.Items { + physicalNetworkMap[physNet.Name] = &physNet + } + + var networkIDs []string + for _, network := range networks { + physNet, found := physicalNetworkMap[network.PhysicalNetwork.ID] + if !found { + return nil, fmt.Errorf("%w: physicalnetwork %s", ErrResourceNotFound, network.PhysicalNetwork.ID) + } + + if physNet.Spec.NetworkID == nil { + return nil, fmt.Errorf("%w: physicalnetwork %s", ErrResouceDependency, network.PhysicalNetwork.ID) + } + + networkIDs = append(networkIDs, *physNet.Spec.NetworkID) + } + + return networkIDs, nil +} + +// CreateServer creates a new server. +func (p *Provider) CreateServer(ctx context.Context, identity *unikornv1.Identity, server *unikornv1.Server) error { + openstackIdentity, err := p.GetOpenstackIdentity(ctx, identity) + if err != nil { + return err + } + + openstackServer, create, err := p.GetOrCreateOpenstackServer(ctx, identity, server) + if err != nil { + return err + } + + // Always attempt to record where we are up to for idempotency. + record := func() { + log := log.FromContext(ctx) + + if create { + if err := p.client.Create(ctx, openstackServer); err != nil { + log.Error(err, "failed to create openstack server") + } + + return + } + + if err := p.client.Update(ctx, openstackServer); err != nil { + log.Error(err, "failed to update openstack server") + } + } + + defer record() + + // Rescope to the project... + providerClient := NewPasswordProvider(p.region.Spec.Openstack.Endpoint, p.credentials.userID, p.credentials.password, *openstackIdentity.Spec.ProjectID) + + computeService, err := NewComputeClient(ctx, providerClient, p.region.Spec.Openstack.Compute) + if err != nil { + return err + } + + if err := p.createServer(ctx, computeService, openstackIdentity, server, openstackServer); err != nil { + return err + } + + providerServer, err := computeService.GetServer(ctx, *openstackServer.Spec.ServerID) + if err != nil { + return err + } + + // wait for server to be active + if providerServer.Status != "ACTIVE" { + return provisioners.ErrYield + } + + addr, err := p.getServerFixedIP(providerServer) + if err != nil { + return err + } + + server.Status.PrivateIP = addr + + if server.Spec.PublicIPAllocation != nil && server.Spec.PublicIPAllocation.Enabled { + networkService, err := NewNetworkClient(ctx, providerClient, p.region.Spec.Openstack.Network) + if err != nil { + return err + } + + if err := p.allocateServerFloatingIP(ctx, networkService, server, openstackServer); err != nil { + return err + } + } + + return nil +} + +func (p *Provider) createServer(ctx context.Context, computeService *ComputeClient, identity *unikornv1.OpenstackIdentity, server *unikornv1.Server, openstackServer *unikornv1.OpenstackServer) error { + if openstackServer.Spec.ServerID != nil { + return nil + } + + flavor, err := p.getServerFlavor(ctx, server) + if err != nil { + return err + } + + image, err := p.getServerImage(ctx, server) + if err != nil { + return err + } + + networkIDs, err := p.serverNetworksToIDs(ctx, identity, server.Spec.Networks) + if err != nil { + return err + } + + // placeholder for metadata + metadata := map[string]string{} + + providerServer, err := computeService.CreateServer(ctx, server.Labels[coreconstants.NameLabel], image.ID, flavor.ID, *identity.Spec.SSHKeyName, networkIDs, identity.Spec.ServerGroupID, metadata) + if err != nil { + return err + } + + openstackServer.Spec.ServerID = &providerServer.ID + + if err := p.createServerCredentialsSecret(ctx, openstackServer, providerServer.AdminPass); err != nil { + return err + } + + return provisioners.ErrYield +} + +func (p *Provider) createServerCredentialsSecret(ctx context.Context, openstackServer *unikornv1.OpenstackServer, password string) error { + resource := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: openstackServer.Namespace, + Name: openstackServer.Name, + }, + StringData: map[string]string{ + "password": password, + }, + } + + // Ensure the secret is owned by the openstackserver so it is automatically cleaned + // up on openstackserver deletion. + if err := controllerutil.SetOwnerReference(openstackServer, resource, p.client.Scheme()); err != nil { + return err + } + + if err := p.client.Create(ctx, resource); err != nil { + return err + } + + return nil +} + +func (p *Provider) deleteServerCredentialSecret(ctx context.Context, openstackServer *unikornv1.OpenstackServer) error { + resource := &corev1.Secret{} + if err := p.client.Get(ctx, client.ObjectKey{Namespace: openstackServer.Namespace, Name: openstackServer.Name}, resource); err != nil { + if kerrors.IsNotFound(err) { + // nothing to do here + return nil + } + + return err + } + + if err := p.client.Delete(ctx, resource); err != nil { + return err + } + + return nil +} + +func (p *Provider) allocateServerFloatingIP(ctx context.Context, networkService *NetworkClient, server *unikornv1.Server, openstackServer *unikornv1.OpenstackServer) error { + if openstackServer.Spec.PublicIPAllocationId != nil { + return nil + } + + ports, err := networkService.ListServerPorts(ctx, *openstackServer.Spec.ServerID) + if err != nil { + return err + } + + if len(ports) == 0 { + return fmt.Errorf("%w: no ports found for server %s", ErrResourceNotFound, *openstackServer.Spec.ServerID) + } + + port := ports[0] + if port.Status != "ACTIVE" { + return fmt.Errorf("%w: port %s is not active", ErrResouceDependency, port.ID) + } + + floatingIP, err := networkService.CreateFloatingIP(ctx, port.ID) + if err != nil { + return err + } + + server.Status.PublicIP = &floatingIP.FloatingIP + openstackServer.Spec.PublicIPAllocationId = &floatingIP.ID + + return nil +} + +func (p *Provider) getServerFixedIP(server *servers.Server) (*string, error) { + + // Iterate through the server's addresses and extract the fixed IP. + for _, network := range server.Addresses { + for _, addr := range network.([]interface{}) { + iptype, ok := addr.(map[string]interface{})["OS-EXT-IPS:type"].(string) + if !ok || iptype != "fixed" { + continue + } + ipaddr, ok := addr.(map[string]interface{})["addr"].(string) + if !ok { + continue + } + return &ipaddr, nil + } + } + + return nil, fmt.Errorf("%w: no ip address found for server %s", ErrResourceNotFound, server.ID) +} + +// DeleteServer deletes a server. +func (p *Provider) DeleteServer(ctx context.Context, identity *unikornv1.Identity, server *unikornv1.Server) error { + openstackIdentity, err := p.GetOpenstackIdentity(ctx, identity) + if err != nil { + return err + } + + openstackServer, err := p.GetOpenstackServer(ctx, server) + if err != nil { + if !kerrors.IsNotFound(err) { + return err + } + + return nil + } + + complete := false + + // Always attempt to record where we are up to for idempotency. + record := func() { + if complete { + return + } + + log := log.FromContext(ctx) + + if err := p.client.Update(ctx, openstackServer); err != nil { + log.Error(err, "failed to update openstack server") + } + } + + defer record() + + // Rescope to the project... + providerClient := NewPasswordProvider(p.region.Spec.Openstack.Endpoint, p.credentials.userID, p.credentials.password, *openstackIdentity.Spec.ProjectID) + + if openstackServer.Spec.PublicIPAllocationId != nil { + networkService, err := NewNetworkClient(ctx, providerClient, p.region.Spec.Openstack.Network) + if err != nil { + return err + } + + if err := networkService.DeleteFloatingIP(ctx, *openstackServer.Spec.PublicIPAllocationId); err != nil { + // ignore not found errors + if !gophercloud.ResponseCodeIs(err, http.StatusNotFound) { + return err + } + } + + openstackServer.Spec.PublicIPAllocationId = nil + } + + computeService, err := NewComputeClient(ctx, providerClient, p.region.Spec.Openstack.Compute) + if err != nil { + return err + } + + if openstackServer.Spec.ServerID != nil { + if err := computeService.DeleteServer(ctx, *openstackServer.Spec.ServerID); err != nil { + // ignore not found errors + if !gophercloud.ResponseCodeIs(err, http.StatusNotFound) { + return err + } + } + + openstackServer.Spec.ServerID = nil + } + + if err := p.deleteServerCredentialSecret(ctx, openstackServer); err != nil { + return err + } + + if err := p.client.Delete(ctx, openstackServer); err != nil { + return err + } + + complete = true + + return nil +} diff --git a/pkg/provisioners/managers/server/provisioner.go b/pkg/provisioners/managers/server/provisioner.go index 27f8a60..968da27 100644 --- a/pkg/provisioners/managers/server/provisioner.go +++ b/pkg/provisioners/managers/server/provisioner.go @@ -19,11 +19,18 @@ package server import ( "context" "errors" + "fmt" unikornv1core "github.com/unikorn-cloud/core/pkg/apis/unikorn/v1alpha1" + coreclient "github.com/unikorn-cloud/core/pkg/client" coremanager "github.com/unikorn-cloud/core/pkg/manager" "github.com/unikorn-cloud/core/pkg/provisioners" unikornv1 "github.com/unikorn-cloud/region/pkg/apis/unikorn/v1alpha1" + "github.com/unikorn-cloud/region/pkg/constants" + "github.com/unikorn-cloud/region/pkg/handler/region" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" ) var ( @@ -54,12 +61,78 @@ func (p *Provisioner) Object() unikornv1core.ManagableResourceInterface { // Provision implements the Provision interface. func (p *Provisioner) Provision(ctx context.Context) error { - // TODO: Implement server provisioning + log := log.FromContext(ctx) + + cli, err := coreclient.ProvisionerClientFromContext(ctx) + if err != nil { + return err + } + + provider, err := region.NewClient(cli, p.server.Namespace).Provider(ctx, p.server.Labels[constants.RegionLabel]) + if err != nil { + return err + } + + identity, err := p.getIdentity(ctx, cli) + if err != nil { + return err + } + + // Inhibit provisioning until the identity is ready, as we may need the identity information + // to create the security group e.g. the project ID in the case of OpenStack. + status, err := identity.StatusConditionRead(unikornv1core.ConditionAvailable) + if err != nil { + log.Info("waiting for identity status update") + + return provisioners.ErrYield + } + + switch status.Reason { + case unikornv1core.ConditionReasonProvisioned: + break + case unikornv1core.ConditionReasonProvisioning: + return provisioners.ErrYield + default: + return fmt.Errorf("%w: identity in unexpected condition %v", ErrResouceDependency, status.Reason) + } + + if err := provider.CreateServer(ctx, identity, p.server); err != nil { + return err + } + return nil } // Deprovision implements the Provision interface. func (p *Provisioner) Deprovision(ctx context.Context) error { - // TODO: Implement server deprovisioning + cli, err := coreclient.ProvisionerClientFromContext(ctx) + if err != nil { + return err + } + + provider, err := region.NewClient(cli, p.server.Namespace).Provider(ctx, p.server.Labels[constants.RegionLabel]) + if err != nil { + return err + } + + identity, err := p.getIdentity(ctx, cli) + if err != nil { + return err + } + + if err := provider.DeleteServer(ctx, identity, p.server); err != nil { + return err + } + return nil } + +func (p *Provisioner) getIdentity(ctx context.Context, cli client.Client) (*unikornv1.Identity, error) { + identity := &unikornv1.Identity{} + + if err := cli.Get(ctx, client.ObjectKey{Namespace: p.server.Namespace, Name: p.server.Labels[constants.IdentityLabel]}, identity); err != nil { + return nil, err + } + + return identity, nil +} From 3746b2dc173ad76408c7dfe133a2447b736186c1 Mon Sep 17 00:00:00 2001 From: Ricardo R Date: Thu, 14 Nov 2024 10:23:53 +1300 Subject: [PATCH 2/3] update metadata --- pkg/providers/openstack/provider.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/providers/openstack/provider.go b/pkg/providers/openstack/provider.go index 9895c2c..5534a5d 100644 --- a/pkg/providers/openstack/provider.go +++ b/pkg/providers/openstack/provider.go @@ -1717,8 +1717,15 @@ func (p *Provider) createServer(ctx context.Context, computeService *ComputeClie return err } - // placeholder for metadata - metadata := map[string]string{} + // These are defined to make cross referencing between unikorn + // and openstack logging easier. + metadata := map[string]string{ + "serverID": server.Name, + "organizationID": server.Labels[coreconstants.OrganizationLabel], + "projectID": server.Labels[coreconstants.ProjectLabel], + "regionID": server.Labels[constants.RegionLabel], + "identityID": identity.Name, + } providerServer, err := computeService.CreateServer(ctx, server.Labels[coreconstants.NameLabel], image.ID, flavor.ID, *identity.Spec.SSHKeyName, networkIDs, identity.Spec.ServerGroupID, metadata) if err != nil { From b4e62420b773858d2a4243356bd53f5aef7f5b31 Mon Sep 17 00:00:00 2001 From: Ricardo R Date: Thu, 14 Nov 2024 21:52:16 +1300 Subject: [PATCH 3/3] Cascade delete server --- .../managers/identity/provisioner.go | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pkg/provisioners/managers/identity/provisioner.go b/pkg/provisioners/managers/identity/provisioner.go index a8c67bf..fa92ce7 100644 --- a/pkg/provisioners/managers/identity/provisioner.go +++ b/pkg/provisioners/managers/identity/provisioner.go @@ -93,6 +93,10 @@ func (p *Provisioner) Deprovision(ctx context.Context) error { // Block identity deletion until all owned resources are deleted, we cannot guarantee // the underlying cloud implementation will not just orphan them and leak resources. + if err := p.triggerServerDeletion(ctx, cli, selector); err != nil { + return err + } + if err := p.triggerSecurityGroupDeletion(ctx, cli, selector); err != nil { return err } @@ -174,3 +178,34 @@ func (p *Provisioner) triggerSecurityGroupDeletion(ctx context.Context, cli clie return nil } + +func (p *Provisioner) triggerServerDeletion(ctx context.Context, cli client.Client, selector labels.Selector) error { + log := log.FromContext(ctx) + + var servers unikornv1.ServerList + + if err := cli.List(ctx, &servers, &client.ListOptions{Namespace: p.identity.Namespace, LabelSelector: selector}); err != nil { + return err + } + + if len(servers.Items) != 0 { + for i := range servers.Items { + resource := &servers.Items[i] + + if resource.DeletionTimestamp != nil { + log.Info("awaiting server deletion", "server", resource.Name) + continue + } + + log.Info("triggering server deletion", "server", resource.Name) + + if err := cli.Delete(ctx, resource); err != nil { + return err + } + } + + return provisioners.ErrYield + } + + return nil +}