Add IPI installation on AWS dedicated hosts

Author: vr4manta

data/data/install.openshift.io_installconfigs.yaml

@@ -0,0 +1,55 @@
+package aws
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+// Host holds metadata for a dedicated host.
+type Host struct {
+	ID   string
+	Name string
+	Zone string
+}
+
+// dedicatedHosts retrieves a list of dedicated hosts for the given region and
+// returns them in a map keyed by the host ID.
+func dedicatedHosts(ctx context.Context, session *session.Session, region string) (map[string]Host, error) {
+	hostsByID := map[string]Host{}
+
+	client := ec2.New(session, aws.NewConfig().WithRegion(region))
+	input := &ec2.DescribeHostsInput{}
+
+	if err := client.DescribeHostsPagesWithContext(ctx, input, func(page *ec2.DescribeHostsOutput, lastPage bool) bool {
+		for _, h := range page.Hosts {
+			id := aws.StringValue(h.HostId)
+			if id == "" {
+				// Skip entries lacking an ID (should not happen)
+				continue
+			}
+			var name string
+			for _, tag := range h.Tags {
+				if aws.StringValue(tag.Key) == "Name" {
+					name = aws.StringValue(tag.Value)
+					break
+				}
+			}
+
+			fmt.Printf("Found dedicated host %s\n", id)
+			hostsByID[id] = Host{
+				ID:   id,
+				Name: name,
+				Zone: aws.StringValue(h.AvailabilityZone),
+			}
+		}
+		return !lastPage
+	}); err != nil {
+		return nil, fmt.Errorf("fetching dedicated hosts: %w", err)
+	}
+
+	return hostsByID, nil
+}

pkg/asset/installconfig/aws/dedicatedhosts.go

@@ -27,6 +27,7 @@ type Metadata struct {
 	vpc               VPC
 	instanceTypes     map[string]InstanceType
 
+	Hosts           map[string]Host
 	Region          string                     `json:"region,omitempty"`
 	ProvidedSubnets []typesaws.Subnet          `json:"subnets,omitempty"`
 	Services        []typesaws.ServiceEndpoint `json:"services,omitempty"`
@@ -390,3 +391,21 @@ func (m *Metadata) InstanceTypes(ctx context.Context) (map[string]InstanceType,
 
 	return m.instanceTypes, nil
 }
+
+// DedicatedHosts retrieves all hosts available for use to verify against this installation for configured region.
+func (m *Metadata) DedicatedHosts(ctx context.Context) (map[string]Host, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	awsSession, err := m.unlockedSession(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	m.Hosts, err = dedicatedHosts(ctx, awsSession, m.Region)
+	if err != nil {
+		return nil, fmt.Errorf("error listing dedicated hosts: %w", err)
+	}
+
+	return m.Hosts, nil
+}

pkg/asset/installconfig/aws/metadata.go

@@ -466,6 +466,8 @@ func validateMachinePool(ctx context.Context, meta *Metadata, fldPath *field.Pat
 		}
 	}
 
+	allErrs = append(allErrs, validateHostPlacement(ctx, meta, fldPath, pool)...)
+
 	return allErrs
 }
 
@@ -484,6 +486,32 @@ func translateEC2Arches(arches []string) sets.Set[string] {
 	return res
 }
 
+func validateHostPlacement(ctx context.Context, meta *Metadata, fldPath *field.Path, pool *awstypes.MachinePool) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if pool.HostPlacement != nil {
+		placementPath := fldPath.Child("hostPlacement")
+		if pool.HostPlacement.DedicatedHost != nil {
+			configuredHosts := pool.HostPlacement.DedicatedHost
+			foundHosts, err := meta.DedicatedHosts(ctx)
+			if err != nil {
+				allErrs = append(allErrs, field.InternalError(placementPath.Child("dedicatedHost"), err))
+			} else {
+				// Check the returned configured hosts to see if the dedicated hosts defined in install-config exists.
+				for _, host := range *configuredHosts {
+					_, ok := foundHosts[host.ID]
+					if !ok {
+						errMsg := fmt.Sprintf("dedicated host %s not found", host.ID)
+						allErrs = append(allErrs, field.Invalid(placementPath.Child("dedicatedHost"), pool.InstanceType, errMsg))
+					}
+				}
+			}
+		}
+	}
+
+	return allErrs
+}
+
 func validateSecurityGroupIDs(ctx context.Context, meta *Metadata, fldPath *field.Path, platform *awstypes.Platform, pool *awstypes.MachinePool) field.ErrorList {
 	allErrs := field.ErrorList{}
 

pkg/asset/installconfig/aws/validation.go

@@ -33,6 +33,8 @@ type MachineInput struct {
 	PublicIP       bool
 	PublicIpv4Pool string
 	Ignition       *capa.Ignition
+	HostAffinity   string
+	HostIDs        []string
 }
 
 // GenerateMachines returns manifests and runtime objects to provision the control plane (including bootstrap, if applicable) nodes using CAPI.

pkg/asset/machines/aws/awsmachines.go

@@ -11,6 +11,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 
 	v1 "github.com/openshift/api/config/v1"
 	machinev1 "github.com/openshift/api/machine/v1"
@@ -35,6 +36,7 @@ type machineProviderInput struct {
 	userTags         map[string]string
 	publicSubnet     bool
 	securityGroupIDs []string
+	dedicatedHost    string
 }
 
 // Machines returns a list of machines for a machinepool.
@@ -291,6 +293,15 @@ func provider(in *machineProviderInput) (*machineapi.AWSMachineProviderConfig, e
 		config.MetadataServiceOptions.Authentication = machineapi.MetadataServiceAuthentication(in.imds.Authentication)
 	}
 
+	if in.dedicatedHost != "" {
+		config.HostPlacement = &machineapi.HostPlacement{
+			Affinity: ptr.To(machineapi.HostAffinityDedicatedHost),
+			DedicatedHost: &machineapi.DedicatedHost{
+				ID: in.dedicatedHost,
+			},
+		}
+	}
+
 	return config, nil
 }
 
@@ -340,3 +351,18 @@ func ConfigMasters(machines []machineapi.Machine, controlPlane *machinev1.Contro
 	providerSpec := controlPlane.Spec.Template.OpenShiftMachineV1Beta1Machine.Spec.ProviderSpec.Value.Object.(*machineapi.AWSMachineProviderConfig)
 	providerSpec.LoadBalancers = lbrefs
 }
+
+// DedicatedHost sets dedicated hosts for the specified zone.
+func DedicatedHost(hosts map[string]aws.Host, placement *awstypes.HostPlacement, zone string) string {
+	// If install-config has HostPlacements configured, lets check the DedicatedHosts to see if one matches our region & zone.
+	if placement != nil {
+		// We only support one host ID currently for an instance.  Need to also get host that matches the zone the machines will be put into.
+		for _, host := range *placement.DedicatedHost {
+			hostDetails, found := hosts[host.ID]
+			if found && hostDetails.Zone == zone {
+				return hostDetails.ID
+			}
+		}
+	}
+	return ""
+}

pkg/asset/machines/aws/machines.go

@@ -25,6 +25,7 @@ type MachineSetInput struct {
 	Pool                     *types.MachinePool
 	Role                     string
 	UserDataSecret           string
+	Hosts                    map[string]icaws.Host
 }
 
 // MachineSets returns a list of machinesets for a machinepool.
@@ -87,6 +88,8 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 			instanceProfile = fmt.Sprintf("%s-worker-profile", in.ClusterID)
 		}
 
+		dedicatedHost := DedicatedHost(in.Hosts, mpool.HostPlacement, az)
+
 		provider, err := provider(&machineProviderInput{
 			clusterID:        in.ClusterID,
 			region:           in.InstallConfigPlatformAWS.Region,
@@ -102,12 +105,21 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 			userTags:         in.InstallConfigPlatformAWS.UserTags,
 			publicSubnet:     publicSubnet,
 			securityGroupIDs: in.Pool.Platform.AWS.AdditionalSecurityGroupIDs,
+			dedicatedHost:    dedicatedHost,
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create provider")
 		}
+
+		// If we are using any feature that is only available via CAPI, we must set the authoritativeAPI = ClusterAPI
+		authoritativeAPI := machineapi.MachineAuthorityMachineAPI
+		if isAuthoritativeClusterAPIRequired(provider) {
+			authoritativeAPI = machineapi.MachineAuthorityClusterAPI
+		}
+
 		name := fmt.Sprintf("%s-%s-%s", in.ClusterID, in.Pool.Name, az)
 		spec := machineapi.MachineSpec{
+			AuthoritativeAPI: authoritativeAPI,
 			ProviderSpec: machineapi.ProviderSpec{
 				Value: &runtime.RawExtension{Object: provider},
 			},
@@ -130,7 +142,8 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 				},
 			},
 			Spec: machineapi.MachineSetSpec{
-				Replicas: &replicas,
+				AuthoritativeAPI: authoritativeAPI,
+				Replicas:         &replicas,
 				Selector: metav1.LabelSelector{
 					MatchLabels: map[string]string{
 						"machine.openshift.io/cluster-api-machineset": name,
@@ -151,8 +164,17 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 				},
 			},
 		}
+
 		machinesets = append(machinesets, mset)
 	}
 
 	return machinesets, nil
 }
+
+// isAuthoritativeClusterAPIRequired is called to determine if the machine spec should have the AuthoritativeAPI set to ClusterAPI.
+func isAuthoritativeClusterAPIRequired(provider *machineapi.AWSMachineProviderConfig) bool {
+	if provider.HostPlacement != nil && *provider.HostPlacement.Affinity != machineapi.HostAffinityAnyAvailable {
+		return true
+	}
+	return false
+}

pkg/asset/machines/aws/machinesets.go

@@ -543,6 +543,7 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error
 				Pool:                     &pool,
 				Role:                     pool.Name,
 				UserDataSecret:           workerUserDataSecretName,
+				Hosts:                    installConfig.AWS.Hosts,
 			})
 			if err != nil {
 				return errors.Wrap(err, "failed to create worker machine objects")

pkg/asset/machines/worker.go

@@ -48,6 +48,14 @@ type MachinePool struct {
 	// +kubebuilder:validation:MaxItems=10
 	// +optional
 	AdditionalSecurityGroupIDs []string `json:"additionalSecurityGroupIDs,omitempty"`
+
+	// hostPlacement configures placement on AWS Dedicated Hosts. This allows admins to assign instances to specific host
+	// for a variety of needs including for regulatory compliance, to leverage existing per-socket or per-core software licenses (BYOL),
+	// and to gain visibility and control over instance placement on a physical server.
+	// When omitted, the instance is not constrained to a dedicated host.
+	// +openshift:enable:FeatureGate=AWSDedicatedHosts
+	// +optional
+	HostPlacement *HostPlacement `json:"hostPlacement,omitempty"`
 }
 
 // Set sets the values from `required` to `a`.
@@ -96,6 +104,10 @@ func (a *MachinePool) Set(required *MachinePool) {
 	if len(required.AdditionalSecurityGroupIDs) > 0 {
 		a.AdditionalSecurityGroupIDs = required.AdditionalSecurityGroupIDs
 	}
+
+	if required.HostPlacement != nil {
+		a.HostPlacement = required.HostPlacement
+	}
 }
 
 // EC2RootVolume defines the storage for an ec2 instance.
@@ -135,3 +147,50 @@ type EC2Metadata struct {
 	// +optional
 	Authentication string `json:"authentication,omitempty"`
 }
+
+// HostPlacement is the type that will be used to configure the placement of AWS instances.
+// This can be configured for default placement (AnyAvailable) and dedicated hosts (DedicatedHost).
+// +kubebuilder:validation:XValidation:rule="has(self.type) && self.affinity == 'DedicatedHost' ?  has(self.dedicatedHost) : !has(self.dedicatedHost)",message="dedicatedHost is required when affinity is DedicatedHost, and forbidden otherwise"
+type HostPlacement struct {
+	// affinity specifies the affinity setting for the instance.
+	// Allowed values are AnyAvailable and DedicatedHost.
+	// When Affinity is set to DedicatedHost, an instance started onto a specific host always restarts on the same host if stopped. In this scenario, the `dedicatedHost` field must be set.
+	// When Affinity is set to AnyAvailable, and you stop and restart the instance, it can be restarted on any available host.
+	// +required
+	// +unionDiscriminator
+	Affinity *HostAffinity `json:"affinity,omitempty"`
+
+	// dedicatedHost specifies the exact host that an instance should be restarted on if stopped.
+	// dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise.
+	// +optional
+	// +unionMember
+	DedicatedHost *[]DedicatedHost `json:"dedicatedHost,omitempty"`
+}
+
+// HostAffinity selects how an instance should be placed on AWS Dedicated Hosts.
+// +kubebuilder:validation:Enum:=DedicatedHost;AnyAvailable
+type HostAffinity string
+
+const (
+	// HostAffinityAnyAvailable lets the platform select any available dedicated host.
+	HostAffinityAnyAvailable HostAffinity = "AnyAvailable"
+
+	// HostAffinityDedicatedHost requires specifying a particular host via dedicatedHost.host.hostID.
+	HostAffinityDedicatedHost HostAffinity = "DedicatedHost"
+)
+
+// DedicatedHost represents the configuration for the usage of dedicated host.
+type DedicatedHost struct {
+	// id identifies the AWS Dedicated Host on which the instance must run.
+	// The value must start with "h-" followed by 17 lowercase hexadecimal characters (0-9 and a-f).
+	// Must be exactly 19 characters in length.
+	// +kubebuilder:validation:XValidation:rule="self.matches('^h-[0-9a-f]{17}$')",message="hostID must start with 'h-' followed by 17 lowercase hexadecimal characters (0-9 and a-f)"
+	// +kubebuilder:validation:MinLength=19
+	// +kubebuilder:validation:MaxLength=19
+	// +required
+	ID string `json:"id,omitempty"`
+
+	// zone is the availability zone that the dedicated host belongs to
+	// +optional
+	Zone string `json:"zone,omitempty"`
+}

pkg/types/aws/machinepool.go

@@ -2,6 +2,7 @@ package validation
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -27,6 +28,9 @@ var (
 	}()
 
 	validMetadataAuthValues = sets.NewString("Required", "Optional")
+
+	// awsDedicatedHostNamePattern is a regex expression that defines the dedicated host id format.
+	awsDedicatedHostNamePattern = regexp.MustCompile(`^h-[0-9a-f]{17}$`)
 )
 
 // AWS has a limit of 16 security groups. See:
@@ -54,6 +58,40 @@ func ValidateMachinePool(platform *aws.Platform, p *aws.MachinePool, fldPath *fi
 
 	allErrs = append(allErrs, validateSecurityGroups(platform, p, fldPath)...)
 
+	if p.HostPlacement != nil {
+		allErrs = append(allErrs, validateHostPlacement(p, fldPath.Child("hostPlacement"))...)
+	}
+
+	return allErrs
+}
+
+func validateHostPlacement(p *aws.MachinePool, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if p.HostPlacement.Affinity != nil {
+		switch *p.HostPlacement.Affinity {
+		case aws.HostAffinityAnyAvailable:
+			if p.HostPlacement.DedicatedHost != nil {
+				allErrs = append(allErrs, field.Required(fldPath.Child("dedicatedHost"), "dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise"))
+			}
+		case aws.HostAffinityDedicatedHost:
+			if p.HostPlacement.DedicatedHost == nil {
+				allErrs = append(allErrs, field.Required(fldPath.Child("dedicatedHost"), "dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise"))
+			} else {
+				for index, host := range *p.HostPlacement.DedicatedHost {
+					hostPath := fldPath.Child("dedicatedHost").Index(index)
+					if len(host.ID) == 0 {
+						allErrs = append(allErrs, field.Required(hostPath.Child("id"), "a hostID must be specified when hostAffinity is set"))
+					} else if awsDedicatedHostNamePattern.FindStringSubmatch(host.ID) == nil {
+						allErrs = append(allErrs, field.Invalid(hostPath.Child("id"), host.ID, "id must start with 'h-' followed by 17 lowercase hexadecimal characters (0-9 and a-f)"))
+					}
+				}
+			}
+		default:
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("affinity"), p.HostPlacement.Affinity, "affinity must be either AnyAvailable or DedicatedHost"))
+		}
+	}
+
 	return allErrs
 }