Add IPI installation on AWS dedicated hosts

Author: vr4manta

data/data/install.openshift.io_installconfigs.yaml

@@ -0,0 +1,47 @@
+package aws
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/sirupsen/logrus"
+)
+
+// Host holds metadata for a dedicated host.
+type Host struct {
+	ID   string
+	Zone string
+}
+
+// dedicatedHosts retrieves a list of dedicated hosts for the given region and
+// returns them in a map keyed by the host ID.
+func dedicatedHosts(ctx context.Context, session *session.Session, region string) (map[string]Host, error) {
+	hostsByID := map[string]Host{}
+
+	client := ec2.New(session, aws.NewConfig().WithRegion(region))
+	input := &ec2.DescribeHostsInput{}
+
+	if err := client.DescribeHostsPagesWithContext(ctx, input, func(page *ec2.DescribeHostsOutput, lastPage bool) bool {
+		for _, h := range page.Hosts {
+			id := aws.StringValue(h.HostId)
+			if id == "" {
+				// Skip entries lacking an ID (should not happen)
+				continue
+			}
+
+			logrus.Debugf("Found dedicatd host: %s", id)
+			hostsByID[id] = Host{
+				ID:   id,
+				Zone: aws.StringValue(h.AvailabilityZone),
+			}
+		}
+		return !lastPage
+	}); err != nil {
+		return nil, fmt.Errorf("fetching dedicated hosts: %w", err)
+	}
+
+	return hostsByID, nil
+}

pkg/asset/installconfig/aws/dedicatedhosts.go

@@ -27,6 +27,7 @@ type Metadata struct {
 	vpc               VPC
 	instanceTypes     map[string]InstanceType
 
+	Hosts           map[string]Host
 	Region          string                     `json:"region,omitempty"`
 	ProvidedSubnets []typesaws.Subnet          `json:"subnets,omitempty"`
 	Services        []typesaws.ServiceEndpoint `json:"services,omitempty"`
@@ -390,3 +391,23 @@ func (m *Metadata) InstanceTypes(ctx context.Context) (map[string]InstanceType,
 
 	return m.instanceTypes, nil
 }
+
+// DedicatedHosts retrieves all hosts available for use to verify against this installation for configured region.
+func (m *Metadata) DedicatedHosts(ctx context.Context) (map[string]Host, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	if len(m.Hosts) == 0 {
+		awsSession, err := m.unlockedSession(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		m.Hosts, err = dedicatedHosts(ctx, awsSession, m.Region)
+		if err != nil {
+			return nil, fmt.Errorf("error listing dedicated hosts: %w", err)
+		}
+	}
+
+	return m.Hosts, nil
+}

pkg/asset/installconfig/aws/metadata.go

@@ -466,6 +466,8 @@ func validateMachinePool(ctx context.Context, meta *Metadata, fldPath *field.Pat
 		}
 	}
 
+	allErrs = append(allErrs, validateHostPlacement(ctx, meta, fldPath, pool)...)
+
 	return allErrs
 }
 
@@ -484,6 +486,36 @@ func translateEC2Arches(arches []string) sets.Set[string] {
 	return res
 }
 
+func validateHostPlacement(ctx context.Context, meta *Metadata, fldPath *field.Path, pool *awstypes.MachinePool) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if pool.HostPlacement == nil {
+		return allErrs
+	}
+
+	if pool.HostPlacement.Affinity != nil && *pool.HostPlacement.Affinity == awstypes.HostAffinityDedicatedHost {
+		placementPath := fldPath.Child("hostPlacement")
+		if pool.HostPlacement.DedicatedHost != nil {
+			configuredHosts := pool.HostPlacement.DedicatedHost
+			foundHosts, err := meta.DedicatedHosts(ctx)
+			if err != nil {
+				allErrs = append(allErrs, field.InternalError(placementPath.Child("dedicatedHost"), err))
+			} else {
+				// Check the returned configured hosts to see if the dedicated hosts defined in install-config exists.
+				for _, host := range configuredHosts {
+					_, ok := foundHosts[host.ID]
+					if !ok {
+						errMsg := fmt.Sprintf("dedicated host %s not found", host.ID)
+						allErrs = append(allErrs, field.Invalid(placementPath.Child("dedicatedHost"), pool.InstanceType, errMsg))
+					}
+				}
+			}
+		}
+	}
+
+	return allErrs
+}
+
 func validateSecurityGroupIDs(ctx context.Context, meta *Metadata, fldPath *field.Path, platform *awstypes.Platform, pool *awstypes.MachinePool) field.ErrorList {
 	allErrs := field.ErrorList{}
 

pkg/asset/installconfig/aws/validation.go

@@ -11,6 +11,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 
 	v1 "github.com/openshift/api/config/v1"
 	machinev1 "github.com/openshift/api/machine/v1"
@@ -35,6 +36,7 @@ type machineProviderInput struct {
 	userTags         map[string]string
 	publicSubnet     bool
 	securityGroupIDs []string
+	dedicatedHost    string
 }
 
 // Machines returns a list of machines for a machinepool.
@@ -291,6 +293,15 @@ func provider(in *machineProviderInput) (*machineapi.AWSMachineProviderConfig, e
 		config.MetadataServiceOptions.Authentication = machineapi.MetadataServiceAuthentication(in.imds.Authentication)
 	}
 
+	if in.dedicatedHost != "" {
+		config.HostPlacement = &machineapi.HostPlacement{
+			Affinity: ptr.To(machineapi.HostAffinityDedicatedHost),
+			DedicatedHost: &machineapi.DedicatedHost{
+				ID: in.dedicatedHost,
+			},
+		}
+	}
+
 	return config, nil
 }
 
@@ -340,3 +351,18 @@ func ConfigMasters(machines []machineapi.Machine, controlPlane *machinev1.Contro
 	providerSpec := controlPlane.Spec.Template.OpenShiftMachineV1Beta1Machine.Spec.ProviderSpec.Value.Object.(*machineapi.AWSMachineProviderConfig)
 	providerSpec.LoadBalancers = lbrefs
 }
+
+// DedicatedHost sets dedicated hosts for the specified zone.
+func DedicatedHost(hosts map[string]aws.Host, placement *awstypes.HostPlacement, zone string) string {
+	// If install-config has HostPlacements configured, lets check the DedicatedHosts to see if one matches our region & zone.
+	if placement != nil {
+		// We only support one host ID currently for an instance.  Need to also get host that matches the zone the machines will be put into.
+		for _, host := range placement.DedicatedHost {
+			hostDetails, found := hosts[host.ID]
+			if found && hostDetails.Zone == zone {
+				return hostDetails.ID
+			}
+		}
+	}
+	return ""
+}

pkg/asset/machines/aws/machines.go

@@ -25,6 +25,7 @@ type MachineSetInput struct {
 	Pool                     *types.MachinePool
 	Role                     string
 	UserDataSecret           string
+	Hosts                    map[string]icaws.Host
 }
 
 // MachineSets returns a list of machinesets for a machinepool.
@@ -87,6 +88,8 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 			instanceProfile = fmt.Sprintf("%s-worker-profile", in.ClusterID)
 		}
 
+		dedicatedHost := DedicatedHost(in.Hosts, mpool.HostPlacement, az)
+
 		provider, err := provider(&machineProviderInput{
 			clusterID:        in.ClusterID,
 			region:           in.InstallConfigPlatformAWS.Region,
@@ -102,12 +105,21 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 			userTags:         in.InstallConfigPlatformAWS.UserTags,
 			publicSubnet:     publicSubnet,
 			securityGroupIDs: in.Pool.Platform.AWS.AdditionalSecurityGroupIDs,
+			dedicatedHost:    dedicatedHost,
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create provider")
 		}
+
+		// If we are using any feature that is only available via CAPI, we must set the authoritativeAPI = ClusterAPI
+		authoritativeAPI := machineapi.MachineAuthorityMachineAPI
+		if isAuthoritativeClusterAPIRequired(provider) {
+			authoritativeAPI = machineapi.MachineAuthorityClusterAPI
+		}
+
 		name := fmt.Sprintf("%s-%s-%s", in.ClusterID, in.Pool.Name, az)
 		spec := machineapi.MachineSpec{
+			AuthoritativeAPI: authoritativeAPI,
 			ProviderSpec: machineapi.ProviderSpec{
 				Value: &runtime.RawExtension{Object: provider},
 			},
@@ -130,7 +142,8 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 				},
 			},
 			Spec: machineapi.MachineSetSpec{
-				Replicas: &replicas,
+				AuthoritativeAPI: authoritativeAPI,
+				Replicas:         &replicas,
 				Selector: metav1.LabelSelector{
 					MatchLabels: map[string]string{
 						"machine.openshift.io/cluster-api-machineset": name,
@@ -151,8 +164,17 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 				},
 			},
 		}
+
 		machinesets = append(machinesets, mset)
 	}
 
 	return machinesets, nil
 }
+
+// isAuthoritativeClusterAPIRequired is called to determine if the machine spec should have the AuthoritativeAPI set to ClusterAPI.
+func isAuthoritativeClusterAPIRequired(provider *machineapi.AWSMachineProviderConfig) bool {
+	if provider.HostPlacement != nil && *provider.HostPlacement.Affinity != machineapi.HostAffinityAnyAvailable {
+		return true
+	}
+	return false
+}

pkg/asset/machines/aws/machinesets.go

@@ -533,6 +533,16 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error
 				}
 			}
 
+			// TODO: See if we can make changes mentioned in the review.
+
+			dHosts := map[string]icaws.Host{}
+			if pool.Platform.AWS.HostPlacement != nil {
+				dHosts, err = installConfig.AWS.DedicatedHosts(ctx)
+				if err != nil {
+					return fmt.Errorf("failed to retrieve dedicated hosts for compute pool: %w", err)
+				}
+			}
+
 			pool.Platform.AWS = &mpool
 			sets, err := aws.MachineSets(&aws.MachineSetInput{
 				ClusterID:                clusterID.InfraID,
@@ -543,6 +553,7 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error
 				Pool:                     &pool,
 				Role:                     pool.Name,
 				UserDataSecret:           workerUserDataSecretName,
+				Hosts:                    dHosts,
 			})
 			if err != nil {
 				return errors.Wrap(err, "failed to create worker machine objects")

pkg/asset/machines/worker.go

@@ -11,6 +11,7 @@ import (
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/ignition/machine"
 	"github.com/openshift/installer/pkg/asset/installconfig"
+	icaws "github.com/openshift/installer/pkg/asset/installconfig/aws"
 	"github.com/openshift/installer/pkg/asset/rhcos"
 	"github.com/openshift/installer/pkg/types"
 	awstypes "github.com/openshift/installer/pkg/types/aws"
@@ -126,36 +127,38 @@ spec:
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			parents := asset.Parents{}
+			cfg := &types.InstallConfig{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-cluster",
+				},
+				SSHKey:     tc.key,
+				BaseDomain: "test-domain",
+				Platform: types.Platform{
+					AWS: &awstypes.Platform{
+						Region: "us-east-1",
+					},
+				},
+				Compute: []types.MachinePool{
+					{
+						Replicas:       pointer.Int64Ptr(1),
+						Hyperthreading: tc.hyperthreading,
+						Platform: types.MachinePoolPlatform{
+							AWS: &awstypes.MachinePool{
+								Zones:        []string{"us-east-1a"},
+								InstanceType: "m5.large",
+							},
+						},
+					},
+				},
+			}
+			icAsset := installconfig.MakeAsset(cfg)
+			icAsset.AWS = icaws.NewMetadata(cfg.Platform.AWS.Region, cfg.Platform.AWS.VPC.Subnets, nil)
 			parents.Add(
 				&installconfig.ClusterID{
 					UUID:    "test-uuid",
 					InfraID: "test-infra-id",
 				},
-				installconfig.MakeAsset(
-					&types.InstallConfig{
-						ObjectMeta: metav1.ObjectMeta{
-							Name: "test-cluster",
-						},
-						SSHKey:     tc.key,
-						BaseDomain: "test-domain",
-						Platform: types.Platform{
-							AWS: &awstypes.Platform{
-								Region: "us-east-1",
-							},
-						},
-						Compute: []types.MachinePool{
-							{
-								Replicas:       pointer.Int64Ptr(1),
-								Hyperthreading: tc.hyperthreading,
-								Platform: types.MachinePoolPlatform{
-									AWS: &awstypes.MachinePool{
-										Zones:        []string{"us-east-1a"},
-										InstanceType: "m5.large",
-									},
-								},
-							},
-						},
-					}),
+				icAsset,
 				rhcos.MakeAsset("test-image"),
 				(*rhcos.Release)(pointer.StringPtr("412.86.202208101040-0")),
 				&machine.Worker{
@@ -183,34 +186,35 @@ spec:
 
 func TestComputeIsNotModified(t *testing.T) {
 	parents := asset.Parents{}
-	installConfig := installconfig.MakeAsset(
-		&types.InstallConfig{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: "test-cluster",
-			},
-			SSHKey:     "ssh-rsa: dummy-key",
-			BaseDomain: "test-domain",
-			Platform: types.Platform{
-				AWS: &awstypes.Platform{
-					Region: "us-east-1",
-					DefaultMachinePlatform: &awstypes.MachinePool{
-						InstanceType: "TEST_INSTANCE_TYPE",
-					},
+	cfg := &types.InstallConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-cluster",
+		},
+		SSHKey:     "ssh-rsa: dummy-key",
+		BaseDomain: "test-domain",
+		Platform: types.Platform{
+			AWS: &awstypes.Platform{
+				Region: "us-east-1",
+				DefaultMachinePlatform: &awstypes.MachinePool{
+					InstanceType: "TEST_INSTANCE_TYPE",
 				},
 			},
-			Compute: []types.MachinePool{
-				{
-					Replicas:       pointer.Int64Ptr(1),
-					Hyperthreading: types.HyperthreadingDisabled,
-					Platform: types.MachinePoolPlatform{
-						AWS: &awstypes.MachinePool{
-							Zones:        []string{"us-east-1a"},
-							InstanceType: "",
-						},
+		},
+		Compute: []types.MachinePool{
+			{
+				Replicas:       pointer.Int64Ptr(1),
+				Hyperthreading: types.HyperthreadingDisabled,
+				Platform: types.MachinePoolPlatform{
+					AWS: &awstypes.MachinePool{
+						Zones:        []string{"us-east-1a"},
+						InstanceType: "",
 					},
 				},
 			},
-		})
+		},
+	}
+	installConfig := installconfig.MakeAsset(cfg)
+	installConfig.AWS = icaws.NewMetadata(cfg.Platform.AWS.Region, cfg.Platform.AWS.VPC.Subnets, nil)
 
 	parents.Add(
 		&installconfig.ClusterID{