diff --git a/data/data/install.openshift.io_installconfigs.yaml b/data/data/install.openshift.io_installconfigs.yaml index 776f8f1c87..c42cbccdf2 100644 --- a/data/data/install.openshift.io_installconfigs.yaml +++ b/data/data/install.openshift.io_installconfigs.yaml @@ -199,6 +199,60 @@ spec: - AMDEncryptedVirtualizationNestedPaging type: string type: object + hostPlacement: + description: |- + hostPlacement configures placement on AWS Dedicated Hosts. This allows admins to assign instances to specific host + for a variety of needs including for regulatory compliance, to leverage existing per-socket or per-core software licenses (BYOL), + and to gain visibility and control over instance placement on a physical server. + When omitted, the instance is not constrained to a dedicated host. + properties: + affinity: + description: |- + affinity specifies the affinity setting for the instance. + Allowed values are AnyAvailable and DedicatedHost. + When Affinity is set to DedicatedHost, an instance started onto a specific host always restarts on the same host if stopped. In this scenario, the `dedicatedHost` field must be set. + When Affinity is set to AnyAvailable, and you stop and restart the instance, it can be restarted on any available host. + enum: + - DedicatedHost + - AnyAvailable + type: string + dedicatedHost: + description: |- + dedicatedHost specifies the exact host that an instance should be restarted on if stopped. + dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise. + items: + description: DedicatedHost represents the configuration + for the usage of dedicated host. + properties: + id: + description: |- + id identifies the AWS Dedicated Host on which the instance must run. + The value must start with "h-" followed by 17 lowercase hexadecimal characters (0-9 and a-f). + Must be exactly 19 characters in length. + maxLength: 19 + minLength: 19 + type: string + x-kubernetes-validations: + - message: hostID must start with 'h-' followed + by 17 lowercase hexadecimal characters (0-9 + and a-f) + rule: self.matches('^h-[0-9a-f]{17}$') + zone: + description: zone is the availability zone that + the dedicated host belongs to + type: string + required: + - id + type: object + type: array + required: + - affinity + type: object + x-kubernetes-validations: + - message: dedicatedHost is required when affinity is DedicatedHost, + and forbidden otherwise + rule: 'has(self.affinity) && self.affinity == ''DedicatedHost'' + ? has(self.dedicatedHost) : !has(self.dedicatedHost)' iamProfile: description: |- IAMProfile is the name of the IAM instance profile to use for the machine. @@ -1745,6 +1799,60 @@ spec: - AMDEncryptedVirtualizationNestedPaging type: string type: object + hostPlacement: + description: |- + hostPlacement configures placement on AWS Dedicated Hosts. This allows admins to assign instances to specific host + for a variety of needs including for regulatory compliance, to leverage existing per-socket or per-core software licenses (BYOL), + and to gain visibility and control over instance placement on a physical server. + When omitted, the instance is not constrained to a dedicated host. + properties: + affinity: + description: |- + affinity specifies the affinity setting for the instance. + Allowed values are AnyAvailable and DedicatedHost. + When Affinity is set to DedicatedHost, an instance started onto a specific host always restarts on the same host if stopped. In this scenario, the `dedicatedHost` field must be set. + When Affinity is set to AnyAvailable, and you stop and restart the instance, it can be restarted on any available host. + enum: + - DedicatedHost + - AnyAvailable + type: string + dedicatedHost: + description: |- + dedicatedHost specifies the exact host that an instance should be restarted on if stopped. + dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise. + items: + description: DedicatedHost represents the configuration + for the usage of dedicated host. + properties: + id: + description: |- + id identifies the AWS Dedicated Host on which the instance must run. + The value must start with "h-" followed by 17 lowercase hexadecimal characters (0-9 and a-f). + Must be exactly 19 characters in length. + maxLength: 19 + minLength: 19 + type: string + x-kubernetes-validations: + - message: hostID must start with 'h-' followed + by 17 lowercase hexadecimal characters (0-9 + and a-f) + rule: self.matches('^h-[0-9a-f]{17}$') + zone: + description: zone is the availability zone that + the dedicated host belongs to + type: string + required: + - id + type: object + type: array + required: + - affinity + type: object + x-kubernetes-validations: + - message: dedicatedHost is required when affinity is DedicatedHost, + and forbidden otherwise + rule: 'has(self.affinity) && self.affinity == ''DedicatedHost'' + ? has(self.dedicatedHost) : !has(self.dedicatedHost)' iamProfile: description: |- IAMProfile is the name of the IAM instance profile to use for the machine. @@ -3231,6 +3339,60 @@ spec: - AMDEncryptedVirtualizationNestedPaging type: string type: object + hostPlacement: + description: |- + hostPlacement configures placement on AWS Dedicated Hosts. This allows admins to assign instances to specific host + for a variety of needs including for regulatory compliance, to leverage existing per-socket or per-core software licenses (BYOL), + and to gain visibility and control over instance placement on a physical server. + When omitted, the instance is not constrained to a dedicated host. + properties: + affinity: + description: |- + affinity specifies the affinity setting for the instance. + Allowed values are AnyAvailable and DedicatedHost. + When Affinity is set to DedicatedHost, an instance started onto a specific host always restarts on the same host if stopped. In this scenario, the `dedicatedHost` field must be set. + When Affinity is set to AnyAvailable, and you stop and restart the instance, it can be restarted on any available host. + enum: + - DedicatedHost + - AnyAvailable + type: string + dedicatedHost: + description: |- + dedicatedHost specifies the exact host that an instance should be restarted on if stopped. + dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise. + items: + description: DedicatedHost represents the configuration + for the usage of dedicated host. + properties: + id: + description: |- + id identifies the AWS Dedicated Host on which the instance must run. + The value must start with "h-" followed by 17 lowercase hexadecimal characters (0-9 and a-f). + Must be exactly 19 characters in length. + maxLength: 19 + minLength: 19 + type: string + x-kubernetes-validations: + - message: hostID must start with 'h-' followed + by 17 lowercase hexadecimal characters (0-9 + and a-f) + rule: self.matches('^h-[0-9a-f]{17}$') + zone: + description: zone is the availability zone that + the dedicated host belongs to + type: string + required: + - id + type: object + type: array + required: + - affinity + type: object + x-kubernetes-validations: + - message: dedicatedHost is required when affinity is DedicatedHost, + and forbidden otherwise + rule: 'has(self.affinity) && self.affinity == ''DedicatedHost'' + ? has(self.dedicatedHost) : !has(self.dedicatedHost)' iamProfile: description: |- IAMProfile is the name of the IAM instance profile to use for the machine. @@ -4910,6 +5072,60 @@ spec: - AMDEncryptedVirtualizationNestedPaging type: string type: object + hostPlacement: + description: |- + hostPlacement configures placement on AWS Dedicated Hosts. This allows admins to assign instances to specific host + for a variety of needs including for regulatory compliance, to leverage existing per-socket or per-core software licenses (BYOL), + and to gain visibility and control over instance placement on a physical server. + When omitted, the instance is not constrained to a dedicated host. + properties: + affinity: + description: |- + affinity specifies the affinity setting for the instance. + Allowed values are AnyAvailable and DedicatedHost. + When Affinity is set to DedicatedHost, an instance started onto a specific host always restarts on the same host if stopped. In this scenario, the `dedicatedHost` field must be set. + When Affinity is set to AnyAvailable, and you stop and restart the instance, it can be restarted on any available host. + enum: + - DedicatedHost + - AnyAvailable + type: string + dedicatedHost: + description: |- + dedicatedHost specifies the exact host that an instance should be restarted on if stopped. + dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise. + items: + description: DedicatedHost represents the configuration + for the usage of dedicated host. + properties: + id: + description: |- + id identifies the AWS Dedicated Host on which the instance must run. + The value must start with "h-" followed by 17 lowercase hexadecimal characters (0-9 and a-f). + Must be exactly 19 characters in length. + maxLength: 19 + minLength: 19 + type: string + x-kubernetes-validations: + - message: hostID must start with 'h-' followed + by 17 lowercase hexadecimal characters (0-9 + and a-f) + rule: self.matches('^h-[0-9a-f]{17}$') + zone: + description: zone is the availability zone that + the dedicated host belongs to + type: string + required: + - id + type: object + type: array + required: + - affinity + type: object + x-kubernetes-validations: + - message: dedicatedHost is required when affinity is DedicatedHost, + and forbidden otherwise + rule: 'has(self.affinity) && self.affinity == ''DedicatedHost'' + ? has(self.dedicatedHost) : !has(self.dedicatedHost)' iamProfile: description: |- IAMProfile is the name of the IAM instance profile to use for the machine. diff --git a/pkg/asset/installconfig/aws/dedicatedhosts.go b/pkg/asset/installconfig/aws/dedicatedhosts.go new file mode 100644 index 0000000000..93b12af542 --- /dev/null +++ b/pkg/asset/installconfig/aws/dedicatedhosts.go @@ -0,0 +1,47 @@ +package aws + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/sirupsen/logrus" +) + +// Host holds metadata for a dedicated host. +type Host struct { + ID string + Zone string +} + +// dedicatedHosts retrieves a list of dedicated hosts for the given region and +// returns them in a map keyed by the host ID. +func dedicatedHosts(ctx context.Context, session *session.Session, region string) (map[string]Host, error) { + hostsByID := map[string]Host{} + + client := ec2.New(session, aws.NewConfig().WithRegion(region)) + input := &ec2.DescribeHostsInput{} + + if err := client.DescribeHostsPagesWithContext(ctx, input, func(page *ec2.DescribeHostsOutput, lastPage bool) bool { + for _, h := range page.Hosts { + id := aws.StringValue(h.HostId) + if id == "" { + // Skip entries lacking an ID (should not happen) + continue + } + + logrus.Debugf("Found dedicatd host: %s", id) + hostsByID[id] = Host{ + ID: id, + Zone: aws.StringValue(h.AvailabilityZone), + } + } + return !lastPage + }); err != nil { + return nil, fmt.Errorf("fetching dedicated hosts: %w", err) + } + + return hostsByID, nil +} diff --git a/pkg/asset/installconfig/aws/metadata.go b/pkg/asset/installconfig/aws/metadata.go index f9ca37ad22..8cdae75784 100644 --- a/pkg/asset/installconfig/aws/metadata.go +++ b/pkg/asset/installconfig/aws/metadata.go @@ -27,6 +27,7 @@ type Metadata struct { vpc VPC instanceTypes map[string]InstanceType + Hosts map[string]Host Region string `json:"region,omitempty"` ProvidedSubnets []typesaws.Subnet `json:"subnets,omitempty"` Services []typesaws.ServiceEndpoint `json:"services,omitempty"` @@ -390,3 +391,23 @@ func (m *Metadata) InstanceTypes(ctx context.Context) (map[string]InstanceType, return m.instanceTypes, nil } + +// DedicatedHosts retrieves all hosts available for use to verify against this installation for configured region. +func (m *Metadata) DedicatedHosts(ctx context.Context) (map[string]Host, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + if len(m.Hosts) == 0 { + awsSession, err := m.unlockedSession(ctx) + if err != nil { + return nil, err + } + + m.Hosts, err = dedicatedHosts(ctx, awsSession, m.Region) + if err != nil { + return nil, fmt.Errorf("error listing dedicated hosts: %w", err) + } + } + + return m.Hosts, nil +} diff --git a/pkg/asset/installconfig/aws/validation.go b/pkg/asset/installconfig/aws/validation.go index 8401bfcf06..81a6d2a39b 100644 --- a/pkg/asset/installconfig/aws/validation.go +++ b/pkg/asset/installconfig/aws/validation.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "net/url" + "slices" "sort" ec2v2 "github.com/aws/aws-sdk-go-v2/service/ec2" @@ -466,6 +467,8 @@ func validateMachinePool(ctx context.Context, meta *Metadata, fldPath *field.Pat } } + allErrs = append(allErrs, validateHostPlacement(ctx, meta, fldPath, pool)...) + return allErrs } @@ -484,6 +487,54 @@ func translateEC2Arches(arches []string) sets.Set[string] { return res } +func validateHostPlacement(ctx context.Context, meta *Metadata, fldPath *field.Path, pool *awstypes.MachinePool) field.ErrorList { + allErrs := field.ErrorList{} + + if pool.HostPlacement == nil { + return allErrs + } + + if pool.HostPlacement.Affinity != nil && *pool.HostPlacement.Affinity == awstypes.HostAffinityDedicatedHost { + placementPath := fldPath.Child("hostPlacement") + if pool.HostPlacement.DedicatedHost != nil { + configuredHosts := pool.HostPlacement.DedicatedHost + + // Check to see if all configured hosts exist + foundHosts, err := meta.DedicatedHosts(ctx) + if err != nil { + allErrs = append(allErrs, field.InternalError(placementPath.Child("dedicatedHost"), err)) + } else { + // Check the returned configured hosts to see if the dedicated hosts defined in install-config exists. + for idx, host := range configuredHosts { + dhPath := placementPath.Child("dedicatedHost").Index(idx) + + // Is host in AWS? + foundHost, ok := foundHosts[host.ID] + if !ok { + errMsg := fmt.Sprintf("dedicated host %s not found", host.ID) + allErrs = append(allErrs, field.Invalid(dhPath, host, errMsg)) + continue + } + + // Is host valid for pools region and zone config? + if !slices.Contains(pool.Zones, foundHost.Zone) { + errMsg := fmt.Sprintf("dedicated host %s is not available in pool's zone list", host.ID) + allErrs = append(allErrs, field.Invalid(dhPath, host, errMsg)) + } + + // If user configured the zone for the dedicated host, let's check to make sure its correct + if host.Zone != "" && host.Zone != foundHost.Zone { + errMsg := fmt.Sprintf("dedicated host was configured with zone %v but expected zone %v", host.Zone, foundHost.Zone) + allErrs = append(allErrs, field.Invalid(dhPath.Child("zone"), host, errMsg)) + } + } + } + } + } + + return allErrs +} + func validateSecurityGroupIDs(ctx context.Context, meta *Metadata, fldPath *field.Path, platform *awstypes.Platform, pool *awstypes.MachinePool) field.ErrorList { allErrs := field.ErrorList{} diff --git a/pkg/asset/installconfig/aws/validation_test.go b/pkg/asset/installconfig/aws/validation_test.go index 34d155a753..7b09dc9b55 100644 --- a/pkg/asset/installconfig/aws/validation_test.go +++ b/pkg/asset/installconfig/aws/validation_test.go @@ -74,6 +74,7 @@ func TestValidate(t *testing.T) { subnetsInVPC *SubnetGroups vpcTags Tags instanceTypes map[string]InstanceType + hosts map[string]Host proxy string publicOnly bool expectErr string @@ -1200,6 +1201,57 @@ func TestValidate(t *testing.T) { }, expectErr: `^\Qplatform.aws.vpc.subnets: Forbidden: subnet subnet-valid-public-a1 is owned by other clusters [another-cluster] and cannot be used for new installations, another subnet must be created separately\E$`, }, + { + name: "valid dedicated host placement on compute", + installConfig: icBuild.build( + icBuild.withComputePlatformZones([]string{"a"}, true, 0), + icBuild.withComputeHostPlacement([]string{"h-1234567890abcdef0"}, 0), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + hosts: map[string]Host{ + "h-1234567890abcdef0": {ID: "h-1234567890abcdef0", Zone: "a"}, + }, + }, + { + name: "invalid dedicated host not found", + installConfig: icBuild.build( + icBuild.withComputePlatformZones([]string{"a"}, true, 0), + icBuild.withComputeHostPlacement([]string{"h-aaaaaaaaaaaaaaaaa"}, 0), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + hosts: map[string]Host{ + "h-1234567890abcdef0": {ID: "h-1234567890abcdef0", Zone: "a"}, + }, + expectErr: "dedicated host h-aaaaaaaaaaaaaaaaa not found", + }, + { + name: "invalid dedicated host zone not in pool zones", + installConfig: icBuild.build( + icBuild.withComputePlatformZones([]string{"a"}, true, 0), + icBuild.withComputeHostPlacement([]string{"h-bbbbbbbbbbbbbbbbb"}, 0), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + hosts: map[string]Host{ + "h-bbbbbbbbbbbbbbbbb": {ID: "h-bbbbbbbbbbbbbbbbb", Zone: "b"}, + }, + expectErr: "is not available in pool's zone list", + }, + { + name: "dedicated host placement on compute but for a zone that pool is not using", + installConfig: icBuild.build( + icBuild.withComputePlatformZones([]string{"b"}, true, 0), + icBuild.withComputeHostPlacementAndZone([]string{"h-1234567890abcdef0"}, "b", 0), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + hosts: map[string]Host{ + "h-1234567890abcdef0": {ID: "h-1234567890abcdef0", Zone: "a"}, + }, + expectErr: "dedicated host was configured with zone b but expected zone a", + }, } // Register mock http(s) responses for tests. @@ -1232,6 +1284,7 @@ func TestValidate(t *testing.T) { Tags: test.vpcTags, }, instanceTypes: test.instanceTypes, + Hosts: test.hosts, ProvidedSubnets: test.installConfig.Platform.AWS.VPC.Subnets, } @@ -1952,6 +2005,34 @@ func (icBuild icBuildForAWS) withComputePlatformZones(zones []string, overwrite } } +func (icBuild icBuildForAWS) withComputeHostPlacement(hostIDs []string, index int) icOption { + return func(ic *types.InstallConfig) { + aff := aws.HostAffinityDedicatedHost + dhs := make([]aws.DedicatedHost, 0, len(hostIDs)) + for _, id := range hostIDs { + dhs = append(dhs, aws.DedicatedHost{ID: id}) + } + ic.Compute[index].Platform.AWS.HostPlacement = &aws.HostPlacement{ + Affinity: &aff, + DedicatedHost: dhs, + } + } +} + +func (icBuild icBuildForAWS) withComputeHostPlacementAndZone(hostIDs []string, zone string, index int) icOption { + return func(ic *types.InstallConfig) { + aff := aws.HostAffinityDedicatedHost + dhs := make([]aws.DedicatedHost, 0, len(hostIDs)) + for _, id := range hostIDs { + dhs = append(dhs, aws.DedicatedHost{ID: id, Zone: zone}) + } + ic.Compute[index].Platform.AWS.HostPlacement = &aws.HostPlacement{ + Affinity: &aff, + DedicatedHost: dhs, + } + } +} + func (icBuild icBuildForAWS) withControlPlanePlatformAMI(amiID string) icOption { return func(ic *types.InstallConfig) { ic.ControlPlane.Platform.AWS.AMIID = amiID diff --git a/pkg/asset/machines/aws/machines.go b/pkg/asset/machines/aws/machines.go index b5519bea9c..a4bc0111a3 100644 --- a/pkg/asset/machines/aws/machines.go +++ b/pkg/asset/machines/aws/machines.go @@ -37,6 +37,7 @@ type machineProviderInput struct { publicSubnet bool securityGroupIDs []string cpuOptions *awstypes.CPUOptions + dedicatedHost string } // Machines returns a list of machines for a machinepool. @@ -304,6 +305,15 @@ func provider(in *machineProviderInput) (*machineapi.AWSMachineProviderConfig, e config.CPUOptions = &cpuOptions } + if in.dedicatedHost != "" { + config.HostPlacement = &machineapi.HostPlacement{ + Affinity: ptr.To(machineapi.HostAffinityDedicatedHost), + DedicatedHost: &machineapi.DedicatedHost{ + ID: in.dedicatedHost, + }, + } + } + return config, nil } @@ -353,3 +363,18 @@ func ConfigMasters(machines []machineapi.Machine, controlPlane *machinev1.Contro providerSpec := controlPlane.Spec.Template.OpenShiftMachineV1Beta1Machine.Spec.ProviderSpec.Value.Object.(*machineapi.AWSMachineProviderConfig) providerSpec.LoadBalancers = lbrefs } + +// DedicatedHost sets dedicated hosts for the specified zone. +func DedicatedHost(hosts map[string]aws.Host, placement *awstypes.HostPlacement, zone string) string { + // If install-config has HostPlacements configured, lets check the DedicatedHosts to see if one matches our region & zone. + if placement != nil { + // We only support one host ID currently for an instance. Need to also get host that matches the zone the machines will be put into. + for _, host := range placement.DedicatedHost { + hostDetails, found := hosts[host.ID] + if found && hostDetails.Zone == zone { + return hostDetails.ID + } + } + } + return "" +} diff --git a/pkg/asset/machines/aws/machinesets.go b/pkg/asset/machines/aws/machinesets.go index 35cc1f38e6..6b5f89eece 100644 --- a/pkg/asset/machines/aws/machinesets.go +++ b/pkg/asset/machines/aws/machinesets.go @@ -25,6 +25,7 @@ type MachineSetInput struct { Pool *types.MachinePool Role string UserDataSecret string + Hosts map[string]icaws.Host } // MachineSets returns a list of machinesets for a machinepool. @@ -87,6 +88,8 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) { instanceProfile = fmt.Sprintf("%s-worker-profile", in.ClusterID) } + dedicatedHost := DedicatedHost(in.Hosts, mpool.HostPlacement, az) + provider, err := provider(&machineProviderInput{ clusterID: in.ClusterID, region: in.InstallConfigPlatformAWS.Region, @@ -103,12 +106,21 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) { publicSubnet: publicSubnet, securityGroupIDs: in.Pool.Platform.AWS.AdditionalSecurityGroupIDs, cpuOptions: mpool.CPUOptions, + dedicatedHost: dedicatedHost, }) if err != nil { return nil, errors.Wrap(err, "failed to create provider") } + + // If we are using any feature that is only available via CAPI, we must set the authoritativeAPI = ClusterAPI + authoritativeAPI := machineapi.MachineAuthorityMachineAPI + if isAuthoritativeClusterAPIRequired(provider) { + authoritativeAPI = machineapi.MachineAuthorityClusterAPI + } + name := fmt.Sprintf("%s-%s-%s", in.ClusterID, in.Pool.Name, az) spec := machineapi.MachineSpec{ + AuthoritativeAPI: authoritativeAPI, ProviderSpec: machineapi.ProviderSpec{ Value: &runtime.RawExtension{Object: provider}, }, @@ -131,7 +143,8 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) { }, }, Spec: machineapi.MachineSetSpec{ - Replicas: &replicas, + AuthoritativeAPI: authoritativeAPI, + Replicas: &replicas, Selector: metav1.LabelSelector{ MatchLabels: map[string]string{ "machine.openshift.io/cluster-api-machineset": name, @@ -152,8 +165,17 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) { }, }, } + machinesets = append(machinesets, mset) } return machinesets, nil } + +// isAuthoritativeClusterAPIRequired is called to determine if the machine spec should have the AuthoritativeAPI set to ClusterAPI. +func isAuthoritativeClusterAPIRequired(provider *machineapi.AWSMachineProviderConfig) bool { + if provider.HostPlacement != nil && *provider.HostPlacement.Affinity != machineapi.HostAffinityAnyAvailable { + return true + } + return false +} diff --git a/pkg/asset/machines/worker.go b/pkg/asset/machines/worker.go index a90dbdc4ee..f3af959003 100644 --- a/pkg/asset/machines/worker.go +++ b/pkg/asset/machines/worker.go @@ -534,6 +534,14 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error } } + dHosts := map[string]icaws.Host{} + if pool.Platform.AWS.HostPlacement != nil { + dHosts, err = installConfig.AWS.DedicatedHosts(ctx) + if err != nil { + return fmt.Errorf("failed to retrieve dedicated hosts for compute pool: %w", err) + } + } + pool.Platform.AWS = &mpool sets, err := aws.MachineSets(&aws.MachineSetInput{ ClusterID: clusterID.InfraID, @@ -544,6 +552,7 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error Pool: &pool, Role: pool.Name, UserDataSecret: workerUserDataSecretName, + Hosts: dHosts, }) if err != nil { return errors.Wrap(err, "failed to create worker machine objects") diff --git a/pkg/asset/machines/worker_test.go b/pkg/asset/machines/worker_test.go index ba7850919c..bf6f2781a9 100644 --- a/pkg/asset/machines/worker_test.go +++ b/pkg/asset/machines/worker_test.go @@ -6,11 +6,12 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/ignition/machine" "github.com/openshift/installer/pkg/asset/installconfig" + icaws "github.com/openshift/installer/pkg/asset/installconfig/aws" "github.com/openshift/installer/pkg/asset/rhcos" "github.com/openshift/installer/pkg/types" awstypes "github.com/openshift/installer/pkg/types/aws" @@ -126,38 +127,40 @@ spec: for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { parents := asset.Parents{} + cfg := &types.InstallConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + SSHKey: tc.key, + BaseDomain: "test-domain", + Platform: types.Platform{ + AWS: &awstypes.Platform{ + Region: "us-east-1", + }, + }, + Compute: []types.MachinePool{ + { + Replicas: ptr.To(int64(1)), + Hyperthreading: tc.hyperthreading, + Platform: types.MachinePoolPlatform{ + AWS: &awstypes.MachinePool{ + Zones: []string{"us-east-1a"}, + InstanceType: "m5.large", + }, + }, + }, + }, + } + icAsset := installconfig.MakeAsset(cfg) + icAsset.AWS = icaws.NewMetadata(cfg.Platform.AWS.Region, cfg.Platform.AWS.VPC.Subnets, nil) parents.Add( &installconfig.ClusterID{ UUID: "test-uuid", InfraID: "test-infra-id", }, - installconfig.MakeAsset( - &types.InstallConfig{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - }, - SSHKey: tc.key, - BaseDomain: "test-domain", - Platform: types.Platform{ - AWS: &awstypes.Platform{ - Region: "us-east-1", - }, - }, - Compute: []types.MachinePool{ - { - Replicas: pointer.Int64Ptr(1), - Hyperthreading: tc.hyperthreading, - Platform: types.MachinePoolPlatform{ - AWS: &awstypes.MachinePool{ - Zones: []string{"us-east-1a"}, - InstanceType: "m5.large", - }, - }, - }, - }, - }), + icAsset, rhcos.MakeAsset("test-image"), - (*rhcos.Release)(pointer.StringPtr("412.86.202208101040-0")), + (*rhcos.Release)(ptr.To("412.86.202208101040-0")), &machine.Worker{ File: &asset.File{ Filename: "worker-ignition", @@ -183,34 +186,35 @@ spec: func TestComputeIsNotModified(t *testing.T) { parents := asset.Parents{} - installConfig := installconfig.MakeAsset( - &types.InstallConfig{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster", - }, - SSHKey: "ssh-rsa: dummy-key", - BaseDomain: "test-domain", - Platform: types.Platform{ - AWS: &awstypes.Platform{ - Region: "us-east-1", - DefaultMachinePlatform: &awstypes.MachinePool{ - InstanceType: "TEST_INSTANCE_TYPE", - }, + cfg := &types.InstallConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + SSHKey: "ssh-rsa: dummy-key", + BaseDomain: "test-domain", + Platform: types.Platform{ + AWS: &awstypes.Platform{ + Region: "us-east-1", + DefaultMachinePlatform: &awstypes.MachinePool{ + InstanceType: "TEST_INSTANCE_TYPE", }, }, - Compute: []types.MachinePool{ - { - Replicas: pointer.Int64Ptr(1), - Hyperthreading: types.HyperthreadingDisabled, - Platform: types.MachinePoolPlatform{ - AWS: &awstypes.MachinePool{ - Zones: []string{"us-east-1a"}, - InstanceType: "", - }, + }, + Compute: []types.MachinePool{ + { + Replicas: ptr.To(int64(1)), + Hyperthreading: types.HyperthreadingDisabled, + Platform: types.MachinePoolPlatform{ + AWS: &awstypes.MachinePool{ + Zones: []string{"us-east-1a"}, + InstanceType: "", }, }, }, - }) + }, + } + installConfig := installconfig.MakeAsset(cfg) + installConfig.AWS = icaws.NewMetadata(cfg.Platform.AWS.Region, cfg.Platform.AWS.VPC.Subnets, nil) parents.Add( &installconfig.ClusterID{ @@ -219,7 +223,7 @@ func TestComputeIsNotModified(t *testing.T) { }, installConfig, rhcos.MakeAsset("test-image"), - (*rhcos.Release)(pointer.StringPtr("412.86.202208101040-0")), + (*rhcos.Release)(ptr.To("412.86.202208101040-0")), &machine.Worker{ File: &asset.File{ Filename: "worker-ignition", diff --git a/pkg/types/aws/machinepool.go b/pkg/types/aws/machinepool.go index 3829c96c9b..a9371f4351 100644 --- a/pkg/types/aws/machinepool.go +++ b/pkg/types/aws/machinepool.go @@ -56,6 +56,14 @@ type MachinePool struct { // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/cpu-options-supported-instances-values.html // +optional CPUOptions *CPUOptions `json:"cpuOptions,omitempty,omitzero"` + + // hostPlacement configures placement on AWS Dedicated Hosts. This allows admins to assign instances to specific host + // for a variety of needs including for regulatory compliance, to leverage existing per-socket or per-core software licenses (BYOL), + // and to gain visibility and control over instance placement on a physical server. + // When omitted, the instance is not constrained to a dedicated host. + // +openshift:enable:FeatureGate=AWSDedicatedHosts + // +optional + HostPlacement *HostPlacement `json:"hostPlacement,omitempty"` } // Set sets the values from `required` to `a`. @@ -108,6 +116,10 @@ func (a *MachinePool) Set(required *MachinePool) { if required.CPUOptions != nil { a.CPUOptions = required.CPUOptions } + + if required.HostPlacement != nil { + a.HostPlacement = required.HostPlacement + } } // EC2RootVolume defines the storage for an ec2 instance. @@ -178,3 +190,50 @@ type CPUOptions struct { // +optional ConfidentialCompute *ConfidentialComputePolicy `json:"confidentialCompute,omitempty"` } + +// HostPlacement is the type that will be used to configure the placement of AWS instances. +// This can be configured for default placement (AnyAvailable) and dedicated hosts (DedicatedHost). +// +kubebuilder:validation:XValidation:rule="has(self.affinity) && self.affinity == 'DedicatedHost' ? has(self.dedicatedHost) : !has(self.dedicatedHost)",message="dedicatedHost is required when affinity is DedicatedHost, and forbidden otherwise" +type HostPlacement struct { + // affinity specifies the affinity setting for the instance. + // Allowed values are AnyAvailable and DedicatedHost. + // When Affinity is set to DedicatedHost, an instance started onto a specific host always restarts on the same host if stopped. In this scenario, the `dedicatedHost` field must be set. + // When Affinity is set to AnyAvailable, and you stop and restart the instance, it can be restarted on any available host. + // +required + // +unionDiscriminator + Affinity *HostAffinity `json:"affinity,omitempty"` + + // dedicatedHost specifies the exact host that an instance should be restarted on if stopped. + // dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise. + // +optional + // +unionMember + DedicatedHost []DedicatedHost `json:"dedicatedHost,omitempty"` +} + +// HostAffinity selects how an instance should be placed on AWS Dedicated Hosts. +// +kubebuilder:validation:Enum:=DedicatedHost;AnyAvailable +type HostAffinity string + +const ( + // HostAffinityAnyAvailable lets the platform select any available dedicated host. + HostAffinityAnyAvailable HostAffinity = "AnyAvailable" + + // HostAffinityDedicatedHost requires specifying a particular host via dedicatedHost.host.hostID. + HostAffinityDedicatedHost HostAffinity = "DedicatedHost" +) + +// DedicatedHost represents the configuration for the usage of dedicated host. +type DedicatedHost struct { + // id identifies the AWS Dedicated Host on which the instance must run. + // The value must start with "h-" followed by 17 lowercase hexadecimal characters (0-9 and a-f). + // Must be exactly 19 characters in length. + // +kubebuilder:validation:XValidation:rule="self.matches('^h-[0-9a-f]{17}$')",message="hostID must start with 'h-' followed by 17 lowercase hexadecimal characters (0-9 and a-f)" + // +kubebuilder:validation:MinLength=19 + // +kubebuilder:validation:MaxLength=19 + // +required + ID string `json:"id,omitempty"` + + // zone is the availability zone that the dedicated host belongs to + // +optional + Zone string `json:"zone,omitempty"` +} diff --git a/pkg/types/aws/validation/machinepool.go b/pkg/types/aws/validation/machinepool.go index f4f6a21cec..0f0d01bd47 100644 --- a/pkg/types/aws/validation/machinepool.go +++ b/pkg/types/aws/validation/machinepool.go @@ -2,6 +2,7 @@ package validation import ( "fmt" + "regexp" "strings" "k8s.io/apimachinery/pkg/util/sets" @@ -32,6 +33,9 @@ var ( aws.ConfidentialComputePolicyDisabled, aws.ConfidentialComputePolicySEVSNP, } + + // awsDedicatedHostNamePattern is a regex expression that defines the dedicated host id format. + awsDedicatedHostNamePattern = regexp.MustCompile(`^h-[0-9a-f]{17}$`) ) // AWS has a limit of 16 security groups. See: @@ -60,6 +64,43 @@ func ValidateMachinePool(platform *aws.Platform, p *aws.MachinePool, fldPath *fi allErrs = append(allErrs, validateSecurityGroups(platform, p, fldPath)...) allErrs = append(allErrs, ValidateCPUOptions(p, fldPath)...) + if p.HostPlacement != nil { + allErrs = append(allErrs, validateHostPlacement(p, fldPath.Child("hostPlacement"))...) + } + + return allErrs +} + +func validateHostPlacement(p *aws.MachinePool, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if p.HostPlacement.Affinity == nil { + allErrs = append(allErrs, field.Required(fldPath.Child("affinity"), "affinity is required when hostPlacement is configured")) + return allErrs // Can't validate further without affinity + } + + switch *p.HostPlacement.Affinity { + case aws.HostAffinityAnyAvailable: + if len(p.HostPlacement.DedicatedHost) > 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("dedicatedHost"), "dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise")) + } + case aws.HostAffinityDedicatedHost: + if len(p.HostPlacement.DedicatedHost) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("dedicatedHost"), "dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise")) + } else { + for index, host := range p.HostPlacement.DedicatedHost { + hostPath := fldPath.Child("dedicatedHost").Index(index) + if len(host.ID) == 0 { + allErrs = append(allErrs, field.Required(hostPath.Child("id"), "a hostID must be specified when configuring 'dedicatedHost'")) + } else if !awsDedicatedHostNamePattern.MatchString(host.ID) { + allErrs = append(allErrs, field.Invalid(hostPath.Child("id"), host.ID, "id must start with 'h-' followed by 17 lowercase hexadecimal characters (0-9 and a-f)")) + } + } + } + default: + allErrs = append(allErrs, field.NotSupported(fldPath.Child("affinity"), p.HostPlacement.Affinity, []aws.HostAffinity{aws.HostAffinityAnyAvailable, aws.HostAffinityDedicatedHost})) + } + return allErrs } diff --git a/pkg/types/aws/validation/machinepool_test.go b/pkg/types/aws/validation/machinepool_test.go index d8741d720e..a7851044cd 100644 --- a/pkg/types/aws/validation/machinepool_test.go +++ b/pkg/types/aws/validation/machinepool_test.go @@ -129,6 +129,86 @@ func TestValidateMachinePool(t *testing.T) { }, expected: `^test-path\.authentication: Invalid value: \"foobarbaz\": must be either Required or Optional$`, }, + { + name: "host placement any available", + pool: &aws.MachinePool{ + HostPlacement: &aws.HostPlacement{ + Affinity: ptr.To(aws.HostAffinityAnyAvailable), + }, + }, + }, + { + name: "valid dedicated hosts", + pool: &aws.MachinePool{ + HostPlacement: &aws.HostPlacement{ + Affinity: ptr.To(aws.HostAffinityDedicatedHost), + DedicatedHost: []aws.DedicatedHost{ + { + ID: "h-09dcf61cb388b0149", + }, + }, + }, + }, + }, + { + name: "invalid dedicated hosts - missing hostID", + pool: &aws.MachinePool{ + HostPlacement: &aws.HostPlacement{ + Affinity: ptr.To(aws.HostAffinityDedicatedHost), + DedicatedHost: []aws.DedicatedHost{ + {}, + }, + }, + }, + expected: `^test-path.hostPlacement.dedicatedHost\[0].id: Required value: a hostID must be specified when configuring 'dedicatedHost'$`, + }, + { + name: "invalid - hostPlacement without affinity", + pool: &aws.MachinePool{ + HostPlacement: &aws.HostPlacement{}, + }, + expected: `^test-path.hostPlacement.affinity: Required value: affinity is required when hostPlacement is configured$`, + }, + { + name: "invalid unknown affinity", + pool: &aws.MachinePool{ + HostPlacement: &aws.HostPlacement{ + Affinity: ptr.To(aws.HostAffinity("Unknown")), + }, + }, + expected: `^test-path.hostPlacement.affinity: Unsupported value: "Unknown": supported values: "AnyAvailable", "DedicatedHost"$`, + }, + { + name: "any available with dedicated host set", + pool: &aws.MachinePool{ + HostPlacement: &aws.HostPlacement{ + Affinity: ptr.To(aws.HostAffinityAnyAvailable), + DedicatedHost: []aws.DedicatedHost{{ID: "h-09dcf61cb388b0149"}}, + }, + }, + expected: `^test-path.hostPlacement.dedicatedHost: Required value: dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise$`, + }, + { + name: "invalid - DedicatedHost affinity without dedicatedHost", + pool: &aws.MachinePool{ + HostPlacement: &aws.HostPlacement{ + Affinity: ptr.To(aws.HostAffinityDedicatedHost), + }, + }, + expected: `^test-path.hostPlacement.dedicatedHost: Required value: dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise$`, + }, + { + name: "invalid dedicated host - bad hostID", + pool: &aws.MachinePool{ + HostPlacement: &aws.HostPlacement{ + Affinity: ptr.To(aws.HostAffinityDedicatedHost), + DedicatedHost: []aws.DedicatedHost{ + {ID: "h-09DCFABC"}, + }, + }, + }, + expected: `^test-path.hostPlacement.dedicatedHost\[0\].id: Invalid value: "h-09DCFABC": id must start with 'h-' followed by 17 lowercase hexadecimal characters \(0-9 and a-f\)$`, + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { diff --git a/pkg/types/aws/zz_generated.deepcopy.go b/pkg/types/aws/zz_generated.deepcopy.go index fd19e02a2b..b435ebb4d9 100644 --- a/pkg/types/aws/zz_generated.deepcopy.go +++ b/pkg/types/aws/zz_generated.deepcopy.go @@ -26,6 +26,22 @@ func (in *CPUOptions) DeepCopy() *CPUOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DedicatedHost) DeepCopyInto(out *DedicatedHost) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedHost. +func (in *DedicatedHost) DeepCopy() *DedicatedHost { + if in == nil { + return nil + } + out := new(DedicatedHost) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EC2Metadata) DeepCopyInto(out *EC2Metadata) { *out = *in @@ -58,6 +74,32 @@ func (in *EC2RootVolume) DeepCopy() *EC2RootVolume { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostPlacement) DeepCopyInto(out *HostPlacement) { + *out = *in + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(HostAffinity) + **out = **in + } + if in.DedicatedHost != nil { + in, out := &in.DedicatedHost, &out.DedicatedHost + *out = make([]DedicatedHost, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostPlacement. +func (in *HostPlacement) DeepCopy() *HostPlacement { + if in == nil { + return nil + } + out := new(HostPlacement) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MachinePool) DeepCopyInto(out *MachinePool) { *out = *in @@ -78,6 +120,11 @@ func (in *MachinePool) DeepCopyInto(out *MachinePool) { *out = new(CPUOptions) (*in).DeepCopyInto(*out) } + if in.HostPlacement != nil { + in, out := &in.HostPlacement, &out.HostPlacement + *out = new(HostPlacement) + (*in).DeepCopyInto(*out) + } return }