diff --git a/lib/PTO/Transforms/PTOValidateVPTOIR.cpp b/lib/PTO/Transforms/PTOValidateVPTOIR.cpp index e6fafa37a8..75ca0c1707 100644 --- a/lib/PTO/Transforms/PTOValidateVPTOIR.cpp +++ b/lib/PTO/Transforms/PTOValidateVPTOIR.cpp @@ -21,7 +21,9 @@ //===----------------------------------------------------------------------===// #include "PTO/IR/PTO.h" +#include "PTO/IR/PTOTypeUtils.h" +#include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/SCF/IR/SCF.h" @@ -998,6 +1000,23 @@ class VPTOLegalityValidator { } LogicalResult validateAuthoringOperationSurface() { + WalkResult constantWalkResult = + helper.getModule().walk([&](arith::ConstantOp constant) { + Type resultType = constant.getType(); + Type elementType = resultType; + if (auto vectorType = dyn_cast(resultType)) + elementType = vectorType.getElementType(); + if (!pto::isPTOFloat8Type(elementType)) + return WalkResult::advance(); + + constant.emitOpError() + << "does not support directly constructed FP8 constants in " + "the VPTO backend; produce FP8 values with pto.convert"; + return WalkResult::interrupt(); + }); + if (constantWalkResult.wasInterrupted()) + return failure(); + WalkResult loopWalkResult = helper.getModule().walk([&](scf::ForOp loop) { if (!VPTOLegalityHelper::isAIVectorScopeCarrier(loop)) return WalkResult::advance(); diff --git a/lib/PTO/Transforms/VPTOCANN900LLVMEmitter.cpp b/lib/PTO/Transforms/VPTOCANN900LLVMEmitter.cpp index c82a9141d8..9c50f1ec7e 100644 --- a/lib/PTO/Transforms/VPTOCANN900LLVMEmitter.cpp +++ b/lib/PTO/Transforms/VPTOCANN900LLVMEmitter.cpp @@ -183,8 +183,15 @@ static unsigned getNaturalByteAlignment(Type type) { } static bool hasVPTOConvertibleType(Type type) { - return isa(type); + if (!type) + return false; + if (isa(type) || + pto::isPTOLowPrecisionType(type)) + return true; + if (auto vecType = dyn_cast(type)) + return hasVPTOConvertibleType(vecType.getElementType()); + return false; } static bool hasVPTOConvertibleType(TypeRange types) { @@ -10441,17 +10448,41 @@ class ConvertVPTOTypedCarrierOp final : public ConversionPattern { ConversionPatternRewriter &rewriter) const override { if (isa(op)) return failure(); + Type propertyType; + if (auto allocaOp = dyn_cast(op)) + propertyType = allocaOp.getElemType(); + else if (auto gepOp = dyn_cast(op)) + propertyType = gepOp.getElemType(); if (!hasVPTOConvertibleType(op->getOperandTypes()) && - !hasVPTOConvertibleType(op->getResultTypes())) + !hasVPTOConvertibleType(op->getResultTypes()) && + !hasVPTOConvertibleType(propertyType)) return failure(); if (op->getNumRegions() != 0) return rewriter.notifyMatchFailure( op, "region ops with VPTO types are handled structurally"); - FailureOr converted = - convertOpResultTypes(op, operands, *typeConverter, rewriter); - if (failed(converted)) - return failure(); + SmallVector convertedResultTypes; + if (failed(typeConverter->convertTypes(op->getResultTypes(), + convertedResultTypes))) + return rewriter.notifyMatchFailure(op, "failed to convert result types"); + OperationState state(op->getLoc(), op->getName()); + state.addOperands(operands); + state.addTypes(convertedResultTypes); + state.addAttributes(op->getAttrs()); + state.addSuccessors(op->getSuccessors()); + state.propertiesAttr = op->getPropertiesAsAttribute(); + Operation *converted = rewriter.create(state); + if (propertyType) { + Type convertedPropertyType = typeConverter->convertType(propertyType); + if (!convertedPropertyType) + return rewriter.notifyMatchFailure( + op, "failed to convert LLVM element type"); + if (auto allocaOp = dyn_cast(converted)) + allocaOp.setElemType(convertedPropertyType); + else + cast(converted).setElemType(convertedPropertyType); + } + rewriter.replaceOp(op, converted->getResults()); return success(); } }; @@ -10881,6 +10912,16 @@ static LogicalResult lowerVPTOTypes(ModuleOp module, llvm::raw_ostream &diagOS) return !hasVPTOConvertibleType(op->getOperandTypes()) && !hasVPTOConvertibleType(op->getResultTypes()); }); + target.addDynamicallyLegalOp([&](LLVM::AllocaOp op) { + return typeConverter.isLegal(op->getOperandTypes()) && + typeConverter.isLegal(op->getResultTypes()) && + typeConverter.isLegal(op.getElemType()); + }); + target.addDynamicallyLegalOp([&](LLVM::GEPOp op) { + return typeConverter.isLegal(op->getOperandTypes()) && + typeConverter.isLegal(op->getResultTypes()) && + typeConverter.isLegal(op.getElemType()); + }); target.markUnknownOpDynamicallyLegal([&](Operation *op) { return typeConverter.isLegal(op->getOperandTypes()) && typeConverter.isLegal(op->getResultTypes()); diff --git a/lib/PTO/Transforms/VPTOLLVMEmitter.cpp b/lib/PTO/Transforms/VPTOLLVMEmitter.cpp index 24ef9a6df6..e7d5a79906 100644 --- a/lib/PTO/Transforms/VPTOLLVMEmitter.cpp +++ b/lib/PTO/Transforms/VPTOLLVMEmitter.cpp @@ -185,8 +185,15 @@ static unsigned getNaturalByteAlignment(Type type) { } static bool hasVPTOConvertibleType(Type type) { - return isa(type); + if (!type) + return false; + if (isa(type) || + pto::isPTOLowPrecisionType(type)) + return true; + if (auto vecType = dyn_cast(type)) + return hasVPTOConvertibleType(vecType.getElementType()); + return false; } static bool hasVPTOConvertibleType(TypeRange types) { @@ -11095,17 +11102,41 @@ class ConvertVPTOTypedCarrierOp final : public ConversionPattern { ConversionPatternRewriter &rewriter) const override { if (isa(op)) return failure(); + Type propertyType; + if (auto allocaOp = dyn_cast(op)) + propertyType = allocaOp.getElemType(); + else if (auto gepOp = dyn_cast(op)) + propertyType = gepOp.getElemType(); if (!hasVPTOConvertibleType(op->getOperandTypes()) && - !hasVPTOConvertibleType(op->getResultTypes())) + !hasVPTOConvertibleType(op->getResultTypes()) && + !hasVPTOConvertibleType(propertyType)) return failure(); if (op->getNumRegions() != 0) return rewriter.notifyMatchFailure( op, "region ops with VPTO types are handled structurally"); - FailureOr converted = - convertOpResultTypes(op, operands, *typeConverter, rewriter); - if (failed(converted)) - return failure(); + SmallVector convertedResultTypes; + if (failed(typeConverter->convertTypes(op->getResultTypes(), + convertedResultTypes))) + return rewriter.notifyMatchFailure(op, "failed to convert result types"); + OperationState state(op->getLoc(), op->getName()); + state.addOperands(operands); + state.addTypes(convertedResultTypes); + state.addAttributes(op->getAttrs()); + state.addSuccessors(op->getSuccessors()); + state.propertiesAttr = op->getPropertiesAsAttribute(); + Operation *converted = rewriter.create(state); + if (propertyType) { + Type convertedPropertyType = typeConverter->convertType(propertyType); + if (!convertedPropertyType) + return rewriter.notifyMatchFailure( + op, "failed to convert LLVM element type"); + if (auto allocaOp = dyn_cast(converted)) + allocaOp.setElemType(convertedPropertyType); + else + cast(converted).setElemType(convertedPropertyType); + } + rewriter.replaceOp(op, converted->getResults()); return success(); } }; @@ -11635,6 +11666,16 @@ static LogicalResult lowerVPTOTypes(ModuleOp module, llvm::raw_ostream &diagOS) return !hasVPTOConvertibleType(op->getOperandTypes()) && !hasVPTOConvertibleType(op->getResultTypes()); }); + target.addDynamicallyLegalOp([&](LLVM::AllocaOp op) { + return typeConverter.isLegal(op->getOperandTypes()) && + typeConverter.isLegal(op->getResultTypes()) && + typeConverter.isLegal(op.getElemType()); + }); + target.addDynamicallyLegalOp([&](LLVM::GEPOp op) { + return typeConverter.isLegal(op->getOperandTypes()) && + typeConverter.isLegal(op->getResultTypes()) && + typeConverter.isLegal(op.getElemType()); + }); target.markUnknownOpDynamicallyLegal([&](Operation *op) { return typeConverter.isLegal(op->getOperandTypes()) && typeConverter.isLegal(op->getResultTypes()); diff --git a/test/lit/vpto/simt_fp8_direct_constant_unsupported.pto b/test/lit/vpto/simt_fp8_direct_constant_unsupported.pto new file mode 100644 index 0000000000..d1cae102ed --- /dev/null +++ b/test/lit/vpto/simt_fp8_direct_constant_unsupported.pto @@ -0,0 +1,40 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: split-file %s %t +// RUN: not ptoas --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %t/e4m3.pto -o - 2>&1 | FileCheck %s --check-prefix=E4M3 +// RUN: not ptoas --cann-output-version=9.0.0 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %t/e4m3.pto -o - 2>&1 | FileCheck %s --check-prefix=E4M3 +// RUN: not ptoas --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %t/e5m2.pto -o - 2>&1 | FileCheck %s --check-prefix=E5M2 +// RUN: not ptoas --cann-output-version=9.0.0 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %t/e5m2.pto -o - 2>&1 | FileCheck %s --check-prefix=E5M2 + +// E4M3: error: 'arith.constant' op does not support directly constructed FP8 constants in the VPTO backend; produce FP8 values with pto.convert +// E5M2: error: 'arith.constant' op does not support directly constructed FP8 constants in the VPTO backend; produce FP8 values with pto.convert + +//--- e4m3.pto +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @direct_e4m3_constant(%dst: !pto.ptr, gm>) attributes {pto.entry} { + pto.section.simt<<<32, 1, 1>>> { + %value = arith.constant dense<0.0> : vector<4xf8E4M3FN> + %c0 = arith.constant 0 : index + pto.stg %value, %dst[%c0] : !pto.ptr, gm>, vector<4xf8E4M3FN> + } + return + } +} + +//--- e5m2.pto +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @direct_e5m2_constant(%dst: !pto.ptr, gm>) attributes {pto.entry} { + pto.section.simt<<<32, 1, 1>>> { + %value = arith.constant dense<0.0> : vector<4xf8E5M2> + %c0 = arith.constant 0 : index + pto.stg %value, %dst[%c0] : !pto.ptr, gm>, vector<4xf8E5M2> + } + return + } +} diff --git a/test/lit/vpto/simt_fp8_local_contiguous_memory_vpto_llvm.pto b/test/lit/vpto/simt_fp8_local_contiguous_memory_vpto_llvm.pto new file mode 100644 index 0000000000..ab4503e0b7 --- /dev/null +++ b/test/lit/vpto/simt_fp8_local_contiguous_memory_vpto_llvm.pto @@ -0,0 +1,71 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s +// RUN: ptoas --cann-output-version=9.0.0 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @simt_fp8_local_contiguous_memory( + %src: !pto.ptr, + %dst_e4: !pto.ptr, + %dst_e5: !pto.ptr) attributes {pto.entry} { + pto.section.simt<<<32, 1, 1>>> { + %c8_i32 = arith.constant 8 : i32 + %local_e4 = llvm.alloca %c8_i32 x f8E4M3FN : (i32) -> !llvm.ptr + %local_e5 = llvm.alloca %c8_i32 x f8E5M2 : (i32) -> !llvm.ptr + + %src_addr = pto.castptr %src : !pto.ptr -> i64 + %src_ptr = llvm.inttoptr %src_addr : i64 to !llvm.ptr<6> + %src_pair = llvm.load %src_ptr : !llvm.ptr<6> -> vector<2xf32> + %e4_pair = pto.convert %src_pair round(r) sat : vector<2xf32> -> vector<2xf8E4M3FN> + %e5_pair = pto.convert %src_pair round(r) sat : vector<2xf32> -> vector<2xf8E5M2> + llvm.store %e4_pair, %local_e4 : vector<2xf8E4M3FN>, !llvm.ptr + llvm.store %e5_pair, %local_e5 : vector<2xf8E5M2>, !llvm.ptr + + %local_e4_second = llvm.getelementptr %local_e4[2] : (!llvm.ptr) -> !llvm.ptr, f8E4M3FN + %local_e5_second = llvm.getelementptr %local_e5[2] : (!llvm.ptr) -> !llvm.ptr, f8E5M2 + llvm.store %e4_pair, %local_e4_second : vector<2xf8E4M3FN>, !llvm.ptr + llvm.store %e5_pair, %local_e5_second : vector<2xf8E5M2>, !llvm.ptr + + %e4_quad = llvm.load %local_e4 : !llvm.ptr -> vector<4xf8E4M3FN> + %e5_quad = llvm.load %local_e5 : !llvm.ptr -> vector<4xf8E5M2> + %e4_oct = llvm.load %local_e4 : !llvm.ptr -> vector<8xf8E4M3FN> + %e5_oct = llvm.load %local_e5 : !llvm.ptr -> vector<8xf8E5M2> + + %dst_e4_addr = pto.castptr %dst_e4 : !pto.ptr -> i64 + %dst_e5_addr = pto.castptr %dst_e5 : !pto.ptr -> i64 + %dst_e4_ptr = llvm.inttoptr %dst_e4_addr : i64 to !llvm.ptr<6> + %dst_e5_ptr = llvm.inttoptr %dst_e5_addr : i64 to !llvm.ptr<6> + llvm.store %e4_quad, %dst_e4_ptr : vector<4xf8E4M3FN>, !llvm.ptr<6> + llvm.store %e5_quad, %dst_e5_ptr : vector<4xf8E5M2>, !llvm.ptr<6> + llvm.store %e4_oct, %dst_e4_ptr : vector<8xf8E4M3FN>, !llvm.ptr<6> + llvm.store %e5_oct, %dst_e5_ptr : vector<8xf8E5M2>, !llvm.ptr<6> + } + return + } +} + +// CHECK-LABEL: define linkonce_odr simt_entry void @simt_fp8_local_contiguous_memory_simt_0 +// CHECK: alloca float8e4m3, i32 8 +// CHECK: alloca float8e5m2, i32 8 +// CHECK: call <2 x float8e4m3> @llvm.hivm.f32x2.to.f8e4m3x2 +// CHECK: call <2 x float8e5m2> @llvm.hivm.f32x2.to.f8e5m2x2 +// CHECK: store <2 x float8e4m3> +// CHECK: store <2 x float8e5m2> +// CHECK: getelementptr float8e4m3, ptr {{.*}}, i32 2 +// CHECK: getelementptr float8e5m2, ptr {{.*}}, i32 2 +// CHECK: load <4 x float8e4m3> +// CHECK: load <4 x float8e5m2> +// CHECK: load <8 x float8e4m3> +// CHECK: load <8 x float8e5m2> +// CHECK: store <4 x float8e4m3> {{.*}} ptr addrspace(6) +// CHECK: store <4 x float8e5m2> {{.*}} ptr addrspace(6) +// CHECK: store <8 x float8e4m3> {{.*}} ptr addrspace(6) +// CHECK: store <8 x float8e5m2> {{.*}} ptr addrspace(6) +// CHECK-NOT: f8E4M3FN +// CHECK-NOT: f8E5M2