Feature proposal: support dynamic IL injection

Samples/DynamicILInjectionKernel/App.config

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+    <startup> 
+        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7"/>
+    </startup>
+</configuration>

Samples/DynamicILInjectionKernel/AssemblyAttributes.cs

@@ -0,0 +1,3 @@
+using System;
+
+[assembly: CLSCompliant(true)]

Samples/DynamicILInjectionKernel/DynamicILInjectionKernel.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFrameworks>$(LibrarySamplesTargetFrameworks)</TargetFrameworks>
+    <OutputType>Exe</OutputType>
+    <LangVersion>8.0</LangVersion>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <EnableNETAnalyzers>true</EnableNETAnalyzers>
+    <AnalysisMode>AllEnabledByDefault</AnalysisMode>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Src\ILGPU\ILGPU.csproj" />
+  </ItemGroup>
+</Project>

Samples/DynamicILInjectionKernel/Program.cs

@@ -0,0 +1,204 @@
+// ---------------------------------------------------------------------------------------
+//                                    ILGPU Samples
+//                           Copyright (c) 2021 ILGPU Project
+//                                    www.ilgpu.net
+//
+// File: Program.cs
+//
+// This file is part of ILGPU and is distributed under the University of Illinois Open
+// Source License. See LICENSE.txt for details.
+// ---------------------------------------------------------------------------------------
+
+#pragma warning disable CA1031
+using ILGPU;
+using ILGPU.Backends;
+using ILGPU.Backends.EntryPoints;
+using ILGPU.Frontend;
+using ILGPU.IR;
+using ILGPU.Runtime;
+using ILGPU.Runtime.Cuda;
+using System;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Reflection;
+using System.Reflection.Emit;
+using System.Runtime.CompilerServices;
+
+namespace SimpleKernel
+{
+    class Program
+    {
+        private static float[] GetInputs() =>
+            Enumerable.Range(0, 1024).Select(i => MathF.Tau * i / 1024f).ToArray();
+
+        /// <summary>
+        /// It's a simple method that calls the sine function but contains
+        /// internal branching for generic specialization. Because the method
+        /// uses the <see cref="System.Type"/> type and <c>throw</c> instructions,
+        /// it cannot be compiled by ILGPU's normal compilation process.
+        /// </summary>
+        /// <typeparam name="T">The type of input and output.</typeparam>
+        /// <param name="x">The input number.</param>
+        /// <returns>Sine of <paramref name="x"/>.</returns>
+        /// <exception cref="NotSupportedException"></exception>
+        static T GenericSin<T>(T x)
+            where T : unmanaged
+        {
+            if (typeof(T) == typeof(float))
+                return Unsafe.BitCast<float, T>(MathF.Sin(Unsafe.BitCast<T, float>(x)));
+            else if (typeof(T) == typeof(double))
+                return Unsafe.BitCast<double, T>(Math.Cos(Unsafe.BitCast<T, double>(x)));
+            else
+                throw new NotSupportedException();
+        }
+
+        /// <summary>
+        /// The first case that uses <see cref="GenericSin{T}(T)"/>. The method
+        /// itself is not generic, but it internally references the type-
+        /// parameterized <see cref="GenericSin{float}(float)"/>.
+        /// </summary>
+        /// <param name="index"></param>
+        /// <param name="dataView"></param>
+        static void MyKernel1(
+            Index1D index,
+            ArrayView<float> dataView)
+        {
+            dataView[index] = GenericSin(dataView[index]);
+        }
+
+        /// <summary>
+        /// The another case that uses <see cref="GenericSin{T}(T)"/>. The
+        /// method itself is generic, and it binds <see cref="GenericSin{T}(T)"/>
+        /// when this method is called.
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="index"></param>
+        /// <param name="dataView"></param>
+        static void MyKernel2<T>(
+            Index1D index,
+            ArrayView<T> dataView)
+            where T : unmanaged
+        {
+            dataView[index] = GenericSin(dataView[index]);
+        }
+
+        /// <summary>
+        /// Experiments invoking <see cref="MyKernel1(Index1D, ArrayView{float})"/> as ILGPU compiled kernel.
+        /// </summary>
+        /// <param name="accelerator"></param>
+        static void TestInvokeMyKernel1(Accelerator accelerator)
+        {
+            try
+            {
+                var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<float>>(MyKernel1);
+                using var buffer = accelerator.Allocate1D<float>(1024);
+                buffer.CopyFromCPU(GetInputs());
+                kernel((int)buffer.Length, buffer.View);
+                var data1 = buffer.GetAsArray1D();
+                Console.WriteLine($"[{string.Join(", ", data1.Take(10))}, ...]");
+            }
+            catch (Exception exc)
+            {
+                Console.WriteLine($"An exception occurred: {exc.Message}");
+            }
+        }
+
+        /// <summary>
+        /// Experiments invoking <see cref="MyKernel2{float}(Index1D, ArrayView{float})"/> as ILGPU compiled kernel.
+        /// </summary>
+        /// <param name="accelerator"></param>
+        static void TestInvokeMyKernel2(Accelerator accelerator)
+        {
+            try
+            {
+                var kernel = accelerator.LoadAutoGroupedStreamKernel<Index1D, ArrayView<float>>(MyKernel2);
+                using var buffer = accelerator.Allocate1D<float>(1024);
+                buffer.CopyFromCPU(GetInputs());
+                kernel((int)buffer.Length, buffer.View);
+                var data1 = buffer.GetAsArray1D();
+                Console.WriteLine($"[{string.Join(", ", data1.Take(10))}, ...]");
+            }
+            catch (Exception exc)
+            {
+                Console.WriteLine($"An exception occurred: {exc.Message}");
+            }
+        }
+
+
+        /// <summary>
+        /// Launches a simple 1D kernel.
+        /// </summary>
+        static void Main()
+        {
+            // Due to using `GenericSin<T>`, this operation will be failed.
+            Console.WriteLine("<<< without dynamic IL injection for `GenericSin<float>` >>>");
+            using (var context = Context.CreateDefault())
+            {
+                using var accelerator = context
+                    .GetPreferredDevice(false)
+                    .CreateAccelerator(context);
+                Console.WriteLine($"Performing operations on {accelerator}");
+
+                Console.Write("Results from MyKernel1: ");
+                TestInvokeMyKernel1(accelerator);
+            }
+            Console.WriteLine();
+
+            // Since `GenericSin<float>` has been replaced with IL that contains no compilation errors,
+            // `MyKernel1` and `MyKernel2` can be compiled and executed without errors.
+            Console.WriteLine("<<< with dynamic IL injection for `GenericSin<float>` >>>");
+            using (var context = Context.CreateDefault())
+            {
+                using var accelerator = context
+                    .GetPreferredDevice(false)
+                    .CreateAccelerator(context);
+                Console.WriteLine($"Performing operations on {accelerator}");
+
+                // Injects a dynamic IL implementation for `GenericSin<float>`.
+                // When the type parameter `T` is instantiated as `float`, the conditional branches
+                // can be removed and the method can be optimized to a direct call to `MathF.Sin(float)`.
+                // Therefore, we inject the precomputed optimized IL here.
+                using (var codeGenerationPhase = accelerator.GetBackend().Context.BeginCodeGeneration())
+                {
+                    var methodInfoGenericSin = typeof(Program).GetMethod(nameof(GenericSin), BindingFlags.NonPublic | BindingFlags.Static);
+                    var genericSinEPDesc = EntryPointDescription.FromExplicitlyGroupedKernel(methodInfoGenericSin.MakeGenericMethod(typeof(float)));
+                    var mainContext = codeGenerationPhase.IRContext;
+
+                    CodeGenerationResult generationResult;
+                    using (var frontendPhase = codeGenerationPhase.BeginFrontendCodeGeneration())
+                    {
+                        var methodInfoMathFSin = typeof(MathF).GetMethod(nameof(MathF.Sin), BindingFlags.Public | BindingFlags.Static);
+                        var disassembledMethod = new DisassembledMethod(
+                            genericSinEPDesc.MethodSource,
+                            ImmutableArray.Create<ILInstruction>(
+                                new ILInstruction(0, ILInstructionType.Ldarg, default, popCount: 0, pushCount: 1, argument: 0, Location.Unknown),
+                                new ILInstruction(1, ILInstructionType.Call, default, popCount: 1, pushCount: 1, argument: methodInfoMathFSin, Location.Unknown),
+                                new ILInstruction(6, ILInstructionType.Ret, default, popCount: 1, pushCount: 0, argument: OpCodes.Ret, Location.Unknown)
+                            ),
+                            8);
+                        generationResult = frontendPhase.GenerateCode(genericSinEPDesc.MethodSource, disassembledMethod);
+                    }
+
+                    if (codeGenerationPhase.IsFaulted)
+                    {
+                        throw codeGenerationPhase.LastException!;
+                    }
+
+                    var result = generationResult;
+                    if (!result.HasResult)
+                    {
+                        throw new InvalidOperationException();
+                    }
+                    codeGenerationPhase.Optimize();
+                }
+
+                Console.WriteLine("<<< after dynamic IL injection for `GenericSin<float>` >>>");
+                Console.Write("Results from MyKernel1: ");
+                TestInvokeMyKernel1(accelerator);
+
+                Console.Write("Results from MyKernel2: ");
+                TestInvokeMyKernel2(accelerator);
+            }
+        }
+    }
+}

Samples/ILGPU.Samples.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.1.32319.34
+# Visual Studio Version 18
+VisualStudioVersion = 18.0.11111.16 d18.0
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DeviceInfo", "DeviceInfo\DeviceInfo.csproj", "{715584CD-87DA-4F1F-86C3-A3D9AB50EE53}"
 EndProject
@@ -135,6 +135,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ILGPU.Analyzers", "..\Src\I
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "InterleaveFields", "InterleaveFields\InterleaveFields.csproj", "{1E6D0BC6-CFA1-4F52-9EB9-CAA62DD2F33A}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DynamicILInjectionKernel", "DynamicILInjectionKernel\DynamicILInjectionKernel.csproj", "{493A5D16-AE8D-5E0B-4CB1-4BA2F3A17099}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -381,6 +383,10 @@ Global
 		{1E6D0BC6-CFA1-4F52-9EB9-CAA62DD2F33A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{1E6D0BC6-CFA1-4F52-9EB9-CAA62DD2F33A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{1E6D0BC6-CFA1-4F52-9EB9-CAA62DD2F33A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{493A5D16-AE8D-5E0B-4CB1-4BA2F3A17099}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{493A5D16-AE8D-5E0B-4CB1-4BA2F3A17099}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{493A5D16-AE8D-5E0B-4CB1-4BA2F3A17099}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{493A5D16-AE8D-5E0B-4CB1-4BA2F3A17099}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -449,6 +455,7 @@ Global
 		{70B69CE3-24A9-463C-B14C-E2934988BBEE} = {25BA2234-5778-40BC-9386-9CE87AB87D1F}
 		{1C5E9E39-3C14-4B52-8D97-04555D5F6331} = {03FCC663-945D-4982-90D8-B14BE52D8FCD}
 		{1E6D0BC6-CFA1-4F52-9EB9-CAA62DD2F33A} = {C1D99632-ED4A-4B08-A14D-4C8DB375934F}
+		{493A5D16-AE8D-5E0B-4CB1-4BA2F3A17099} = {C1D99632-ED4A-4B08-A14D-4C8DB375934F}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {30E502BD-3826-417F-888F-1CE19CF5C6DA}

Src/ILGPU/Frontend/DisassembledMethod.cs

@@ -1,4 +1,4 @@
-// ---------------------------------------------------------------------------------------
+// ---------------------------------------------------------------------------------------
 //                                        ILGPU
 //                        Copyright (c) 2018-2021 ILGPU Project
 //                                    www.ilgpu.net
@@ -26,7 +26,16 @@ public sealed class DisassembledMethod
     {
         #region Instance
 
-        internal DisassembledMethod(
+        /// <summary>
+        /// Constructs a new IL instructions stream.
+        /// </summary>
+        /// <param name="method">The method who has been disassembled. Must not be null.</param>
+        /// <param name="instructions">
+        /// The collection of intermediate language (IL) instructions associated with the method.
+        /// Must not be null and must contain at least one instruction.
+        /// </param>
+        /// <param name="maxStackSize">The maximum stack size required by the method during execution.</param>
+        public DisassembledMethod(
             MethodBase method,
             ImmutableArray<ILInstruction> instructions,
             int maxStackSize)