Adds versioned benchmarking (#4206)

Mpdreamz · russcam · commit 6da2b9379d14 · 2019-11-15T10:48:48.000+10:00
* Adds versioned benchmarking Allows the benchmarking project to reference Nest.v7 and so makes it easier for us to benchmark proposed changes. * fix compilation errors and temporarily disable dependency on #4202 (cherry picked from commit 16d8a08)
diff --git a/nuget.config b/nuget.config
@@ -3,6 +3,7 @@
 	<packageSources>
 		<add key="nuget.org" value="https://www.nuget.org/api/v2/" />
 		<add key="Elastic Abstractions CI" value="https://ci.appveyor.com/nuget/elasticsearch-net-abstractions" />
+		<add key="Versioned Clients" value="https://ci.appveyor.com/nuget/elasticsearch-net" />
 		<add key="AssemblyRewriter" value="https://ci.appveyor.com/nuget/assemblyrewriter" />
 		<add key="AssemblyDiffer" value="https://ci.appveyor.com/nuget/assemblydiffer" />
 	</packageSources>
diff --git a/src/Elasticsearch.Net/Providers/MemoryStreamFactory.cs b/src/Elasticsearch.Net/Providers/MemoryStreamFactory.cs
@@ -8,6 +8,8 @@ namespace Elasticsearch.Net
 	/// </summary>
 	public class MemoryStreamFactory : IMemoryStreamFactory
 	{
+		public static MemoryStreamFactory Default { get; } = new MemoryStreamFactory();
+
 		/// <inheritdoc />
 		public MemoryStream Create() => new MemoryStream();
 
diff --git a/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs b/src/Tests/Tests.Benchmarking/BenchmarkProgram.cs
@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Runtime.InteropServices;
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Diagnosers;
@@ -30,21 +31,21 @@ static Program()
 			if (!Directory.Exists(Path.Combine(dirInfo.FullName, ".git"))) Environment.Exit(2);
 
 			Console.WriteLine(dirInfo.FullName);
-			using (var repos = new Repository(dirInfo.FullName))
-			{
-				Commit = repos.Head.Tip.Sha;
-				CommitMessage = repos.Head.Tip.Message?.Trim(' ', '\t', '\r', '\n');
-				Branch = repos.Head.FriendlyName;
-				var remoteName = repos.Head.RemoteName;
-				Repository =
-					repos.Network.Remotes.FirstOrDefault(r => r.Name == remoteName)?.Url
-					?? repos.Network.Remotes.FirstOrDefault()?.Url;
-			}
+//			using (var repos = new Repository(dirInfo.FullName))
+//			{
+//				Commit = repos.Head.Tip.Sha;
+//				CommitMessage = repos.Head.Tip.Message?.Trim(' ', '\t', '\r', '\n');
+//				Branch = repos.Head.FriendlyName;
+//				var remoteName = repos.Head.RemoteName;
+//				Repository =
+//					repos.Network.Remotes.FirstOrDefault(r => r.Name == remoteName)?.Url
+//					?? repos.Network.Remotes.FirstOrDefault()?.Url;
+//			}
 		}
 
 		public static int Main(string[] arguments)
 		{
-			Console.WriteLine($"Tests.Benchmarking: [{Branch}]@({Commit}) on {Repository} : {CommitMessage} - ");
+			//Console.WriteLine($"Tests.Benchmarking: [{Branch}]@({Commit}) on {Repository} : {CommitMessage} - ");
 			var config = CreateDefaultConfig();
 			if (arguments.Any() && arguments[0].Equals("--all", StringComparison.OrdinalIgnoreCase))
 			{
@@ -61,14 +62,15 @@ public static int Main(string[] arguments)
 
 		private static IConfig CreateDefaultConfig()
 		{
-			var jobs = new[]
+			var jobs = new List<Job>
 			{
-				Job.ShortRun.With(Runtime.Core).With(Jit.RyuJit),
-				Job.ShortRun.With(Runtime.Clr).With(Jit.RyuJit),
-				Job.ShortRun.With(Runtime.Clr).With(Jit.LegacyJit),
+				Job.MediumRun.With(CoreRuntime.Core30).With(Jit.RyuJit),
 			};
+			if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+				jobs.Add(Job.MediumRun.With(ClrRuntime.Net472).With(Jit.LegacyJit));
+
 			var config = DefaultConfig.Instance
-				.With(jobs)
+				.With(jobs.ToArray())
 				.With(MemoryDiagnoser.Default);
 			return config;
 		}
diff --git a/src/Tests/Tests.Benchmarking/BulkBenchmarkTests.cs b/src/Tests/Tests.Benchmarking/BulkBenchmarkTests.cs
@@ -0,0 +1,78 @@
+using System.Collections.Generic;
+using System.Linq;
+using BenchmarkDotNet.Attributes;
+using Elasticsearch.Net;
+using Nest;
+using Tests.Benchmarking.Framework;
+using Tests.Core.Client;
+using Tests.Domain;
+
+namespace Tests.Benchmarking
+{
+	[BenchmarkConfig(5)]
+	public class BulkBenchmarkTests
+	{
+		private static readonly IList<Project> Projects = Project.Generator.Clone().Generate(10000);
+		private static readonly byte[] Response = TestClient.DefaultInMemoryClient.ConnectionSettings.RequestResponseSerializer.SerializeToBytes(ReturnBulkResponse(Projects));
+
+		private static readonly IElasticClient Client =
+			new ElasticClient(new ConnectionSettings(new InMemoryConnection(Response, 200, null, null))
+				.DefaultIndex("index")
+				.EnableHttpCompression(false)
+			);
+
+		//TODO can uncomment when https://github.com/elastic/elasticsearch-net/pull/4202 lands
+//		private static readonly IElasticClient ClientNoRecyclableMemory =
+//			new ElasticClient(new ConnectionSettings(new InMemoryConnection(Response, 200, null, null))
+//				.DefaultIndex("index")
+//				.EnableHttpCompression(false)
+//				.MemoryStreamFactory(MemoryStreamFactory.Default)
+//			);
+
+		private static readonly Nest7.IElasticClient ClientV7 =
+			new Nest7.ElasticClient(new Nest7.ConnectionSettings(
+					new Elasticsearch.Net7.InMemoryConnection(Response, 200, null, null))
+				.DefaultIndex("index")
+				.EnableHttpCompression(false)
+			);
+
+		[GlobalSetup]
+		public void Setup() { }
+
+		[Benchmark(Description = "PR")]
+		public BulkResponse NestUpdatedBulk() => Client.Bulk(b => b.IndexMany(Projects));
+
+//		[Benchmark(Description = "PR no recyclable")]
+//		public BulkResponse NoRecyclableMemory() => ClientNoRecyclableMemory.Bulk(b => b.IndexMany(Projects));
+
+		[Benchmark(Description = "7.x")]
+		public Nest7.BulkResponse NestCurrentBulk() => ClientV7.Bulk(b => b.IndexMany(Projects));
+
+		private static object BulkItemResponse(Project project) => new
+		{
+			index = new
+			{
+				_index = "nest-52cfd7aa",
+				_id = project.Name,
+				_version = 1,
+				_shards = new
+				{
+					total = 2,
+					successful = 1,
+					failed = 0
+				},
+				created = true,
+				status = 201
+			}
+		};
+
+		private static object ReturnBulkResponse(IList<Project> projects) => new
+		{
+			took = 276,
+			errors = false,
+			items = projects
+				.Select(p => BulkItemResponse(p))
+				.ToArray()
+		};
+	}
+}
diff --git a/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj b/src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj
@@ -3,6 +3,7 @@
   <PropertyGroup>
     <TargetFrameworks>netcoreapp3.0;net472</TargetFrameworks>
     <OutputType>Exe</OutputType>
+    <TieredCompilation>false</TieredCompilation>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
     <Optimize>true</Optimize>
@@ -11,8 +12,9 @@
     <ProjectReference Include="..\Tests.Core\Tests.Core.csproj" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.11.5" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.12.0" />
     <PackageReference Include="Elastic.BenchmarkDotNetExporter" Version="0.1.0-ci20191017T152836" />
     <PackageReference Include="LibGit2Sharp" Version="0.26.0-preview-0062" />
+    <PackageReference Include="NEST.v7" Version="7.5.0-ci20191031T180108" />
   </ItemGroup>
 </Project>
diff --git a/src/Tests/Tests.Core/Client/Settings/AlwaysInMemoryConnectionSettings.cs b/src/Tests/Tests.Core/Client/Settings/AlwaysInMemoryConnectionSettings.cs
@@ -13,6 +13,8 @@ public class AlwaysInMemoryConnectionSettings : TestConnectionSettings
 	{
 		public AlwaysInMemoryConnectionSettings() : base(forceInMemory: true) { }
 
+		public AlwaysInMemoryConnectionSettings(byte[] bytes) : base(forceInMemory: true, response: bytes) { }
+
 		public AlwaysInMemoryConnectionSettings(
 			Func<ICollection<Uri>, IConnectionPool> createPool = null,
 			SourceSerializerFactory sourceSerializerFactory = null,
@@ -23,7 +25,7 @@ public AlwaysInMemoryConnectionSettings(
 				createPool,
 				sourceSerializerFactory,
 				propertyMappingProvider,
-				true,
+				forceInMemory: true,
 				port
 			) { }
 	}
diff --git a/src/Tests/Tests.Core/Client/Settings/TestConnectionSettings.cs b/src/Tests/Tests.Core/Client/Settings/TestConnectionSettings.cs
@@ -21,11 +21,12 @@ public TestConnectionSettings(
 			SourceSerializerFactory sourceSerializerFactory = null,
 			IPropertyMappingProvider propertyMappingProvider = null,
 			bool forceInMemory = false,
-			int port = 9200
+			int port = 9200,
+			byte[] response = null
 		)
 			: base(
 				CreatePool(createPool, port),
-				TestConfiguration.Instance.CreateConnection(forceInMemory),
+				TestConfiguration.Instance.CreateConnection(forceInMemory, response),
 				CreateSerializerFactory(sourceSerializerFactory),
 				propertyMappingProvider
 			) =>
@@ -40,7 +41,7 @@ public TestConnectionSettings(
 
 		private static string LocalHost => "localhost";
 
-		private void ApplyTestSettings() => 
+		private void ApplyTestSettings() =>
 			RerouteToProxyIfNeeded()
 			.EnableDebugMode()
 			.EnableHttpCompression(TestConfiguration.Instance.Random.HttpCompression)
diff --git a/src/Tests/Tests.Core/Client/TestClient.cs b/src/Tests/Tests.Core/Client/TestClient.cs
@@ -11,6 +11,12 @@ public static class TestClient
 		public static readonly TestConfigurationBase Configuration = TestConfiguration.Instance;
 		public static readonly IElasticClient Default = new ElasticClient(new TestConnectionSettings().ApplyDomainSettings());
 		public static readonly IElasticClient DefaultInMemoryClient = new ElasticClient(new AlwaysInMemoryConnectionSettings().ApplyDomainSettings());
+		public static IElasticClient FixedInMemoryClient(byte[] response) => new ElasticClient(
+			new AlwaysInMemoryConnectionSettings(response)
+				.ApplyDomainSettings()
+				.DisableDirectStreaming()
+				.EnableHttpCompression(false)
+			);
 
 		public static readonly IElasticClient DisabledStreaming =
 			new ElasticClient(new TestConnectionSettings().ApplyDomainSettings().DisableDirectStreaming());
diff --git a/src/Tests/Tests.Core/Extensions/TestConfigurationExtensions.cs b/src/Tests/Tests.Core/Extensions/TestConfigurationExtensions.cs
@@ -7,8 +7,12 @@ namespace Tests.Core.Extensions
 {
 	public static class TestConfigurationExtensions
 	{
-		public static IConnection CreateConnection(this TestConfigurationBase configuration, bool forceInMemory = false) =>
-			configuration.RunIntegrationTests && !forceInMemory ? (IConnection)new HttpConnection() : new InMemoryConnection();
+		public static IConnection CreateConnection(this TestConfigurationBase configuration, bool forceInMemory = false, byte[] response = null) =>
+			forceInMemory
+				? new InMemoryConnection(response)
+				: configuration.RunIntegrationTests
+					? (IConnection)new HttpConnection()
+					: new InMemoryConnection(response);
 
 		public static bool InRange(this TestConfigurationBase configuration, string range) =>
 			configuration.ElasticsearchVersion.InRange(range);
diff --git a/src/Tests/Tests.Domain/Project.cs b/src/Tests/Tests.Domain/Project.cs
@@ -78,7 +78,7 @@ public class Project
 				.RuleFor(p => p.DateString, (p, d) => d.StartedOn.ToString("yyyy-MM-ddTHH\\:mm\\:ss.fffffffzzz"))
 				.RuleFor(p => p.LastActivity, p => p.Date.Recent())
 				.RuleFor(p => p.LeadDeveloper, p => Developer.Developers[Gimme.Random.Number(0, Developer.Developers.Count - 1)])
-				.RuleFor(p => p.Tags, f => Tag.Generator.Generate(Gimme.Random.Number(2, 50)))
+				.RuleFor(p => p.Tags, f => Tag.Generator.Generate(Gimme.Random.Number(2, 10)))
 				.RuleFor(p => p.CuratedTags, f => Tag.Generator.Generate(Gimme.Random.Number(1, 5)))
 				.RuleFor(p => p.LocationPoint, f => SimpleGeoPoint.Generator.Generate())
 				.RuleFor(p => p.LocationShape, f => new PointGeoShape(new GeoCoordinate(f.Address.Latitude(), f.Address.Latitude())))
diff --git a/src/Tests/Tests.ScratchPad/Program.cs b/src/Tests/Tests.ScratchPad/Program.cs
@@ -3,10 +3,13 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Running;
 using Elasticsearch.Net;
 using Elasticsearch.Net.Diagnostics;
 using Nest;
+using Tests.Core.Client;
+using Tests.Domain;
 using Xunit.Sdk;
 
 namespace Tests.ScratchPad
@@ -46,33 +49,60 @@ void WriteToConsole<T>(string eventName, T data)
 			}
 		}
 
-		private static async Task Main(string[] args)
-		{
-			DiagnosticListener.AllListeners.Subscribe(new ListenerObserver());
-
-			using (var node = new Elastic.Managed.Ephemeral.EphemeralCluster("7.0.0"))
-			{
-				node.Start();
+		private static readonly IList<Project> Projects = Project.Generator.Clone().Generate(10000);
+		private static readonly byte[] Response = TestClient.DefaultInMemoryClient.ConnectionSettings.RequestResponseSerializer.SerializeToBytes(ReturnBulkResponse(Projects));
 
-				var settings = new ConnectionSettings(new StaticConnectionPool(new[] { node.NodesUris("ipv4.fiddler").First() }))
-					.EnableHttpCompression()
-					.Proxy(new Uri("http://127.0.0.1:8080"), (string)null, (string)null)
-					;
-				var client = new ElasticClient(settings);
+		private static readonly IElasticClient Client =
+			new ElasticClient(new ConnectionSettings(new InMemoryConnection(Response, 200, null, null))
+				.DefaultIndex("index")
+				.EnableHttpCompression(false)
+			);
 
-				var x = client.Search<object>(s=>s.AllIndices());
 
-				await Task.Delay(TimeSpan.FromSeconds(7));
-
-				Console.WriteLine(new string('-', Console.WindowWidth - 1));
+		private static async Task Main(string[] args)
+		{
+			Console.Write($"Warmup...");
+			var response = Client.Bulk(b => b.IndexMany(Projects));
+			Console.WriteLine("\rWarmed up kicking off in 2 seconds!");
 
-				var y = client.Search<object>(s=>s.Index("does-not-exist"));
+			await Task.Delay(TimeSpan.FromSeconds(2));
+			Console.WriteLine($"Kicking off");
 
-				await Task.Delay(TimeSpan.FromSeconds(7));
+			for (var i = 0; i < 10_000; i++)
+			{
+				var r = Client.Bulk(b => b.IndexMany(Projects));
+				Console.Write($"\r{i}: {r.IsValid} {r.Items.Count}");
+			}
+		}
 
 
+		private static object BulkItemResponse(Project project) => new
+		{
+			index = new
+			{
+				_index = "nest-52cfd7aa",
+				_type = "_doc",
+				_id = project.Name,
+				_version = 1,
+				_shards = new
+				{
+					total = 2,
+					successful = 1,
+					failed = 0
+				},
+				created = true,
+				status = 201
 			}
-		}
+		};
+
+		private static object ReturnBulkResponse(IList<Project> projects) => new
+		{
+			took = 276,
+			errors = false,
+			items = projects
+				.Select(p => BulkItemResponse(p))
+				.ToArray()
+		};
 
 		private static void Bench<TBenchmark>() where TBenchmark : RunBase => BenchmarkRunner.Run<TBenchmark>();
 
diff --git a/src/Tests/Tests.ScratchPad/Tests.ScratchPad.csproj b/src/Tests/Tests.ScratchPad/Tests.ScratchPad.csproj
@@ -3,13 +3,12 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>netcoreapp3.0</TargetFramework>
-  </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
     <Optimize>true</Optimize>
+    <DebugSymbols>true</DebugSymbols>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="System.Diagnostics.DiagnosticSource" Version="4.5.1" />
     <ProjectReference Include="..\Tests.Core\Tests.Core.csproj" />
     <PackageReference Include="BenchMarkDotNet" Version="0.11.5" />
   </ItemGroup>
-</Project>
+</Project>
diff --git a/src/Tests/Tests/ClientConcepts/Connection/HttpConnectionTests.cs b/src/Tests/Tests/ClientConcepts/Connection/HttpConnectionTests.cs
@@ -160,10 +160,7 @@ public class TestableHttpConnection : HttpConnection
 			public int ClientCount => Clients.Count;
 			public HttpClientHandler LastHttpClientHandler => (HttpClientHandler)_handler.InnerHandler;
 
-			public TestableHttpConnection(Action<HttpResponseMessage> response)
-			{
-				_response = response;
-			}
+			public TestableHttpConnection(Action<HttpResponseMessage> response) => _response = response;
 
 			public TestableHttpConnection()
 			{

Original file line number	Diff line number	Diff line change
`@@ -8,6 +8,8 @@ namespace Elasticsearch.Net`
`8`	`8`	`/// </summary>`
`9`	`9`	`public class MemoryStreamFactory : IMemoryStreamFactory`
`10`	`10`	`{`
	`11`	`+ public static MemoryStreamFactory Default { get; } = new MemoryStreamFactory();`
	`12`	`+`
`11`	`13`	`/// <inheritdoc />`
`12`	`14`	`public MemoryStream Create() => new MemoryStream();`
`13`	`15`
Original file line number	Diff line number	Diff line change
`@@ -7,8 +7,12 @@ namespace Tests.Core.Extensions`
`7`	`7`	`{`
`8`	`8`	`public static class TestConfigurationExtensions`
`9`	`9`	`{`
`10`		`- public static IConnection CreateConnection(this TestConfigurationBase configuration, bool forceInMemory = false) =>`
`11`		`- configuration.RunIntegrationTests && !forceInMemory ? (IConnection)new HttpConnection() : new InMemoryConnection();`
	`10`	`+ public static IConnection CreateConnection(this TestConfigurationBase configuration, bool forceInMemory = false, byte[] response = null) =>`
	`11`	`+ forceInMemory`
	`12`	`+ ? new InMemoryConnection(response)`
	`13`	`+ : configuration.RunIntegrationTests`
	`14`	`+ ? (IConnection)new HttpConnection()`
	`15`	`+ : new InMemoryConnection(response);`
`12`	`16`
`13`	`17`	`public static bool InRange(this TestConfigurationBase configuration, string range) =>`
`14`	`18`	`configuration.ElasticsearchVersion.InRange(range);`