Merge branch 'fix/2.x-sourcefilter' into 2.x

Author: russcam

src/Nest/Aggregations/Metric/TopHits/TopHitsAggregation.cs

@@ -91,7 +91,7 @@ public TopHitsAggregationDescriptor<T> Sort(Func<SortFieldDescriptor<T>, IFieldS
 		});
 
 		public TopHitsAggregationDescriptor<T> Source(bool include = true) =>
-			Assign(a => a.Source = !include ? SourceFilter.ExcludeAll : null);
+			Assign(a => a.Source = new SourceFilter { Disable = !include });
 
 		public TopHitsAggregationDescriptor<T> Source(Func<SourceFilterDescriptor<T>, ISourceFilter> sourceSelector) =>
 			Assign(a => a.Source = sourceSelector?.Invoke(new SourceFilterDescriptor<T>()));

src/Nest/CommonAbstractions/Infer/Fields/Fields.cs

@@ -20,6 +20,8 @@ string IUrlParameter.GetString(IConnectionConfigurationValues settings) =>
 
 		public static implicit operator Fields(string[] fields) => new Fields(fields.Select(f => (Field)f));
 
+		public static implicit operator Fields(string field) => new Fields(new[] { (Field)field });
+
 		public static implicit operator Fields(Expression[] fields) => new Fields(fields.Select(f => (Field)f));
 
 		public static implicit operator Fields(Field[] fields) => new Fields(fields);

src/Nest/Search/Search/InnerHits/GlobalInnerHit.cs

@@ -38,10 +38,10 @@ public class GlobalInnerHitDescriptor<T> : DescriptorBase<GlobalInnerHitDescript
 		IList<Field> IInnerHits.FielddataFields { get; set; }
 		IScriptFields IInnerHits.ScriptFields { get; set; }
 
-		public GlobalInnerHitDescriptor<T> Query(Func<QueryContainerDescriptor<T>, QueryContainer> querySelector) => 
+		public GlobalInnerHitDescriptor<T> Query(Func<QueryContainerDescriptor<T>, QueryContainer> querySelector) =>
 			Assign(a => a.Query = querySelector?.Invoke(new QueryContainerDescriptor<T>()));
-		
-		public GlobalInnerHitDescriptor<T> InnerHits(Func<NamedInnerHitsDescriptor<T>, IPromise<INamedInnerHits>> selector) => 
+
+		public GlobalInnerHitDescriptor<T> InnerHits(Func<NamedInnerHitsDescriptor<T>, IPromise<INamedInnerHits>> selector) =>
 			Assign(a => a.InnerHits = selector?.Invoke(new NamedInnerHitsDescriptor<T>())?.Value);
 
 		public GlobalInnerHitDescriptor<T> From(int? from) => Assign(a => a.From = from);
@@ -52,7 +52,7 @@ public GlobalInnerHitDescriptor<T> InnerHits(Func<NamedInnerHitsDescriptor<T>, I
 
 		public GlobalInnerHitDescriptor<T> FielddataFields(params string[] fielddataFields) =>
 			Assign(a => a.FielddataFields = fielddataFields?.Select(f => (Field) f).ToListOrNullIfEmpty());
-		
+
 		public GlobalInnerHitDescriptor<T> FielddataFields(params Expression<Func<T, object>>[] fielddataFields) =>
 			Assign(a => a.FielddataFields = fielddataFields?.Select(f => (Field) f).ToListOrNullIfEmpty());
 
@@ -63,17 +63,17 @@ public GlobalInnerHitDescriptor<T> FielddataFields(params Expression<Func<T, obj
 		public GlobalInnerHitDescriptor<T> Sort(Func<SortDescriptor<T>, IPromise<IList<ISort>>> sortSelector) => Assign(a => a.Sort = sortSelector?.Invoke(new SortDescriptor<T>())?.Value);
 
 		/// <summary>
-		/// Allow to highlight search results on one or more fields. The implementation uses the either lucene fast-vector-highlighter or highlighter. 
+		/// Allow to highlight search results on one or more fields. The implementation uses the either lucene fast-vector-highlighter or highlighter.
 		/// </summary>
 		public GlobalInnerHitDescriptor<T> Highlight(Func<HighlightDescriptor<T>, IHighlight> highlightSelector) =>
 			Assign(a => a.Highlight = highlightSelector?.Invoke(new HighlightDescriptor<T>()));
-		
-		public GlobalInnerHitDescriptor<T> Source(bool include = true)=> Assign(a => a.Source = !include ? SourceFilter.ExcludeAll : null);
-		
+
+		public GlobalInnerHitDescriptor<T> Source(bool include = true) => Assign(a => a.Source = new SourceFilter { Disable = !include });
+
 		public GlobalInnerHitDescriptor<T> Source(Func<SourceFilterDescriptor<T>, ISourceFilter> sourceSelector) =>
 			Assign(a => a.Source = sourceSelector?.Invoke(new SourceFilterDescriptor<T>()));
 
-		public GlobalInnerHitDescriptor<T> ScriptFields(Func<ScriptFieldsDescriptor, IPromise<IScriptFields>> selector) => 
+		public GlobalInnerHitDescriptor<T> ScriptFields(Func<ScriptFieldsDescriptor, IPromise<IScriptFields>> selector) =>
 			Assign(a => a.ScriptFields = selector?.Invoke(new ScriptFieldsDescriptor())?.Value);
 	}
-}
+}

src/Nest/Search/Search/InnerHits/InnerHits.cs

@@ -96,18 +96,18 @@ public InnerHitsDescriptor<T> FielddataFields(params Expression<Func<T, object>>
 		public InnerHitsDescriptor<T> Sort(Func<SortDescriptor<T>, IPromise<IList<ISort>>> sortSelector) => Assign(a => a.Sort = sortSelector?.Invoke(new SortDescriptor<T>())?.Value);
 
 		/// <summary>
-		/// Allow to highlight search results on one or more fields. The implementation uses the either lucene fast-vector-highlighter or highlighter. 
+		/// Allow to highlight search results on one or more fields. The implementation uses the either lucene fast-vector-highlighter or highlighter.
 		/// </summary>
 		public InnerHitsDescriptor<T> Highlight(Func<HighlightDescriptor<T>, IHighlight> highlightSelector) =>
 			Assign(a => a.Highlight = highlightSelector?.Invoke(new HighlightDescriptor<T>()));
 
 		//TODO map source of union bool/SourceFileter
-		public InnerHitsDescriptor<T> Source(bool include = true) => Assign(a => a.Source = !include ? SourceFilter.ExcludeAll : null);
+		public InnerHitsDescriptor<T> Source(bool include = true) => Assign(a => a.Source = new SourceFilter { Disable = !include });
 
 		public InnerHitsDescriptor<T> Source(Func<SourceFilterDescriptor<T>, ISourceFilter> sourceSelector) =>
 			Assign(a => a.Source = sourceSelector?.Invoke(new SourceFilterDescriptor<T>()));
 
-		public InnerHitsDescriptor<T> ScriptFields(Func<ScriptFieldsDescriptor, IPromise<IScriptFields>> selector) => 
+		public InnerHitsDescriptor<T> ScriptFields(Func<ScriptFieldsDescriptor, IPromise<IScriptFields>> selector) =>
 			Assign(a => a.ScriptFields = selector?.Invoke(new ScriptFieldsDescriptor())?.Value);
 	}
 }

src/Nest/Search/Search/SearchRequest.cs

@@ -64,7 +64,6 @@ public partial interface ISearchRequest : ICovariantSearchRequest
 		IScriptFields ScriptFields { get; set; }
 
 		[JsonProperty(PropertyName = "_source")]
-		[JsonConverter(typeof(ReadAsTypeJsonConverter<SourceFilter>))]
 		ISourceFilter Source { get; set; }
 
 		[JsonProperty(PropertyName = "aggs")]
@@ -224,7 +223,7 @@ public partial class SearchDescriptor<T> where T : class
 		public SearchDescriptor<T> Aggregations(Func<AggregationContainerDescriptor<T>, IAggregationContainer> aggregationsSelector) =>
 			Assign(a => a.Aggregations = aggregationsSelector(new AggregationContainerDescriptor<T>())?.Aggregations);
 
-		public SearchDescriptor<T> Source(bool include = true) => Assign(a => a.Source = !include ? SourceFilter.ExcludeAll : null);
+		public SearchDescriptor<T> Source(bool include = true) => Assign(a => a.Source = new SourceFilter { Disable = !include });
 
 		public SearchDescriptor<T> Source(Func<SourceFilterDescriptor<T>, ISourceFilter> sourceSelector) =>
 			Assign(a => a.Source = sourceSelector?.Invoke(new SourceFilterDescriptor<T>()));
@@ -416,8 +415,8 @@ public SearchDescriptor<T> Highlight(Func<HighlightDescriptor<T>, IHighlight> hi
 		public SearchDescriptor<T> Rescore(Func<RescoreDescriptor<T>, IRescore> rescoreSelector) =>
 			Assign(a =>
 			{
-				a.Rescore = a.Rescore != null 
-					? new MultiRescore { a.Rescore, rescoreSelector?.Invoke(new RescoreDescriptor<T>()) } 
+				a.Rescore = a.Rescore != null
+					? new MultiRescore { a.Rescore, rescoreSelector?.Invoke(new RescoreDescriptor<T>()) }
 					: rescoreSelector?.Invoke(new RescoreDescriptor<T>());
 			});
 

src/Nest/Search/Search/SourceFiltering/SourceFilter.cs

@@ -11,6 +11,9 @@ public interface ISourceFilter
 
 		[JsonProperty("exclude")]
 		Fields Exclude { get; set; }
+
+		[JsonIgnore]
+		bool Disable { get; set; }
 	}
 
 	public class SourceFilter : ISourceFilter
@@ -20,6 +23,8 @@ public class SourceFilter : ISourceFilter
 
 		public Fields Include { get; set; }
 		public Fields Exclude { get; set; }
+
+		public bool Disable { get; set; }
 	}
 
 	public class SourceFilterDescriptor<T> : DescriptorBase<SourceFilterDescriptor<T>, ISourceFilter>, ISourceFilter
@@ -29,11 +34,19 @@ public class SourceFilterDescriptor<T> : DescriptorBase<SourceFilterDescriptor<T
 
 		Fields ISourceFilter.Exclude { get; set; }
 
-		public SourceFilterDescriptor<T> Include(Func<FieldsDescriptor<T>, IPromise<Fields>> fields) => 
+		bool ISourceFilter.Disable { get; set; }
+
+
+		public SourceFilterDescriptor<T> Include(Func<FieldsDescriptor<T>, IPromise<Fields>> fields) =>
 			Assign(a => a.Include = fields?.Invoke(new FieldsDescriptor<T>())?.Value);
 
-		public SourceFilterDescriptor<T> Exclude(Func<FieldsDescriptor<T>, IPromise<Fields>> fields) => 
+		public SourceFilterDescriptor<T> IncludeAll() => Assign(a => a.Include = new[] { "*" });
+
+		public SourceFilterDescriptor<T> Exclude(Func<FieldsDescriptor<T>, IPromise<Fields>> fields) =>
 			Assign(a => a.Exclude = fields?.Invoke(new FieldsDescriptor<T>())?.Value);
 
+		public SourceFilterDescriptor<T> ExcludeAll() => Assign(a => a.Exclude = new[] { "*" });
+
+		public SourceFilterDescriptor<T> Disable(bool disable = true) => Assign(a => a.Disable = disable);
 	}
 }

src/Nest/Search/Search/SourceFiltering/SourceFilterJsonConverter.cs

@@ -7,25 +7,48 @@ namespace Nest
 	public class SourceFilterJsonConverter : JsonConverter
 	{
 		public override bool CanRead => true;
-		public override bool CanWrite => false;
+		public override bool CanWrite => true;
 		public override bool CanConvert(Type objectType) => true;
 
 		public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
 		{
+			var filter = value as ISourceFilter;
+			if (filter == null)
+				return;
+			if (filter.Disable)
+				writer.WriteValue(false);
+			else
+			{
+				if (filter.Include == null && filter.Exclude == null)
+					return;
+
+				writer.WriteStartObject();
+				if (filter.Include != null)
+				{
+					writer.WritePropertyName("include");
+					serializer.Serialize(writer, filter.Include);
+				}
+				if (filter.Exclude != null)
+				{
+					writer.WritePropertyName("exclude");
+					serializer.Serialize(writer, filter.Exclude);
+				}
+				writer.WriteEndObject();
+			}
 		}
 
 		public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
 		{
 			if (reader.TokenType == JsonToken.Null) return null;
 
-			var filter =  new SourceFilter();
+			var filter = new SourceFilter();
 			switch (reader.TokenType)
 			{
 				case JsonToken.Boolean:
-					filter.Exclude = new[] { "*" };
+					filter.Disable = !(bool)reader.Value;
 					break;
 				case JsonToken.String:
-					filter.Include = new [] { (string)reader.Value };
+					filter.Include = new[] { (string)reader.Value };
 					break;
 				case JsonToken.StartArray:
 					var include = new List<string>();
@@ -41,4 +64,4 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
 			return filter;
 		}
 	}
-}
+}

src/Tests/Search/Request/SourceFilteringUsageTests.cs

@@ -24,16 +24,17 @@ public SourceFilteringUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) :
 			{
 				_source = new
 				{
-					include = new[] { "name", "startedOn" }
+					include = new[] { "*" },
+					exclude = new [] { "description" }
 				}
 			};
 
 		protected override Func<SearchDescriptor<Project>, ISearchRequest> Fluent => s => s
-			.Source(so => so
-				.Include(f => f
+			.Source(src => src
+				.IncludeAll()
+				.Exclude(e => e
 					.Fields(
-						p => p.Name,
-						p => p.StartedOn
+						p => p.Description
 					)
 				)
 			);
@@ -43,7 +44,8 @@ public SourceFilteringUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) :
 			{
 				Source = new SourceFilter
 				{
-					Include = Fields<Project>(p => p.Name, prop => prop.StartedOn)
+					Include = "*",
+					Exclude = Fields<Project>(p => p.Description)
 				}
 			};
 
@@ -60,6 +62,36 @@ protected override void ExpectResponse(ISearchResponse<Project> response)
 		}
 	}
 
+	public class SourceFilteringDisabledUsageTests : SearchUsageTestBase
+	{
+		public SourceFilteringDisabledUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
+
+		protected override object ExpectJson =>
+			new
+			{
+				_source = false
+			};
+
+		protected override Func<SearchDescriptor<Project>, ISearchRequest> Fluent => s => s
+			.Source(src => src.Disable());
+
+		protected override SearchRequest<Project> Initializer =>
+			new SearchRequest<Project>
+			{
+				Source = new SourceFilter
+				{
+					Disable = true
+				}
+			};
+
+		protected override void ExpectResponse(ISearchResponse<Project> response)
+		{
+			response.ShouldBeValid();
+			foreach (var hit in response.Hits)
+				hit.Source.Should().BeNull();
+		}
+	}
+
 	//hide
 	public class SourceFilteringSerializationTests : SerializationTestBase
 	{
@@ -75,7 +107,9 @@ public void CanDeserializeBoolean()
 			var o = base.Deserialize<WithSourceFilterProperty>("{ \"_source\": false }");
 			o.Should().NotBeNull();
 			o.SourceFilter.Should().NotBeNull();
-			o.SourceFilter.Exclude.Should().Contain("*");
+			o.SourceFilter.Exclude.Should().BeNull();
+			o.SourceFilter.Include.Should().BeNull();
+			o.SourceFilter.Disable.Should().BeTrue();
 		}
 
 		[U]

src/Tests/tests.yaml

@@ -1,5 +1,5 @@
 # mode either u (unit test), i (integration test) or m (mixed mode)
-mode: m
+mode: u
 # the elasticsearch version that should be started
 elasticsearch_version: 2.3.5
 # whether we want to forcefully reseed on the node, if you are starting the tests with a node already running