[data views] Allow data views created on hidden and system indices - second attempt (#168882)

## Summary

Previously, the 'Allow hidden and system indices' advanced option when
creating a data view was only a UI convenience. It allowed you to see
which hidden and system indices you were matching but they would be
would be selected just the same once the data view was loaded. At some
point something changed and now there are system and hidden indices that
require `expandWildcards: hidden` to be passed to field caps in order to
see anything. `allowHidden: boolean` is added to the DataView and
DataViewSpec and passed through when field caps requests are made.

This is primarily a tool for troubleshooting. For instance, instead of
hitting a full data stream across a number of data tiers you can select
a specific index to compare its performance.

NOTE: This is a second attempt. What I learned - the whole
`expand_wildcards` param is literal - you can directly query a hidden
index without `expandWildcards: hidden` since its not using a wildcard.
Tests now use a wildcard.

Closes: #164652

---------

Co-authored-by: Lukas Olson <[email protected]>
Co-authored-by: Kibana Machine <[email protected]>

Author: 3 people

src/plugins/data/common/search/search_source/fetch/get_search_params.test.ts

@@ -9,6 +9,7 @@
 import { UI_SETTINGS } from '../../../constants';
 import { GetConfigFn } from '../../../types';
 import { getSearchParams, getSearchParamsFromRequest } from './get_search_params';
+import { createStubDataView } from '@kbn/data-views-plugin/common/data_views/data_view.stub';
 
 function getConfigStub(config: any = {}): GetConfigFn {
   return (key) => config[key];
@@ -46,4 +47,50 @@ describe('getSearchParams', () => {
       query: 123,
     });
   });
+
+  test('sets expand_wildcards=all if data view has allowHidden=true', () => {
+    const getConfig = getConfigStub({
+      [UI_SETTINGS.COURIER_SET_REQUEST_PREFERENCE]: 'custom',
+      [UI_SETTINGS.COURIER_CUSTOM_REQUEST_PREFERENCE]: 'aaa',
+    });
+    const index = createStubDataView({
+      spec: {
+        allowHidden: true,
+      },
+    });
+    const searchParams = getSearchParamsFromRequest(
+      {
+        index,
+        body: {
+          query: 123,
+          track_total_hits: true,
+        },
+      },
+      { getConfig }
+    );
+    expect(searchParams).toHaveProperty('expand_wildcards', 'all');
+  });
+
+  test('does not set expand_wildcards if data view has allowHidden=false', () => {
+    const getConfig = getConfigStub({
+      [UI_SETTINGS.COURIER_SET_REQUEST_PREFERENCE]: 'custom',
+      [UI_SETTINGS.COURIER_CUSTOM_REQUEST_PREFERENCE]: 'aaa',
+    });
+    const index = createStubDataView({
+      spec: {
+        allowHidden: false,
+      },
+    });
+    const searchParams = getSearchParamsFromRequest(
+      {
+        index,
+        body: {
+          query: 123,
+          track_total_hits: true,
+        },
+      },
+      { getConfig }
+    );
+    expect(searchParams).not.toHaveProperty('expand_wildcards', 'all');
+  });
 });

src/plugins/data/common/search/search_source/fetch/get_search_params.ts

@@ -42,8 +42,8 @@ export function getSearchParamsFromRequest(
   return {
     index: searchRequest.index.title || searchRequest.index,
     body,
-    // @ts-expect-error `track_total_hits` not allowed at top level for `typesWithBodyKey`
     track_total_hits,
+    ...(searchRequest.index?.allowHidden && { expand_wildcards: 'all' }),
     ...searchParams,
   };
 }

src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_content.tsx

@@ -30,14 +30,16 @@ interface AdvancedParamsContentProps {
   disableAllowHidden: boolean;
   disableId: boolean;
   onAllowHiddenChange?: (value: boolean) => void;
+  defaultVisible?: boolean;
 }
 
 export const AdvancedParamsContent = ({
   disableAllowHidden,
   disableId,
   onAllowHiddenChange,
+  defaultVisible = false,
 }: AdvancedParamsContentProps) => (
-  <AdvancedParamsSection>
+  <AdvancedParamsSection defaultVisible={defaultVisible}>
     <EuiFlexGroup>
       <EuiFlexItem>
         <UseField<boolean, IndexPatternConfig>

src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_section.tsx

@@ -13,10 +13,11 @@ import { EuiButtonEmpty, EuiSpacer } from '@elastic/eui';
 
 interface Props {
   children: React.ReactNode;
+  defaultVisible: boolean;
 }
 
-export const AdvancedParamsSection = ({ children }: Props) => {
-  const [isVisible, setIsVisible] = useState<boolean>(false);
+export const AdvancedParamsSection = ({ children, defaultVisible = false }: Props) => {
+  const [isVisible, setIsVisible] = useState<boolean>(defaultVisible);
 
   const toggleIsVisible = useCallback(() => {
     setIsVisible(!isVisible);

src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx

@@ -105,6 +105,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
             title: editData.getIndexPattern(),
             id: editData.id,
             name: editData.name,
+            allowHidden: editData.getAllowHidden(),
             ...(editData.timeFieldName
               ? {
                   timestampField: { label: editData.timeFieldName, value: editData.timeFieldName },
@@ -124,6 +125,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
         timeFieldName: formData.timestampField?.value,
         id: formData.id,
         name: formData.name,
+        allowHidden: formData.allowHidden,
       };
 
       if (type === INDEX_PATTERN_TYPE.ROLLUP && rollupIndex) {
@@ -293,6 +295,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
             onAllowHiddenChange={() => {
               form.getFields().title.validate();
             }}
+            defaultVisible={editData?.getAllowHidden()}
           />
         </Form>
         <Footer

src/plugins/data_view_editor/public/components/data_view_flyout_content_container.tsx

@@ -50,10 +50,11 @@ const DataViewFlyoutContentContainer = ({
     try {
       let saveResponse;
       if (editData) {
-        const { name = '', timeFieldName, title = '' } = dataViewSpec;
+        const { name = '', timeFieldName, title = '', allowHidden = false } = dataViewSpec;
         editData.setIndexPattern(title);
         editData.name = name;
         editData.timeFieldName = timeFieldName;
+        editData.setAllowHidden(allowHidden);
         saveResponse = editData.isPersisted()
           ? await dataViews.updateSavedObject(editData)
           : editData;

src/plugins/data_view_editor/public/data_view_editor_service.ts

@@ -342,6 +342,7 @@ export class DataViewEditorService {
 
     const getFieldsOptions: GetFieldsOptions = {
       pattern: this.indexPattern,
+      allowHidden: this.allowHidden,
     };
     if (this.type === INDEX_PATTERN_TYPE.ROLLUP) {
       getFieldsOptions.type = INDEX_PATTERN_TYPE.ROLLUP;

src/plugins/data_views/common/content_management/v1/cm_services.ts

@@ -46,6 +46,7 @@ const dataViewAttributesSchema = schema.object(
     allowNoIndex: schema.maybe(schema.boolean()),
     runtimeFieldMap: schema.maybe(schema.any()),
     name: schema.maybe(schema.string()),
+    allowHidden: schema.maybe(schema.boolean()),
   },
   { unknowns: 'forbid' }
 );