Use Kibana data plugin and search strategy

src/plugins/profiling/public/app.tsx

@@ -33,19 +33,23 @@ import { Services } from './services';
 
 type Props = Services;
 
-function App({ fetchTopN, fetchElasticFlamechart, fetchPixiFlamechart }: Props) {
+function App({ fetchTopN, fetchElasticFlamechart, fetchPixiFlamechart, fetchTopNData }: Props) {
   const [topn, setTopN] = useState({
     samples: [],
     series: new Map(),
   });
+  const [topnData, setTopNData] = useState({
+    samples: [],
+    series: new Map(),
+  });
 
   const [elasticFlamegraph, setElasticFlamegraph] = useState({ leaves: [] });
   const [pixiFlamegraph, setPixiFlamegraph] = useState({});
 
   const tabs = [
     {
       id: 'stacktrace-elastic',
-      name: 'Stack Traces (Elastic)',
+      name: 'Stack Traces (API)',
       content: (
         <>
           <EuiSpacer />
@@ -57,6 +61,20 @@ function App({ fetchTopN, fetchElasticFlamechart, fetchPixiFlamechart }: Props)
         </>
       ),
     },
+    {
+      id: 'stacktrace-elastic-data',
+      name: 'Stack Traces (Data Plugin)',
+      content: (
+        <>
+          <EuiSpacer />
+          <TopNContext.Provider value={topnData}>
+            <StackTraceNavigation fetchTopN={fetchTopNData} setTopN={setTopNData} />
+            <StackedBarChart id="topn-data" name="topn-data" height={400} x="x" y="y" category="g" />
+            <ChartGrid maximum={10} />
+          </TopNContext.Provider>
+        </>
+      ),
+    },
     {
       id: 'flamegraph-elastic',
       name: 'FlameGraph (Elastic)',

src/plugins/profiling/public/components/contexts/topn.tsx

@@ -8,4 +8,4 @@
 
 import { createContext } from 'react';
 
-export const TopNContext = createContext();
+export const TopNContext = createContext({});

src/plugins/profiling/public/components/stacktrace-nav.tsx

@@ -101,13 +101,19 @@ export const StackTraceNavigation = ({ fetchTopN, setTopN }) => {
     });
 
     console.log(new Date().toISOString(), 'started payload retrieval');
-    fetchTopN(topnValue[0].value, dateValue[0].value).then((response) => {
-      console.log(new Date().toISOString(), 'finished payload retrieval');
-      const samples = getTopN(response);
-      const series = groupSamplesByCategory(samples);
-      setTopN({ samples, series });
-      console.log(new Date().toISOString(), 'updated local state');
-    });
+    fetchTopN(topnValue[0].value, dateValue[0].value)
+      .then((response) => {
+        console.log(new Date().toISOString(), 'finished payload retrieval');
+        const samples = getTopN(response);
+        const series = groupSamplesByCategory(samples);
+        console.log('sample %o', samples);
+        console.log('series %o', series);
+        setTopN({ samples, series });
+        console.log(new Date().toISOString(), 'updated local state');
+      })
+      .catch((err) => {
+        console.log('error when reading topN data: ' + err.message);
+      });
   }, [toggleTopNSelected, toggleDateSelected]);
 
   return (

src/plugins/profiling/public/plugin.tsx

@@ -6,18 +6,29 @@
  * Side Public License, v 1.
  */
 
-import { AppMountParameters, CoreSetup, CoreStart, Plugin } from '../../../core/public';
+import { AppMountParameters, CoreSetup, CoreStart, Plugin } from 'kibana/public';
 import { getServices } from './services';
+import { DataPublicPluginSetup, DataPublicPluginStart } from '../../data/public';
 
-export class ProdfilerPlugin implements Plugin {
-  public setup(core: CoreSetup) {
+export interface ProdfilerPluginStartDeps {
+  data: DataPublicPluginStart;
+}
+
+export interface ProdfilerPluginSetupDeps {
+  data: DataPublicPluginSetup;
+}
+
+export class ProdfilerPlugin
+  implements Plugin<void, void, ProdfilerPluginSetupDeps, ProdfilerPluginStartDeps>
+{
+  public setup(core: CoreSetup<ProdfilerPluginStartDeps>) {
     // Register an application into the side navigation menu
     core.application.register({
       id: 'prodfiler',
       title: 'Prodfiler',
       async mount({ element }: AppMountParameters) {
-        const [coreStart] = await core.getStartServices();
-        const startServices = getServices(coreStart);
+        const [coreStart, dataPlugin] = await core.getStartServices();
+        const startServices = getServices(coreStart, dataPlugin);
         const { renderApp } = await import('./app');
         return renderApp(startServices, element);
       },

src/plugins/profiling/public/services.ts

@@ -8,11 +8,19 @@
 
 import { CoreStart, HttpFetchError, HttpFetchQuery } from 'kibana/public';
 import { getRemoteRoutePaths } from '../common';
+import { ProdfilerPluginStartDeps } from './plugin';
+import {
+  DOWNSAMPLED_TOPN_STRATEGY,
+  DownsampledRequest,
+  DownsampledTopNResponse,
+  TopNAggregateResponse,
+} from '../common/types';
 
 export interface Services {
   fetchTopN: (type: string, seconds: string) => Promise<any[] | HttpFetchError>;
   fetchElasticFlamechart: (seconds: string) => Promise<any[] | HttpFetchError>;
   fetchPixiFlamechart: (seconds: string) => Promise<any[] | HttpFetchError>;
+  fetchTopNData: (searchField: string, seconds: string) => Promise<TopNAggregateResponse>;
 }
 
 function getFetchQuery(seconds: string): HttpFetchQuery {
@@ -27,11 +35,51 @@ function getFetchQuery(seconds: string): HttpFetchQuery {
   } as HttpFetchQuery;
 }
 
-export function getServices(core: CoreStart): Services {
+export function getServices(core: CoreStart, data?: ProdfilerPluginStartDeps): Services {
   // To use local fixtures instead, use getLocalRoutePaths
   const paths = getRemoteRoutePaths();
 
   return {
+    fetchTopNData: async (searchField: string, seconds: string): Promise<TopNAggregateResponse> => {
+      const unixTime = Math.floor(Date.now() / 1000);
+      const response: TopNAggregateResponse = { topN: { histogram: { buckets: [] } } };
+      data!.data.search
+        .search<DownsampledRequest, DownsampledTopNResponse>(
+          {
+            params: {
+              projectID: 5,
+              timeFrom: unixTime - parseInt(seconds, 10),
+              timeTo: unixTime,
+              // FIXME remove hard-coded value for topN items length and expose it through the UI
+              topNItems: 100,
+              searchField,
+            },
+          },
+          {
+            strategy: DOWNSAMPLED_TOPN_STRATEGY,
+          }
+        )
+        .subscribe({
+          next: (result) => {
+            console.log('subscription data plugin rawResponse: %o', result.rawResponse);
+            response.topN.histogram = result.rawResponse.aggregations.histogram;
+          },
+          // TODO error handling
+          error: (err) => {
+            console.log('subscription error: %o', err);
+          },
+          // FIXME remove this, used for debugging only
+          complete: () => {
+            console.log('subscription completed');
+          },
+        });
+
+      console.log('returning Promise of TopNAggregateResponse');
+      return await new Promise<TopNAggregateResponse>((resolve, _) => {
+        return resolve(response);
+      });
+    },
+
     fetchTopN: async (type: string, seconds: string) => {
       try {
         const query = getFetchQuery(seconds);

src/plugins/profiling/server/plugin.ts

@@ -5,12 +5,14 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-import { PluginInitializerContext, CoreSetup, CoreStart, Plugin, Logger } from 'kibana/server';
+import { CoreSetup, CoreStart, Logger, Plugin, PluginInitializerContext } from 'kibana/server';
 
 import type { DataRequestHandlerContext } from '../../data/server';
 
 import { ProfilingPluginSetupDeps, ProfilingPluginStartDeps } from './types';
 import { registerRoutes } from './routes';
+import { DownsampledTopNFactory } from './search/strategy';
+import { DOWNSAMPLED_TOPN_STRATEGY } from '../common/types';
 
 export class ProfilingPlugin
   implements Plugin<void, void, ProfilingPluginSetupDeps, ProfilingPluginStartDeps>
@@ -21,14 +23,18 @@ export class ProfilingPlugin
     this.logger = initializerContext.logger.get();
   }
 
-  public setup(core: CoreSetup<ProfilingPluginStartDeps>, { data }: ProfilingPluginSetupDeps) {
+  public setup(core: CoreSetup<ProfilingPluginStartDeps>, deps: ProfilingPluginSetupDeps) {
     this.logger.debug('profiling: Setup');
     // TODO we should create a query here using "data".
     // We should ensure there are profiling data in the expected indices
     // and return an error otherwise.
     // This should be done only once at startup and before exposing the routed APIs.
     const router = core.http.createRouter<DataRequestHandlerContext>();
-    core.getStartServices().then(() => {
+    core.getStartServices().then(([_, depsStart]) => {
+      deps.data.search.registerSearchStrategy(
+        DOWNSAMPLED_TOPN_STRATEGY,
+        DownsampledTopNFactory(depsStart.data)
+      );
       registerRoutes(router, this.logger);
     });
 

src/plugins/profiling/server/search/strategy.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types';
+import { IEsSearchRequest, ISearchStrategy, PluginStart } from '../../../data/server';
+import {
+  autoHistogramSumCountOnGroupByField,
+  newProjectTimeQuery,
+  ProjectTimeQuery,
+} from '../routes/mappings';
+import { downsampledIndex, getSampledTraceEventsIndex } from '../routes/search_flamechart';
+import { DownsampledRequest, DownsampledTopNResponse } from '../../common/types';
+
+export const DownsampledTopNFactory = (
+  data: PluginStart
+): ISearchStrategy<DownsampledRequest, DownsampledTopNResponse> => {
+  const es = data.search.getSearchStrategy();
+
+  // FIXME these 2 constants should be configurable?
+  const initialExp = 6;
+  const targetSampleSize = 20000; // minimum number of samples to get statistically sound results
+
+  // Calculate the right down-sampled index to query data from
+  const sampleCountFromInitialExp = (filter: ProjectTimeQuery, options, deps): number => {
+    // By default, we return no samples and use the un-sampled index
+    let sampleCount = 0;
+    es.search(
+      {
+        params: {
+          index: downsampledIndex + initialExp,
+          body: {
+            query: filter,
+            size: 0,
+            track_total_hits: true,
+          },
+        },
+      },
+      options,
+      deps
+    ).subscribe({
+      next: (value) => {
+        sampleCount = (value.rawResponse.hits.total as SearchTotalHits).value;
+      },
+    });
+    return sampleCount;
+  };
+  return {
+    search: (request, options, deps) => {
+      const { projectID, timeFrom, timeTo, topNItems, searchField } = request.params!;
+      const filter = newProjectTimeQuery(
+        projectID.toString(),
+        timeFrom.toString(),
+        timeTo.toString()
+      );
+
+      // Create the query for the actual data
+      const downsampledReq = {
+        params: {
+          index: getSampledTraceEventsIndex(
+            targetSampleSize,
+            sampleCountFromInitialExp(filter, options, deps),
+            initialExp
+          ).name,
+          body: {
+            query: filter,
+            aggs: {
+              histogram: autoHistogramSumCountOnGroupByField(searchField, topNItems),
+            },
+          },
+        },
+      } as IEsSearchRequest;
+      return es.search(downsampledReq, options, deps);
+    },
+    cancel: async (id, options, deps) => {
+      if (es.cancel) {
+        await es.cancel(id, options, deps);
+      }
+    },
+  };
+};