Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 55 additions & 25 deletions querybook/server/datasources_socketio/ai_assistant.py
Original file line number Diff line number Diff line change
@@ -1,49 +1,79 @@
from contextlib import contextmanager

from app.flask_app import socketio
from const.ai_assistant import AI_ASSISTANT_NAMESPACE, AICommandType
from lib.ai_assistant import ai_assistant
from lib.ai_assistant.ai_socket import AIWebSocket
from lib.logger import get_logger

from .helper import register_socket

LOG = get_logger(__file__)


@contextmanager
def ai_socket(command_type: AICommandType, payload: dict):
socket = AIWebSocket(socketio, command_type, payload.get("request_id"))
try:
yield socket
except Exception as e:
LOG.error(f"AI command {command_type.value} failed: {e}")
socket.send_error(str(e))


@register_socket(AICommandType.TEXT_TO_SQL.value, namespace=AI_ASSISTANT_NAMESPACE)
def text_to_sql(payload={}):
original_query = payload["original_query"]
query_engine_id = payload["query_engine_id"]
tables = payload.get("tables", [])
question = payload["question"]
ai_assistant.generate_sql_query(
query_engine_id=query_engine_id,
tables=tables,
question=question,
original_query=original_query,
)
with ai_socket(AICommandType.TEXT_TO_SQL, payload) as socket:
original_query = payload["original_query"]
query_engine_id = payload["query_engine_id"]
tables = payload.get("tables", [])
question = payload["question"]
ai_assistant.generate_sql_query(
query_engine_id=query_engine_id,
tables=tables,
question=question,
original_query=original_query,
socket=socket,
)


@register_socket(AICommandType.SQL_TITLE.value, namespace=AI_ASSISTANT_NAMESPACE)
def sql_title(payload={}):
query = payload["query"]
ai_assistant.generate_title_from_query(query=query)
with ai_socket(AICommandType.SQL_TITLE, payload) as socket:
query = payload["query"]
ai_assistant.generate_title_from_query(query=query, socket=socket)


@register_socket(AICommandType.SQL_FIX.value, namespace=AI_ASSISTANT_NAMESPACE)
def sql_fix(payload={}):
query_execution_id = payload["query_execution_id"]
ai_assistant.query_auto_fix(
query_execution_id=query_execution_id,
)
with ai_socket(AICommandType.SQL_FIX, payload) as socket:
query_execution_id = payload["query_execution_id"]
ai_assistant.query_auto_fix(
query_execution_id=query_execution_id,
socket=socket,
)


@register_socket(AICommandType.SQL_COMPLETE.value, namespace=AI_ASSISTANT_NAMESPACE)
def sql_complete(payload={}):
prefix = payload["prefix"]
suffix = payload["suffix"]
query_engine_id = payload["query_engine_id"]
tables = payload.get("tables", [])
ai_assistant.get_sql_completion(
query_engine_id=query_engine_id, tables=tables, prefix=prefix, suffix=suffix
)
with ai_socket(AICommandType.SQL_COMPLETE, payload) as socket:
prefix = payload["prefix"]
suffix = payload["suffix"]
query_engine_id = payload["query_engine_id"]
tables = payload.get("tables", [])
ai_assistant.get_sql_completion(
query_engine_id=query_engine_id,
tables=tables,
prefix=prefix,
suffix=suffix,
socket=socket,
)


@register_socket(AICommandType.DATA_DOC_TITLE.value, namespace=AI_ASSISTANT_NAMESPACE)
def data_doc_data_doc_title(payload={}):
cell_contents = payload["cell_contents"]
ai_assistant.generate_data_doc_title_from_query(cell_contents=cell_contents)
with ai_socket(AICommandType.DATA_DOC_TITLE, payload) as socket:
cell_contents = payload["cell_contents"]
ai_assistant.generate_data_doc_title_from_query(
cell_contents=cell_contents, socket=socket
)
7 changes: 6 additions & 1 deletion querybook/server/lib/ai_assistant/ai_socket.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import functools
from typing import Optional

from flask import request

Expand All @@ -7,9 +8,12 @@


class AIWebSocket:
def __init__(self, socketio, command_type: AICommandType):
def __init__(
self, socketio, command_type: AICommandType, request_id: Optional[str] = None
):
self.socketio = socketio
self.command_type = command_type
self.request_id = request_id
self.room = request.sid

def _send(self, event_type, payload: dict = None):
Expand All @@ -18,6 +22,7 @@ def _send(self, event_type, payload: dict = None):
(
event_type,
payload,
self.request_id,
),
namespace=AI_ASSISTANT_NAMESPACE,
room=self.room,
Expand Down
60 changes: 60 additions & 0 deletions querybook/tests/test_logic/test_ai_assistant_datasource.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
from types import SimpleNamespace
from unittest.mock import MagicMock

import pytest

from app.flask_app import socketio
from const.ai_assistant import AI_ASSISTANT_NAMESPACE
from datasources_socketio import ai_assistant as ai_assistant_ds
from lib.ai_assistant import ai_socket


@pytest.fixture
def emitted(monkeypatch):
calls = []
monkeypatch.setattr(ai_socket, "request", SimpleNamespace(sid="sid-1"))
monkeypatch.setattr(
socketio,
"emit",
lambda event, args, **kwargs: calls.append((event, args, kwargs)),
)
return calls


@pytest.fixture
def fake_assistant(monkeypatch):
assistant = MagicMock()
monkeypatch.setattr(ai_assistant_ds, "ai_assistant", assistant)
return assistant


def sql_title(payload):
return ai_assistant_ds.sql_title.__wrapped__(payload)


def test_events_are_tagged_with_request_id(emitted, fake_assistant):
def generate(query, socket):
socket.send_data({"title": "Daily users"})
socket.close()

fake_assistant.generate_title_from_query.side_effect = generate

sql_title({"query": "SELECT 1", "request_id": "ai_request_1"})

room = {"namespace": AI_ASSISTANT_NAMESPACE, "room": "sid-1"}
assert emitted == [
("sql_title", ("data", {"title": "Daily users"}, "ai_request_1"), room),
("sql_title", ("close", None, "ai_request_1"), room),
]


def test_failure_sends_error_then_close_on_command_channel(emitted, fake_assistant):
fake_assistant.generate_title_from_query.side_effect = Exception("LLM timeout")

sql_title({"query": "SELECT 1", "request_id": "ai_request_2"})

assert [args for _, args, _ in emitted] == [
("error", "LLM timeout", "ai_request_2"),
("close", None, "ai_request_2"),
]
assert {event for event, _, _ in emitted} == {"sql_title"}
128 changes: 128 additions & 0 deletions querybook/webapp/__tests__/hooks/useAISocket.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
import { EventEmitter } from 'events';
import React from 'react';
import toast from 'react-hot-toast';
import { act, create } from 'react-test-renderer';

import { AICommandType, AISocketEvent } from 'const/aiAssistant';
import { AISocket, useAISocket } from 'hooks/useAISocket';

const mockSocket = new EventEmitter();
const mockEmit = jest.fn();

jest.mock('lib/ai-assistant/ai-assistant-socketio', () => ({
__esModule: true,
default: {
emit: (...args) => mockEmit(...args),
addListener: (event, listener) => mockSocket.on(event, listener),
removeListener: (event, listener) => mockSocket.off(event, listener),
},
}));

jest.mock('react-hot-toast', () => ({
__esModule: true,
default: { error: jest.fn() },
}));

const command = AICommandType.TEXT_TO_SQL;

function renderHook(onData: jest.Mock) {
const result: { current: AISocket } = { current: null };
const Harness = ({ onData }) => {
result.current = useAISocket(command, (data) => onData(data));
return null;
};

let renderer;
act(() => {
renderer = create(<Harness onData={onData} />);
});

return {
result,
unmount: () => act(() => renderer.unmount()),
rerender: (onData: jest.Mock) =>
act(() => renderer.update(<Harness onData={onData} />)),
};
}

function lastRequestId() {
return mockEmit.mock.calls[mockEmit.mock.calls.length - 1][1].request_id;
}

function serverSends(event: AISocketEvent, payload, requestId: string) {
act(() => {
mockSocket.emit(command, event, payload, requestId);
});
}

beforeEach(() => {
mockSocket.removeAllListeners();
mockEmit.mockClear();
(toast.error as jest.Mock).mockClear();
});

it('ignores events from a superseded request', () => {
const onData = jest.fn();
const { result } = renderHook(onData);

act(() => result.current.emit({ question: 'first' }));
const staleRequestId = lastRequestId();
act(() => result.current.emit({ question: 'second' }));

serverSends(AISocketEvent.DATA, { query: 'stale' }, staleRequestId);
expect(onData).not.toHaveBeenCalled();

serverSends(AISocketEvent.DATA, { query: 'fresh' }, lastRequestId());
expect(onData).toHaveBeenCalledWith({
type: 'data',
data: { query: 'fresh' },
});
});

it('handles events without a request id from older servers', () => {
const onData = jest.fn();
const { result } = renderHook(onData);

act(() => result.current.emit({ question: 'first' }));
serverSends(AISocketEvent.DATA, { query: 'q' }, null);

expect(onData).toHaveBeenCalledWith({ type: 'data', data: { query: 'q' } });
});

it('handles each event once when emitting over an in-flight request', () => {
const { result } = renderHook(jest.fn());

act(() => result.current.emit({ question: 'first' }));
act(() => result.current.emit({ question: 'second' }));

serverSends(AISocketEvent.ERROR, 'LLM timeout', lastRequestId());

expect(toast.error).toHaveBeenCalledTimes(1);
expect(result.current.failed).toBe(true);
expect(result.current.loading).toBe(false);
});

it('removes its listeners on close after re-rendering with a new onData', () => {
const { result, rerender } = renderHook(jest.fn());

act(() => result.current.emit({ question: 'first' }));
const latestOnData = jest.fn();
rerender(latestOnData);

serverSends(AISocketEvent.DATA, { query: 'q' }, lastRequestId());
expect(latestOnData).toHaveBeenCalledTimes(1);

serverSends(AISocketEvent.CLOSE, null, lastRequestId());
expect(mockSocket.listenerCount(command)).toBe(0);
expect(mockSocket.listenerCount('error')).toBe(0);
});

it('removes its listeners when unmounted mid-request', () => {
const { result, unmount } = renderHook(jest.fn());

act(() => result.current.emit({ question: 'first' }));
unmount();

expect(mockSocket.listenerCount(command)).toBe(0);
expect(mockSocket.listenerCount('error')).toBe(0);
});
Loading
Loading