diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index dff5e27c7..0d84d0361 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -16,6 +16,7 @@ from .auth import * # noqa: E402, F403 from .ai_embeddings import * # noqa: E402, F403 +from .ai_apps import * # noqa: E402, F403 from .ai_scenes import * # noqa: E402, F403 from .apps import * # noqa: E402, F403 from .assets import * # noqa: E402, F403 diff --git a/backend/app/api/ai_apps.py b/backend/app/api/ai_apps.py new file mode 100644 index 000000000..6e57fec40 --- /dev/null +++ b/backend/app/api/ai_apps.py @@ -0,0 +1,155 @@ +from datetime import datetime +from http import HTTPStatus +from uuid import uuid4 + +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from app.database import get_db +from app.models.chat import Chat, ChatMessage +from app.models.settings import get_settings_dict +from app.schemas.ai_apps import AiAppChatRequest, AiAppChatResponse +from app.utils.ai_app import answer_app_question, edit_app_sources, route_app_chat +from . import api_with_auth + + +def _build_app_context_id(scene_id: str | None, node_id: str | None) -> str | None: + if not scene_id or not node_id: + return None + return f"{scene_id}::{node_id}" + + +@api_with_auth.post("/ai/apps/chat", response_model=AiAppChatResponse) +async def chat_app( + data: AiAppChatRequest, + db: Session = Depends(get_db), +): + request_id = data.request_id or str(uuid4()) + prompt = data.prompt.strip() + if not prompt: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail="Prompt is required") + + if not isinstance(data.sources, dict) or not data.sources: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail="App sources are required") + + chat = None + context_id = _build_app_context_id(data.scene_id, data.node_id) + if data.frame_id is not None: + if data.chat_id: + chat = db.query(Chat).filter(Chat.id == data.chat_id).first() + if chat and chat.frame_id != data.frame_id: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail="Chat does not belong to frame") + if chat and chat.context_type not in (None, "app"): + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail="Chat context does not match app chat") + if not chat: + if data.chat_id: + chat = Chat( + id=data.chat_id, + frame_id=data.frame_id, + context_type="app", + context_id=context_id, + ) + else: + chat = Chat( + frame_id=data.frame_id, + context_type="app", + context_id=context_id, + ) + db.add(chat) + db.commit() + db.refresh(chat) + if context_id: + chat.context_type = "app" + chat.context_id = context_id + if chat: + chat.updated_at = datetime.utcnow() + db.add(chat) + db.add(ChatMessage(chat_id=chat.id, role="user", content=prompt)) + db.commit() + + def _record_assistant_message(reply: str, tool: str) -> None: + if not chat: + return + chat.updated_at = datetime.utcnow() + db.add(chat) + db.add(ChatMessage(chat_id=chat.id, role="assistant", content=reply, tool=tool)) + db.commit() + + settings = get_settings_dict(db) + openai_settings = settings.get("openAI", {}) + api_key = openai_settings.get("backendApiKey") + if not api_key: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail="OpenAI backend API key not set") + + history = [item.model_dump() for item in (data.history or [])] + model = openai_settings.get("appChatModel") or openai_settings.get("chatModel") or "gpt-5-mini" + + tool_payload = {} + tool = "ask_about_app" + tool_prompt = prompt + try: + tool_payload = await route_app_chat( + prompt=prompt, + app_name=data.app_name, + app_keyword=data.app_keyword, + scene_id=data.scene_id, + node_id=data.node_id, + history=history, + api_key=api_key, + model=model, + ai_trace_id=request_id, + ai_session_id=None, + ) + if isinstance(tool_payload, dict): + tool = tool_payload.get("tool") or tool + tool_prompt = tool_payload.get("tool_prompt") or prompt + except Exception: + tool = "ask_about_app" + tool_prompt = prompt + + reply = "" + files: dict[str, str] | None = None + if tool == "edit_app": + edit_payload = await edit_app_sources( + prompt=tool_prompt, + sources=data.sources, + app_name=data.app_name, + app_keyword=data.app_keyword, + scene_id=data.scene_id, + node_id=data.node_id, + history=history, + api_key=api_key, + model=openai_settings.get("appEditModel") or openai_settings.get("chatModel") or "gpt-5-mini", + ai_trace_id=request_id, + ai_session_id=None, + ) + if isinstance(edit_payload, dict): + reply = edit_payload.get("reply") or "Updated app files." + files_payload = edit_payload.get("files") + if isinstance(files_payload, dict): + files = {str(key): str(value) for key, value in files_payload.items()} + if not reply: + reply = "Updated app files." + else: + answer_payload = await answer_app_question( + prompt=tool_prompt, + sources=data.sources, + app_name=data.app_name, + app_keyword=data.app_keyword, + scene_id=data.scene_id, + node_id=data.node_id, + history=history, + api_key=api_key, + model=openai_settings.get("appChatModel") or openai_settings.get("chatModel") or "gpt-5-mini", + ai_trace_id=request_id, + ai_session_id=None, + ) + if isinstance(answer_payload, dict): + reply = answer_payload.get("answer") or "" + if not reply: + reply = "Done." + tool = "ask_about_app" + + _record_assistant_message(reply, tool) + + return AiAppChatResponse(reply=reply, tool=tool, chatId=chat.id if chat else None, files=files) diff --git a/backend/app/api/ai_scenes.py b/backend/app/api/ai_scenes.py index c42a479d0..b70eef710 100644 --- a/backend/app/api/ai_scenes.py +++ b/backend/app/api/ai_scenes.py @@ -756,16 +756,35 @@ async def chat_scene( chat = db.query(Chat).filter(Chat.id == data.chat_id).first() if chat and chat.frame_id != data.frame_id: raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail="Chat does not belong to frame") + if chat and chat.context_type not in (None, "scene", "frame"): + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail="Chat context does not match scene chat") if not chat: if data.chat_id: - chat = Chat(id=data.chat_id, frame_id=data.frame_id, scene_id=data.scene_id) + chat = Chat( + id=data.chat_id, + frame_id=data.frame_id, + scene_id=data.scene_id, + context_type="scene" if data.scene_id else "frame", + context_id=data.scene_id, + ) else: - chat = Chat(frame_id=data.frame_id, scene_id=data.scene_id) + chat = Chat( + frame_id=data.frame_id, + scene_id=data.scene_id, + context_type="scene" if data.scene_id else "frame", + context_id=data.scene_id, + ) db.add(chat) db.commit() db.refresh(chat) if data.scene_id and chat.scene_id != data.scene_id: chat.scene_id = data.scene_id + if data.scene_id: + chat.context_type = "scene" + chat.context_id = data.scene_id + elif not chat.context_type: + chat.context_type = "frame" + chat.context_id = None if chat: chat.updated_at = datetime.utcnow() db.add(chat) diff --git a/backend/app/api/chats.py b/backend/app/api/chats.py index fa66e2b07..9d9592828 100644 --- a/backend/app/api/chats.py +++ b/backend/app/api/chats.py @@ -11,7 +11,18 @@ @api_with_auth.post("/ai/chats", response_model=ChatSummary) async def create_chat(data: ChatCreateRequest, db: Session = Depends(get_db)): - chat = Chat(frame_id=data.frame_id, scene_id=data.scene_id) + context_type = data.context_type + context_id = data.context_id + if not context_type: + context_type = "scene" if data.scene_id else "frame" + if context_type == "scene" and not context_id: + context_id = data.scene_id + chat = Chat( + frame_id=data.frame_id, + scene_id=data.scene_id, + context_type=context_type, + context_id=context_id, + ) db.add(chat) db.commit() db.refresh(chat) diff --git a/backend/app/models/chat.py b/backend/app/models/chat.py index 63d656e1b..a3caa2347 100644 --- a/backend/app/models/chat.py +++ b/backend/app/models/chat.py @@ -10,6 +10,8 @@ class Chat(Base): id = mapped_column(String(36), primary_key=True, default=lambda: str(uuid4())) frame_id = mapped_column(Integer, ForeignKey('frame.id'), nullable=False) scene_id = mapped_column(String(128), nullable=True) + context_type = mapped_column(String(32), nullable=True) + context_id = mapped_column(String(256), nullable=True) created_at = mapped_column(DateTime, nullable=False, default=func.current_timestamp()) updated_at = mapped_column( DateTime, @@ -25,6 +27,8 @@ def to_dict(self): 'id': self.id, 'frame_id': self.frame_id, 'scene_id': self.scene_id, + 'context_type': self.context_type, + 'context_id': self.context_id, 'created_at': self.created_at.isoformat() if self.created_at else None, 'updated_at': self.updated_at.isoformat() if self.updated_at else None, } diff --git a/backend/app/schemas/ai_apps.py b/backend/app/schemas/ai_apps.py new file mode 100644 index 000000000..9c31fa146 --- /dev/null +++ b/backend/app/schemas/ai_apps.py @@ -0,0 +1,30 @@ +from typing import Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class AiAppChatMessage(BaseModel): + role: str + content: str + + +class AiAppChatRequest(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + prompt: str + chat_id: Optional[str] = Field(default=None, alias="chatId") + frame_id: Optional[int] = Field(default=None, alias="frameId") + scene_id: Optional[str] = Field(default=None, alias="sceneId") + node_id: Optional[str] = Field(default=None, alias="nodeId") + app_name: Optional[str] = Field(default=None, alias="appName") + app_keyword: Optional[str] = Field(default=None, alias="appKeyword") + sources: Optional[dict[str, str]] = None + history: Optional[list[AiAppChatMessage]] = None + request_id: Optional[str] = Field(default=None, alias="requestId") + + +class AiAppChatResponse(BaseModel): + reply: str + tool: str + chat_id: Optional[str] = Field(default=None, alias="chatId") + files: Optional[dict[str, str]] = None diff --git a/backend/app/schemas/chats.py b/backend/app/schemas/chats.py index 00032ba15..da7472ff6 100644 --- a/backend/app/schemas/chats.py +++ b/backend/app/schemas/chats.py @@ -9,6 +9,8 @@ class ChatCreateRequest(BaseModel): frame_id: int = Field(alias="frameId") scene_id: Optional[str] = Field(default=None, alias="sceneId") + context_type: Optional[str] = Field(default=None, alias="contextType") + context_id: Optional[str] = Field(default=None, alias="contextId") class ChatSummary(BaseModel): @@ -17,6 +19,8 @@ class ChatSummary(BaseModel): id: str frame_id: int = Field(alias="frameId") scene_id: Optional[str] = Field(default=None, alias="sceneId") + context_type: Optional[str] = Field(default=None, alias="contextType") + context_id: Optional[str] = Field(default=None, alias="contextId") created_at: datetime = Field(alias="createdAt") updated_at: datetime = Field(alias="updatedAt") diff --git a/backend/app/utils/ai_app.py b/backend/app/utils/ai_app.py new file mode 100644 index 000000000..e21456b5d --- /dev/null +++ b/backend/app/utils/ai_app.py @@ -0,0 +1,216 @@ +import json +from typing import Any + +from app.config import config +from app.utils.ai_scene import _build_ai_posthog_properties, _new_ai_span_id, _openai_client + +APP_CHAT_ROUTER_SYSTEM_PROMPT = """ +You are a router that decides how to handle a FrameOS app chat request. +Choose exactly one tool: +- edit_app: The user wants changes to the app's source files. +- ask_about_app: The user is asking about how the app works or wants guidance. +Return JSON only with: +- tool: one of "edit_app", "ask_about_app" +- tool_prompt: a concise prompt for the chosen tool (or the original user request if no rewrite is needed) +Rules: +- Use edit_app when the user asks to change, add, fix, refactor, or optimize the app code or config. +- Use ask_about_app for explanations, diagnostics, or how-to questions about the app. +""".strip() + +APP_EDIT_SYSTEM_PROMPT = """ +You are editing a FrameOS app written in Nim. You have access to the Nim version 2.2 STL and the following nimble packages: +pixie v5, chrono 0.3.1, checksums 0.2.1, ws 0.5.0, psutil 0.6.0, QRGen 3.1.0, zippy 0.10, chroma 0.2.7, bumpy 1.1.2 + +Return the modified files in full with the changes inlined. Only modify what is necessary. +Return JSON only with: +- reply: a brief summary of the changes. +- files: an object mapping filenames to their full updated contents (only include files you changed). + +Make these changes: +""".strip() + +APP_EDIT_FILES_PROMPT = """ +------------- +Here are the relevant files of the app: +""".strip() + +APP_CHAT_ANSWER_SYSTEM_PROMPT = """ +You are a friendly assistant for FrameOS apps. +Answer questions about the current app or how to edit it. +Use the provided app sources and context. +Provide helpful context without overwhelming the user; keep replies concise unless they ask for specifics. +Limit answers to a few short paragraphs (2-3 max) and avoid long lists unless the user asks. +If the answer is uncertain, say what is missing and how to proceed. +Return JSON only with the key "answer". +""".strip() + + +def _format_app_sources(sources: dict[str, str]) -> str: + entries = [] + for file, content in sources.items(): + entries.append(f"# {file}\n```\n{content}\n```") + return "\n\n\n-------\n\n".join(entries) + + +def _format_app_context(app_name: str | None, app_keyword: str | None, scene_id: str | None, node_id: str | None) -> str: + parts = [] + if app_name: + parts.append(f"App name: {app_name}") + if app_keyword: + parts.append(f"App keyword: {app_keyword}") + if scene_id: + parts.append(f"Scene id: {scene_id}") + if node_id: + parts.append(f"Node id: {node_id}") + return "\n".join(parts) + + +async def route_app_chat( + *, + prompt: str, + app_name: str | None, + app_keyword: str | None, + scene_id: str | None, + node_id: str | None, + history: list[dict[str, str]] | None, + api_key: str, + model: str, + ai_trace_id: str | None = None, + ai_session_id: str | None = None, +) -> dict[str, Any]: + client = _openai_client(api_key) + span_id = _new_ai_span_id() + context_lines = _format_app_context(app_name, app_keyword, scene_id, node_id) + user_prompt = prompt if not context_lines else f"{context_lines}\n\nUser request: {prompt}" + messages = [{"role": "system", "content": APP_CHAT_ROUTER_SYSTEM_PROMPT}] + if history: + messages.extend(history) + messages.append({"role": "user", "content": user_prompt}) + response = await client.chat.completions.create( + model=model, + messages=messages, + response_format={"type": "json_object"}, + posthog_distinct_id=config.INSTANCE_ID, + posthog_properties=_build_ai_posthog_properties( + model=model, + ai_trace_id=ai_trace_id, + ai_session_id=ai_session_id, + ai_span_id=span_id, + ai_parent_id=None, + extra={"operation": "route_app_chat"}, + ), + ) + message = response.choices[0].message if response.choices else None + content = message.content if message else "{}" + return json.loads(content) + + +async def answer_app_question( + *, + prompt: str, + sources: dict[str, str], + app_name: str | None, + app_keyword: str | None, + scene_id: str | None, + node_id: str | None, + history: list[dict[str, str]] | None, + api_key: str, + model: str, + ai_trace_id: str | None = None, + ai_session_id: str | None = None, +) -> dict[str, Any]: + client = _openai_client(api_key) + span_id = _new_ai_span_id() + context_lines = _format_app_context(app_name, app_keyword, scene_id, node_id) + sources_block = _format_app_sources(sources) + user_prompt = "\n\n".join( + [ + line + for line in [ + context_lines or None, + prompt, + APP_EDIT_FILES_PROMPT, + sources_block, + ] + if line + ] + ) + messages = [{"role": "system", "content": APP_CHAT_ANSWER_SYSTEM_PROMPT}] + if history: + messages.extend(history) + messages.append({"role": "user", "content": user_prompt}) + response = await client.chat.completions.create( + model=model, + messages=messages, + response_format={"type": "json_object"}, + posthog_distinct_id=config.INSTANCE_ID, + posthog_properties=_build_ai_posthog_properties( + model=model, + ai_trace_id=ai_trace_id, + ai_session_id=ai_session_id, + ai_span_id=span_id, + ai_parent_id=None, + extra={"operation": "answer_app_question"}, + ), + ) + message = response.choices[0].message if response.choices else None + content = message.content if message else "{}" + return json.loads(content) + + +async def edit_app_sources( + *, + prompt: str, + sources: dict[str, str], + app_name: str | None, + app_keyword: str | None, + scene_id: str | None, + node_id: str | None, + history: list[dict[str, str]] | None, + api_key: str, + model: str, + ai_trace_id: str | None = None, + ai_session_id: str | None = None, +) -> dict[str, Any]: + client = _openai_client(api_key) + span_id = _new_ai_span_id() + context_lines = _format_app_context(app_name, app_keyword, scene_id, node_id) + sources_block = _format_app_sources(sources) + user_prompt = "\n\n".join( + [ + line + for line in [ + context_lines or None, + prompt, + APP_EDIT_FILES_PROMPT, + sources_block, + ] + if line + ] + ) + messages = [{"role": "system", "content": APP_EDIT_SYSTEM_PROMPT}] + if history: + messages.extend(history) + messages.append( + { + "role": "user", + "content": user_prompt, + } + ) + response = await client.chat.completions.create( + model=model, + messages=messages, + response_format={"type": "json_object"}, + posthog_distinct_id=config.INSTANCE_ID, + posthog_properties=_build_ai_posthog_properties( + model=model, + ai_trace_id=ai_trace_id, + ai_session_id=ai_session_id, + ai_span_id=span_id, + ai_parent_id=None, + extra={"operation": "edit_app_sources"}, + ), + ) + message = response.choices[0].message if response.choices else None + content = message.content if message else "{}" + return json.loads(content) diff --git a/backend/migrations/versions/f78f9e8ec198_ai_context.py b/backend/migrations/versions/f78f9e8ec198_ai_context.py new file mode 100644 index 000000000..cf9c563e2 --- /dev/null +++ b/backend/migrations/versions/f78f9e8ec198_ai_context.py @@ -0,0 +1,30 @@ +"""ai context + +Revision ID: f78f9e8ec198 +Revises: 45c374c4d6ec +Create Date: 2026-01-25 00:30:36.274550 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'f78f9e8ec198' +down_revision = '45c374c4d6ec' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('chat', sa.Column('context_type', sa.String(length=32), nullable=True)) + op.add_column('chat', sa.Column('context_id', sa.String(length=256), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('chat', 'context_id') + op.drop_column('chat', 'context_type') + # ### end Alembic commands ### diff --git a/frameos/src/apps/data/unsplash/app.nim b/frameos/src/apps/data/unsplash/app.nim index 9206ca8d6..1f90749f5 100644 --- a/frameos/src/apps/data/unsplash/app.nim +++ b/frameos/src/apps/data/unsplash/app.nim @@ -35,8 +35,8 @@ proc get*(self: App, context: ExecutionContext): Image = let height = if context.hasImage: context.image.height else: self.frameConfig.renderHeight() let search = self.appConfig.search let orientation = if self.appConfig.orientation == "auto": - if width > height: "portrait" - elif width < height: "landscape" + if height > width: "portrait" + elif width > height: "landscape" else: "squarish" elif self.appConfig.orientation == "any": "" else: self.appConfig.orientation diff --git a/frontend/src/scenes/frame/panels/Chat/Chat.tsx b/frontend/src/scenes/frame/panels/Chat/Chat.tsx index 32941b0cb..68882938f 100644 --- a/frontend/src/scenes/frame/panels/Chat/Chat.tsx +++ b/frontend/src/scenes/frame/panels/Chat/Chat.tsx @@ -25,6 +25,8 @@ export function Chat() { error, chatSceneName, chatSceneId, + chatContextType, + chatAppContext, contextItemsExpanded, chatView, visibleChats, @@ -36,6 +38,7 @@ export function Chat() { isCreatingChat, contextSelectionSummary, logExpanded, + chatLabelForChat, } = useValues(chatLogic({ frameId, sceneId: selectedSceneId })) const { setInput, @@ -48,7 +51,7 @@ export function Chat() { createChat, loadMoreChats, } = useActions(chatLogic({ frameId, sceneId: selectedSceneId })) - const { setPanel } = useActions(panelsLogic({ frameId })) + const { setPanel, editApp } = useActions(panelsLogic({ frameId })) const { focusScene } = useActions(scenesLogic({ frameId })) const [atBottom, setAtBottom] = useState(true) const virtuosoRef = useRef(null) @@ -67,6 +70,9 @@ export function Chat() { focusScene(sceneId) } + const appLabel = + chatAppContext?.nodeData?.name || chatAppContext?.nodeData?.keyword || chatAppContext?.nodeId || 'this app' + useEffect(() => { if (!shouldStickToBottomRef.current) { return @@ -139,6 +145,14 @@ export function Chat() { focusSceneById(chatSceneId) } + const handleOpenApp = (e: React.MouseEvent) => { + e.preventDefault() + if (!chatAppContext?.nodeData) { + return + } + editApp(chatAppContext.sceneId, chatAppContext.nodeId, chatAppContext.nodeData) + } + const formatTimestamp = (timestamp?: string | null) => { if (!timestamp) { return 'Just now' @@ -150,14 +164,6 @@ export function Chat() { return date.toLocaleString() } - const getSceneName = (sceneId?: string | null) => { - if (!sceneId) { - return 'Frame chat' - } - const scene = scenes?.find((item) => item.id === sceneId) - return scene?.name ?? 'Frame chat' - } - const normalizeGeneratedSceneName = (sceneName: string) => sceneName.replace(/^["']|["']$/g, '').trim() const extractGeneratedSceneName = (message: string) => { @@ -343,6 +349,14 @@ export function Chat() { } if (!messageContent) { + if (isStreaming) { + return ( +
+ + Thinking… +
+ ) + } return null } @@ -376,6 +390,21 @@ export function Chat() { "{chatSceneName}" + ) : chatContextType === 'app' ? ( + + Chat about{' '} + {chatAppContext?.nodeData ? ( + + "{appLabel}" + + ) : ( + `"${appLabel}"` + )} + ) : ( 'Chat about this frame' ) @@ -411,7 +440,9 @@ export function Chat() {
Start the conversation
- {chatSceneName + {chatContextType === 'app' + ? 'Ask for edits to this app, or ask questions about how it works.' + : chatSceneName ? 'Ask for a new scene, request edits to the current scene, or ask questions about FrameOS.' : 'Ask for a new scene, or ask questions about this frame or FrameOS.'}
@@ -439,7 +470,29 @@ export function Chat() { const isLog = message.tool === 'log' const isUser = message.role === 'user' if (message.isPlaceholder && !message.content && !message.tool) { - return null + if (!message.isStreaming) { + return null + } + return ( +
+
+
+ {message.role} +
+
+ + Thinking… +
+
+
+ ) } return (
@@ -479,7 +532,11 @@ export function Chat() {