From 69b93649da3eb74eee2abd17f8d92b25f303f019 Mon Sep 17 00:00:00 2001 From: "eysho.dev" Date: Mon, 20 Oct 2025 03:43:06 +0200 Subject: [PATCH] Add Supabase hosting assets and database bootstrap --- .env.local.example | 42 ++++ database/init.sql | 340 +++++++++++++++++++++++++++++++ docs/DEPLOYMENT.md | 186 +++++++++++++++++ docs/FILES_OVERVIEW.md | 28 +++ docs/QUICK_START.md | 77 +++++++ docs/SUPABASE_POSTGRES_SETUP.md | 164 +++++++++++++++ src/lib/supabaseClient.js | 347 ++++++++++++++++++++++++++++++++ 7 files changed, 1184 insertions(+) create mode 100644 .env.local.example create mode 100644 database/init.sql create mode 100644 docs/DEPLOYMENT.md create mode 100644 docs/FILES_OVERVIEW.md create mode 100644 docs/QUICK_START.md create mode 100644 docs/SUPABASE_POSTGRES_SETUP.md create mode 100644 src/lib/supabaseClient.js diff --git a/.env.local.example b/.env.local.example new file mode 100644 index 0000000..be849bf --- /dev/null +++ b/.env.local.example @@ -0,0 +1,42 @@ +# CyberDevStudio environment template for Supabase + PostgreSQL hosting +# Copy this file to `.env.local` (frontend) or `.env` (backend/runtime) and fill in +# the secrets provided by Supabase and your infrastructure providers. + +# --- Supabase ----------------------------------------------------------------- +NEXT_PUBLIC_SUPABASE_URL=https://your-project-id.supabase.co +NEXT_PUBLIC_SUPABASE_ANON_KEY=your-public-anon-key +SUPABASE_SERVICE_ROLE_KEY=your-service-role-key +SUPABASE_JWT_SECRET=your-jwt-secret +SUPABASE_STORAGE_S3_BUCKET=cyberdevstudio-assets + +# --- PostgreSQL / PostgresML -------------------------------------------------- +POSTGRES_HOST=your-postgres-host.supabase.co +POSTGRES_PORT=5432 +POSTGRES_DB=postgres +POSTGRES_USER=postgres +POSTGRES_PASSWORD=super-secret-password +POSTGRES_SSL=verify-full + +# Optional: direct connection string for server components +DATABASE_URL=postgresql://postgres:super-secret-password@your-postgres-host.supabase.co:5432/postgres?sslmode=require + +# --- Platform runtime --------------------------------------------------------- +LLM_SERVER_URL=http://localhost:6988 +API_BASE_URL=http://localhost:6813 +SANDBOX_RPC_URL=http://localhost:7777/jsonrpc +ADMIN_DASHBOARD_URL=http://localhost:6711/admin + +# Token accounting defaults +DEFAULT_TOKEN_BALANCE=100000 +TOKEN_ALERT_THRESHOLD=2000 + +# --- Observability ------------------------------------------------------------ +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +PROMETHEUS_PUSHGATEWAY=http://localhost:9091 +SENTRY_DSN= + +# --- Deployment metadata ------------------------------------------------------ +APP_ENV=development +VERCEL_PROJECT_ID= +NETLIFY_SITE_ID= +DOCKER_REGISTRY= diff --git a/database/init.sql b/database/init.sql new file mode 100644 index 0000000..9354242 --- /dev/null +++ b/database/init.sql @@ -0,0 +1,340 @@ +-- CyberDevStudio database bootstrap for Supabase + PostgreSQL + PostgresML +-- Run this script once inside your Supabase SQL editor or any psql session +-- connected to the project database. + +BEGIN; + +-- Required extensions +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +CREATE EXTENSION IF NOT EXISTS pgcrypto; +CREATE EXTENSION IF NOT EXISTS citext; +CREATE EXTENSION IF NOT EXISTS vector; +CREATE EXTENSION IF NOT EXISTS postgresml; + +-- Dedicated schemas to keep responsibilities separated +CREATE SCHEMA IF NOT EXISTS platform AUTHORIZATION CURRENT_USER; +CREATE SCHEMA IF NOT EXISTS telemetry AUTHORIZATION CURRENT_USER; +CREATE SCHEMA IF NOT EXISTS analytics AUTHORIZATION CURRENT_USER; + +SET search_path TO platform, public; + +-- Shared helper to automatically bump updated_at columns +CREATE OR REPLACE FUNCTION platform.touch_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at := NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Users & auth --------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS platform.users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + username CITEXT UNIQUE NOT NULL, + email CITEXT UNIQUE NOT NULL, + password_hash TEXT, + role TEXT NOT NULL CHECK (role IN ('admin', 'developer', 'viewer')), + api_key_hash TEXT, + balance_tokens BIGINT NOT NULL DEFAULT 0, + last_login_at TIMESTAMPTZ, + metadata JSONB NOT NULL DEFAULT '{}'::JSONB +); + +CREATE TRIGGER trg_users_updated_at +BEFORE UPDATE ON platform.users +FOR EACH ROW EXECUTE FUNCTION platform.touch_updated_at(); + +COMMENT ON TABLE platform.users IS 'Primary user table shared between Supabase auth and the CyberDevStudio platform.'; + +-- API keys ------------------------------------------------------------------ +CREATE TABLE IF NOT EXISTS platform.api_keys ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + user_id UUID NOT NULL REFERENCES platform.users(id) ON DELETE CASCADE, + name TEXT NOT NULL, + api_key_hash TEXT NOT NULL, + expires_at TIMESTAMPTZ, + last_used_at TIMESTAMPTZ, + scopes TEXT[] NOT NULL DEFAULT ARRAY['llm:read'], + is_revoked BOOLEAN NOT NULL DEFAULT FALSE +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_api_keys_user_name + ON platform.api_keys (user_id, name); + +CREATE TRIGGER trg_api_keys_updated_at +BEFORE UPDATE ON platform.api_keys +FOR EACH ROW EXECUTE FUNCTION platform.touch_updated_at(); + +-- Models -------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS platform.models ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + name TEXT NOT NULL UNIQUE, + provider TEXT NOT NULL DEFAULT 'local', + repo_url TEXT, + context_size INTEGER NOT NULL DEFAULT 4096, + cost_per_1k_tokens NUMERIC(12,6) NOT NULL DEFAULT 0, + is_default BOOLEAN NOT NULL DEFAULT FALSE, + metadata JSONB NOT NULL DEFAULT '{}'::JSONB +); + +CREATE TRIGGER trg_models_updated_at +BEFORE UPDATE ON platform.models +FOR EACH ROW EXECUTE FUNCTION platform.touch_updated_at(); + +-- Token accounting ----------------------------------------------------------- +CREATE TABLE IF NOT EXISTS platform.token_usage ( + id BIGSERIAL PRIMARY KEY, + occurred_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + user_id UUID NOT NULL REFERENCES platform.users(id) ON DELETE CASCADE, + model_id UUID REFERENCES platform.models(id) ON DELETE SET NULL, + request_id UUID NOT NULL, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + cost_tokens INTEGER NOT NULL DEFAULT 0, + metadata JSONB NOT NULL DEFAULT '{}'::JSONB +); + +CREATE INDEX IF NOT EXISTS idx_token_usage_user_time + ON platform.token_usage (user_id, occurred_at DESC); + +CREATE INDEX IF NOT EXISTS idx_token_usage_request + ON platform.token_usage (request_id); + +-- Projects ------------------------------------------------------------------ +CREATE TABLE IF NOT EXISTS platform.projects ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + owner_id UUID NOT NULL REFERENCES platform.users(id) ON DELETE CASCADE, + name TEXT NOT NULL, + description TEXT, + visibility TEXT NOT NULL DEFAULT 'private' CHECK (visibility IN ('private', 'internal', 'public')), + metadata JSONB NOT NULL DEFAULT '{}'::JSONB +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_owner_name + ON platform.projects (owner_id, name); + +CREATE TRIGGER trg_projects_updated_at +BEFORE UPDATE ON platform.projects +FOR EACH ROW EXECUTE FUNCTION platform.touch_updated_at(); + +-- Sessions ------------------------------------------------------------------ +CREATE TABLE IF NOT EXISTS platform.sessions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + user_id UUID NOT NULL REFERENCES platform.users(id) ON DELETE CASCADE, + project_id UUID REFERENCES platform.projects(id) ON DELETE SET NULL, + model_id UUID REFERENCES platform.models(id) ON DELETE SET NULL, + status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'idle', 'closed')), + last_activity TIMESTAMPTZ NOT NULL DEFAULT NOW(), + metadata JSONB NOT NULL DEFAULT '{}'::JSONB +); + +CREATE INDEX IF NOT EXISTS idx_sessions_user_status + ON platform.sessions (user_id, status); + +CREATE TRIGGER trg_sessions_updated_at +BEFORE UPDATE ON platform.sessions +FOR EACH ROW EXECUTE FUNCTION platform.touch_updated_at(); + +-- Telemetry schema ----------------------------------------------------------- +SET search_path TO telemetry, public; + +CREATE TABLE IF NOT EXISTS telemetry.agent_events ( + id BIGSERIAL PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + session_id UUID REFERENCES platform.sessions(id) ON DELETE CASCADE, + user_id UUID REFERENCES platform.users(id) ON DELETE CASCADE, + event_type TEXT NOT NULL, + message TEXT, + payload JSONB NOT NULL DEFAULT '{}'::JSONB +); + +CREATE INDEX IF NOT EXISTS idx_agent_events_session + ON telemetry.agent_events (session_id, created_at DESC); + +CREATE TABLE IF NOT EXISTS telemetry.execution_metrics ( + id BIGSERIAL PRIMARY KEY, + collected_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + project_id UUID REFERENCES platform.projects(id) ON DELETE CASCADE, + cpu_percent NUMERIC(5,2), + memory_mb NUMERIC(10,2), + duration_ms INTEGER, + outcome TEXT NOT NULL DEFAULT 'success', + metadata JSONB NOT NULL DEFAULT '{}'::JSONB +); + +CREATE INDEX IF NOT EXISTS idx_execution_metrics_project + ON telemetry.execution_metrics (project_id, collected_at DESC); + +-- Analytics schema ----------------------------------------------------------- +SET search_path TO analytics, public; + +CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.daily_token_summary AS +SELECT + date_trunc('day', occurred_at) AS day, + user_id, + SUM(prompt_tokens) AS prompt_tokens, + SUM(completion_tokens) AS completion_tokens, + SUM(cost_tokens) AS cost_tokens +FROM platform.token_usage +GROUP BY 1, 2 +WITH NO DATA; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_daily_token_summary_day_user + ON analytics.daily_token_summary (day, user_id); + +CREATE OR REPLACE FUNCTION analytics.refresh_daily_token_summary() +RETURNS VOID +LANGUAGE plpgsql +SECURITY DEFINER +AS $$ +BEGIN + REFRESH MATERIALIZED VIEW CONCURRENTLY analytics.daily_token_summary; +END; +$$; + +-- RPC helpers (callable from Supabase client) -------------------------------- +SET search_path TO platform, public; + +CREATE OR REPLACE FUNCTION platform.record_token_usage( + p_user_id UUID, + p_model_name TEXT, + p_request_id UUID, + p_prompt_tokens INTEGER, + p_completion_tokens INTEGER, + p_metadata JSONB DEFAULT '{}'::JSONB +) RETURNS VOID +LANGUAGE plpgsql +SECURITY DEFINER +AS $$ +DECLARE + v_model_id UUID; + v_cost_tokens INTEGER; +BEGIN + SELECT id INTO v_model_id FROM platform.models WHERE name = p_model_name; + v_cost_tokens := COALESCE(p_prompt_tokens, 0) + COALESCE(p_completion_tokens, 0); + + INSERT INTO platform.token_usage (user_id, model_id, request_id, prompt_tokens, completion_tokens, cost_tokens, metadata) + VALUES (p_user_id, v_model_id, p_request_id, p_prompt_tokens, p_completion_tokens, v_cost_tokens, p_metadata); + + UPDATE platform.users + SET balance_tokens = GREATEST(balance_tokens - v_cost_tokens, 0) + WHERE id = p_user_id; +END; +$$; + +CREATE OR REPLACE FUNCTION platform.grant_user_tokens( + p_user_id UUID, + p_amount BIGINT, + p_reason TEXT DEFAULT NULL +) RETURNS BIGINT +LANGUAGE plpgsql +SECURITY DEFINER +AS $$ +DECLARE + v_new_balance BIGINT; +BEGIN + UPDATE platform.users + SET balance_tokens = balance_tokens + p_amount + WHERE id = p_user_id + RETURNING balance_tokens INTO v_new_balance; + + INSERT INTO platform.token_usage (user_id, request_id, prompt_tokens, completion_tokens, cost_tokens, metadata) + VALUES ( + p_user_id, + gen_random_uuid(), + 0, + 0, + -p_amount, + jsonb_build_object('event', 'grant', 'reason', p_reason) + ); + + RETURN v_new_balance; +END; +$$; + +CREATE OR REPLACE FUNCTION platform.get_usage_overview( + p_user_id UUID, + p_limit INTEGER DEFAULT 50 +) RETURNS TABLE ( + occurred_at TIMESTAMPTZ, + model_name TEXT, + prompt_tokens INTEGER, + completion_tokens INTEGER, + cost_tokens INTEGER, + metadata JSONB +) +LANGUAGE sql +SECURITY DEFINER +AS $$ + SELECT + tu.occurred_at, + m.name AS model_name, + tu.prompt_tokens, + tu.completion_tokens, + tu.cost_tokens, + tu.metadata + FROM platform.token_usage tu + LEFT JOIN platform.models m ON m.id = tu.model_id + WHERE tu.user_id = p_user_id + ORDER BY tu.occurred_at DESC + LIMIT p_limit; +$$; + +CREATE OR REPLACE FUNCTION platform.set_default_model(p_model_id UUID) +RETURNS VOID +LANGUAGE plpgsql +SECURITY DEFINER +AS $$ +BEGIN + UPDATE platform.models SET is_default = FALSE; + UPDATE platform.models SET is_default = TRUE WHERE id = p_model_id; +END; +$$; + +-- Basic policies for Supabase row-level security (RLS) ---------------------- +ALTER TABLE platform.users ENABLE ROW LEVEL SECURITY; +ALTER TABLE platform.projects ENABLE ROW LEVEL SECURITY; +ALTER TABLE platform.sessions ENABLE ROW LEVEL SECURITY; +ALTER TABLE platform.token_usage ENABLE ROW LEVEL SECURITY; +ALTER TABLE platform.api_keys ENABLE ROW LEVEL SECURITY; + +CREATE POLICY users_self_access ON platform.users + USING (auth.uid() = id) + WITH CHECK (auth.uid() = id); + +CREATE POLICY projects_owner_access ON platform.projects + USING (auth.uid() = owner_id) + WITH CHECK (auth.uid() = owner_id); + +CREATE POLICY sessions_owner_access ON platform.sessions + USING (auth.uid() = user_id) + WITH CHECK (auth.uid() = user_id); + +CREATE POLICY token_usage_owner_access ON platform.token_usage + USING (auth.uid() = user_id); + +CREATE POLICY api_keys_owner_access ON platform.api_keys + USING (auth.uid() = user_id) + WITH CHECK (auth.uid() = user_id); + +-- Privileges for service role ------------------------------------------------ +GRANT USAGE ON SCHEMA platform, telemetry, analytics TO postgres; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA platform TO postgres; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA telemetry TO postgres; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA analytics TO postgres; +GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA platform TO postgres; + +-- Refresh analytics after initial load +REFRESH MATERIALIZED VIEW CONCURRENTLY analytics.daily_token_summary; + +COMMIT; diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 0000000..fbd50f3 --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,186 @@ +# Deployment Playbook + +This document compiles the most common hosting strategies for CyberDevStudio. Each section assumes you have completed the database setup in [`docs/SUPABASE_POSTGRES_SETUP.md`](./SUPABASE_POSTGRES_SETUP.md) and populated your environment variables from `.env.local.example`. + +--- + +## 1. Shared Prerequisites + +- Node.js 18+ and pnpm/npm/yarn (for the Studio UI and server builds) +- Docker 24+ (for container-based options) +- Supabase project credentials & `DATABASE_URL` +- Access to the LLM server host (local machine or remote VM with `node-llama-cpp`) + +Before deploying, build the applications locally: + +```bash +pnpm install +pnpm run build --filter studio-ui +pnpm run build --filter api +``` + +Commit the optimized build artifacts before deploying to immutable targets (Netlify/Vercel). + +--- + +## 2. Vercel (Studio UI) + +1. Push your repository to GitHub/GitLab/Bitbucket. +2. Import the project in [Vercel](https://vercel.com/import) and select the `apps/studio-ui` directory as the root. +3. Set environment variables under **Settings → Environment Variables** using `.env.local` values (`NEXT_PUBLIC_SUPABASE_URL`, `NEXT_PUBLIC_SUPABASE_ANON_KEY`, etc.). +4. Configure build command: `pnpm install && pnpm run build --filter studio-ui` and output directory: `.vercel/output` or `apps/studio-ui/.next` depending on your setup. +5. Add a custom domain and enforce HTTPS. +6. Optional: create a [cron job](https://vercel.com/docs/cron-jobs) to refresh analytics via the `/api/cron/refresh-analytics` route (if implemented). + +--- + +## 3. Netlify (Static marketing or docs) + +1. Point Netlify to the repository and select the `docs/` directory or your marketing site root. +2. Build command: `pnpm run docs:build` (create this script if missing). +3. Publish directory: `docs/dist`. +4. Inject read-only Supabase credentials if your documentation references live data. +5. Enable [Netlify Identity](https://docs.netlify.com/visitor-access/identity/) only if you need gated documentation; main authentication remains with Supabase. + +--- + +## 4. Docker & Docker Compose (Full stack) + +The repository includes `docker/` manifests. To launch everything locally or on a VM: + +```bash +docker compose -f docker/docker-compose.yml --env-file .env up -d --build +``` + +Key services: + +| Service | Port | Description | +| ------- | ---- | ----------- | +| api | 6813 | JSON-RPC gateway, token accounting | +| llmserver | 6988 | `node-llama-cpp` wrapper with token throttling | +| studio-ui | 6711 | Web interface (Next.js / React) | +| auth | 6971 | Authentication microservice | +| db | 6472 | PostgresML database (externalized for Supabase in production) | + +**Production tips** + +- Replace the `db` service with Supabase by removing it from Compose and pointing `DATABASE_URL` to your managed instance. +- Add Traefik or Caddy in front of the stack for HTTPS termination and path routing. +- Configure persistent volumes (`models/`, `logs/`, `pgdata/`). + +--- + +## 5. Heroku (API + Background workers) + +1. Create two Heroku apps: `cyberdevstudio-api` and `cyberdevstudio-workers`. +2. Provision the **Heroku Postgres** add-on only for ephemeral staging; production should continue using Supabase via `DATABASE_URL`. +3. Define buildpacks: + ```bash + heroku buildpacks:add --app cyberdevstudio-api heroku/nodejs + heroku buildpacks:add --app cyberdevstudio-workers heroku/nodejs + ``` +4. Push code using the `heroku` remote or the Container Registry. +5. Configure config vars copied from `.env` (never commit secrets). +6. Scale workers for scheduled refresh jobs (`heroku ps:scale cron=1:standard-1x`). + +--- + +## 6. Bare-metal / VPS + +1. Provision an Ubuntu 22.04 VM with at least 8 GB RAM for LLM inference. +2. Install Docker, docker-compose-plugin, Node.js (via `nvm`), and `pm2`. +3. Clone the repository and copy `.env`. +4. Start services: + ```bash + pnpm install --frozen-lockfile + pnpm run build --filter api + pm2 start apps/api/dist/main.js --name cyberdevstudio-api -- --port 6813 + pm2 start "node apps/llmserver/index.js" --name cyberdevstudio-llm + pnpm --filter studio-ui start + ``` +5. Use Nginx as a reverse proxy and set up automatic renewals with Certbot. +6. Configure system monitoring (Prometheus Node Exporter, Grafana dashboards) to consume metrics emitted from the platform. + +--- + +## 7. GitHub Pages (Docs only) + +1. Build the documentation to a static directory (`docs/dist`). +2. Commit the artifacts or generate them in the CI workflow. +3. Push to the `gh-pages` branch or use GitHub Actions with the `peaceiris/actions-gh-pages` action. +4. Remember that GitHub Pages is static—do **not** expose Supabase service role secrets here. + +--- + +## 8. Continuous Deployment Workflow + +Here is a sample GitHub Actions workflow that coordinates the deployments: + +```yaml +name: deploy + +on: + push: + branches: [main] + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v3 + with: + version: 8 + - run: pnpm install --frozen-lockfile + - run: pnpm run lint && pnpm run test + - run: pnpm run build --filter studio-ui --filter api + - uses: actions/upload-artifact@v4 + with: + name: web-build + path: apps/studio-ui/.next + + deploy-vercel: + needs: build + runs-on: ubuntu-latest + environment: production + steps: + - uses: actions/download-artifact@v4 + with: + name: web-build + - name: Deploy to Vercel + uses: amondnet/vercel-action@v25 + with: + vercel-token: ${{ secrets.VERCEL_TOKEN }} + vercel-org-id: ${{ secrets.VERCEL_ORG_ID }} + vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }} + + deploy-docker: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: docker build -f docker/Dockerfile.api -t ghcr.io/org/cyberdevstudio-api:latest . + - run: docker push ghcr.io/org/cyberdevstudio-api:latest +``` + +Extend the workflow with notifications, migrations (`supabase db push`), and health checks as needed. + +--- + +## 9. Post-deployment Checklist + +- [ ] Run database migrations (`supabase db push` or `pnpm run prisma:migrate` depending on your stack) +- [ ] Rotate API keys and service-role secrets regularly +- [ ] Validate CORS configuration on the API and Supabase Storage +- [ ] Ensure monitoring dashboards receive metrics from `telemetry.execution_metrics` +- [ ] Test LLM inference end-to-end (UI → API → LLM server → Postgres token log) + +--- + +Continue with the fast onboarding steps in [`docs/QUICK_START.md`](./QUICK_START.md) to bring new team members online quickly. diff --git a/docs/FILES_OVERVIEW.md b/docs/FILES_OVERVIEW.md new file mode 100644 index 0000000..6b88982 --- /dev/null +++ b/docs/FILES_OVERVIEW.md @@ -0,0 +1,28 @@ +# Files Overview + +The Supabase + PostgreSQL configuration introduces seven primary artifacts. Use this document as an index when navigating the repository. + +| File | Purpose | Key Highlights | +| ---- | ------- | -------------- | +| [`database/init.sql`](../database/init.sql) | Bootstraps database schemas (`platform`, `telemetry`, `analytics`) and installs PostgresML extensions | Adds core tables, RLS policies, RPC helpers (`record_token_usage`, `grant_user_tokens`, `get_usage_overview`, `set_default_model`) | +| [`.env.local.example`](../.env.local.example) | Environment template for local and production deployments | Contains placeholders for Supabase keys, Postgres connection, runtime URLs, and observability endpoints | +| [`docs/SUPABASE_POSTGRES_SETUP.md`](./SUPABASE_POSTGRES_SETUP.md) | Step-by-step data layer setup | Covers Supabase project creation, SQL import, PostgresML activation, auth sync, and maintenance | +| [`docs/DEPLOYMENT.md`](./DEPLOYMENT.md) | Hosting strategies reference | Details Vercel, Netlify, Docker Compose, Heroku, VPS, GitHub Pages, and CI/CD workflow | +| [`docs/QUICK_START.md`](./QUICK_START.md) | Five-minute onboarding checklist | Guides new contributors through env setup, SQL execution, local services, and validation | +| [`src/lib/supabaseClient.js`](../src/lib/supabaseClient.js) | Supabase client wrapper with rich helper set | Provides factory functions, typed RPC helpers, admin utilities, and telemetry logging | +| `docs/FILES_OVERVIEW.md` | You are here | Summarizes the created assets | + +## Suggested Reading Order + +1. [`docs/QUICK_START.md`](./QUICK_START.md) – fastest way to get running +2. [`docs/SUPABASE_POSTGRES_SETUP.md`](./SUPABASE_POSTGRES_SETUP.md) – deep dive into database steps +3. [`docs/DEPLOYMENT.md`](./DEPLOYMENT.md) – choose your hosting strategy +4. [`src/lib/supabaseClient.js`](../src/lib/supabaseClient.js) – integrate the client utilities into your codebase + +## Keeping the Index Updated + +- Add a new row whenever you introduce a supporting asset (migration, diagram, script). +- Keep descriptions concise (≤ 120 characters) but actionable. +- Cross-link related files to improve discoverability. + +For questions or improvements, open an issue in the repository and tag the **Platform** team. diff --git a/docs/QUICK_START.md b/docs/QUICK_START.md new file mode 100644 index 0000000..c3c9044 --- /dev/null +++ b/docs/QUICK_START.md @@ -0,0 +1,77 @@ +# CyberDevStudio Quick Start (≤ 5 minutes) + +This checklist brings a new developer from zero to a functional CyberDevStudio environment backed by Supabase and PostgresML. + +--- + +## 0. Clone & Install + +```bash +git clone +cd cyberdevstudio +pnpm install +``` + +--- + +## 1. Configure Environment + +1. Copy the template and fill in Supabase credentials: + ```bash + cp .env.local.example .env.local + cp .env.local .env + ``` +2. Update the following keys at minimum: + - `NEXT_PUBLIC_SUPABASE_URL` + - `NEXT_PUBLIC_SUPABASE_ANON_KEY` + - `SUPABASE_SERVICE_ROLE_KEY` + - `DATABASE_URL` + +--- + +## 2. Bootstrap the Database + +1. Open the Supabase SQL editor. +2. Paste [`database/init.sql`](../database/init.sql) and run it. +3. Verify tables in the `platform`, `telemetry`, and `analytics` schemas appear. + +> Optional: run the trigger snippet from [`docs/SUPABASE_POSTGRES_SETUP.md`](./SUPABASE_POSTGRES_SETUP.md#5-configure-authentication) to mirror Supabase Auth users automatically. + +--- + +## 3. Launch Local Services + +```bash +# 1. Start the Supabase local stack (optional but recommended) +supabase start + +# 2. Start the API + LLM server + Studio UI +docker compose -f docker/docker-compose.yml --env-file .env up -d api llmserver +pnpm --filter studio-ui dev +``` + +- API available at `http://localhost:6813` +- Studio UI available at `http://localhost:3000` (or the port defined by Next.js) +- LLM server at `http://localhost:6988` + +--- + +## 4. Validate the Setup + +1. **Authentication** – Sign up using the Studio UI and ensure the new user record exists in `platform.users`. +2. **Token accounting** – Trigger a completion and inspect `platform.token_usage` for the logged event. +3. **PostgresML** – Run: + ```sql + SELECT postgresml.complete('deepseek-coder-1.3b', 'Say hello to CyberDevStudio'); + ``` +4. **Telemetry** – Check `telemetry.agent_events` for sandbox activity. + +--- + +## 5. Next Steps + +- Follow deployment strategies in [`docs/DEPLOYMENT.md`](./DEPLOYMENT.md) +- Review Supabase best practices in [`docs/SUPABASE_POSTGRES_SETUP.md`](./SUPABASE_POSTGRES_SETUP.md) +- Keep the file index from [`docs/FILES_OVERVIEW.md`](./FILES_OVERVIEW.md) handy when onboarding teammates + +You are ready to build! 🚀 diff --git a/docs/SUPABASE_POSTGRES_SETUP.md b/docs/SUPABASE_POSTGRES_SETUP.md new file mode 100644 index 0000000..508c49d --- /dev/null +++ b/docs/SUPABASE_POSTGRES_SETUP.md @@ -0,0 +1,164 @@ +# Supabase & PostgreSQL Hosting Guide + +This guide walks you through provisioning the complete data layer for **CyberDevStudio**, including Supabase configuration, PostgreSQL hardening, and PostgresML enablement. Follow the steps sequentially—each section builds on the previous one. + +--- + +## 1. Prerequisites + +- Supabase account with project owner permissions +- Local tooling: [`psql`](https://www.postgresql.org/download/), [Supabase CLI](https://supabase.com/docs/guides/cli), [`curl`](https://curl.se/) for verification +- Optional: Docker (for local PostgresML testing) +- Copy of the repository so you can reference the SQL and environment files shipped with this guide + +Before starting, duplicate `.env.local.example` into `.env.local` (frontend) and `.env` (backend/server processes) so you can collect credentials as you go. + +--- + +## 2. Create a Supabase Project + +1. Sign in to [Supabase](https://supabase.com/dashboard/projects). +2. Click **New project**, choose a strong database password, and pick the closest region to your users. +3. Wait for the project to provision (~2 minutes). Once ready, copy: + - **Project URL** + - **anon key** + - **service_role key** + - **JWT secret** +4. Paste the values into your environment file placeholders: + ```bash + cp .env.local.example .env.local + cp .env.local .env + ```` + Edit `.env.local` with the retrieved secrets. + +> **Tip:** Enable **Point-in-time recovery** and set daily backups under *Database → Backups* for production projects. + +--- + +## 3. Apply the Platform Schema + +1. Open the Supabase dashboard and navigate to **SQL Editor**. +2. Paste the contents of [`database/init.sql`](../database/init.sql) and execute. +3. Confirm the script finishes without errors. It will: + - Install required extensions (`uuid-ossp`, `pgcrypto`, `citext`, `vector`, `postgresml`) + - Create `platform`, `telemetry`, and `analytics` schemas + - Provision tables for users, API keys, projects, sessions, and token accounting + - Register helpful RPC functions (`record_token_usage`, `grant_user_tokens`, `get_usage_overview`, `set_default_model`) + - Enable row-level security (RLS) with default policies + - Refresh the analytics materialized view for the first time + +4. Verify the tables exist under **Table Editor**. You should see the three schemas populated with objects. + +--- + +## 4. PostgresML Activation + +Supabase ships with PostgresML in dedicated instances. To enable it: + +1. Go to **Database → Extensions** and confirm `postgresml` is listed. +2. The SQL script already issues `CREATE EXTENSION IF NOT EXISTS postgresml;`. If it fails due to missing privileges, run the command manually as the `postgres` user. +3. Once loaded, test an inference call: + ```sql + SELECT postgresml.embed('bge-small-en-v1.5', ARRAY['CyberDevStudio makes agent workflows easy.']); + ``` +4. Store model metadata in `platform.models` for quick lookups. Example: + ```sql + INSERT INTO platform.models (name, provider, repo_url, context_size, cost_per_1k_tokens, metadata) + VALUES ('bge-small-en-v1.5', 'postgresml', 'https://huggingface.co/BAAI/bge-small-en-v1.5', 1024, 0.0001, + jsonb_build_object('type', 'embedding')) + ON CONFLICT (name) DO NOTHING; + ``` + +--- + +## 5. Configure Authentication + +- Head to **Authentication → Providers** and enable the flows you need (Email, OAuth, Magic Links, etc.). +- Update redirect URLs to include your local dev URL (e.g. `http://localhost:3000`) and production domain. +- In **Authentication → Policies**, ensure email confirmations are enabled for user projects. + +The `platform.users` table is designed to sync with Supabase Auth via [database triggers](https://supabase.com/docs/guides/auth/managing-user-data). Add the following SQL if you want automatic mirroring: + +```sql +CREATE OR REPLACE FUNCTION public.handle_new_user() +RETURNS trigger AS $$ +BEGIN + INSERT INTO platform.users (id, username, email, role) + VALUES (NEW.id, NEW.email, NEW.email, 'developer') + ON CONFLICT (id) DO NOTHING; + RETURN NEW; +END; +$$ LANGUAGE plpgsql SECURITY DEFINER; + +DROP TRIGGER IF EXISTS on_auth_user_created ON auth.users; +CREATE TRIGGER on_auth_user_created + AFTER INSERT ON auth.users + FOR EACH ROW EXECUTE FUNCTION public.handle_new_user(); +``` + +> Adjust the default role if you prefer new users to register as `viewer`. + +--- + +## 6. Grant Service Role Privileges + +To allow background workers (API, LLM server) to bypass RLS when required, connect with the `service_role` key and verify it has access: + +```sql +-- As the service_role user +SET ROLE postgres; +SELECT count(*) FROM platform.users; +``` + +The initialization script already grants schema usage and table privileges to `postgres`. If you use a custom service role, repeat the `GRANT` statements with that role name. + +--- + +## 7. Local Development Connection + +1. Install the Supabase CLI and link the project: + ```bash + supabase login + supabase link --project-ref + ``` +2. Pull secrets into `.env.local` automatically: + ```bash + supabase secrets pull --env-file .env.local + ``` +3. For offline development, start the local stack: + ```bash + supabase start + ``` + This spins up Postgres, Auth, Storage, and the Edge runtime using Docker. + +> Remember to apply `database/init.sql` to the local Postgres container as well using `supabase db reset` or `supabase db push`. + +--- + +## 8. Observability & Maintenance + +- **Backups:** Verify automated backups in Supabase or configure WAL-G for self-managed clusters. +- **Monitoring:** Forward metrics to Prometheus using the `telemetry.execution_metrics` table and OpenTelemetry exporters. +- **Vacuuming:** Schedule `VACUUM ANALYZE` during low-traffic windows, especially for `token_usage`. +- **Materialized views:** Automate `REFRESH MATERIALIZED VIEW CONCURRENTLY analytics.daily_token_summary;` via Supabase scheduled functions or external cron. + +--- + +## 9. Troubleshooting + +| Symptom | Possible Cause | Fix | +| ------- | -------------- | --- | +| `permission denied for schema platform` | RLS misconfiguration | Check that the caller uses the `service_role` key or add policies | +| `postgresml extension is not available` | Project running on shared plan | Upgrade to dedicated or open a support ticket | +| `duplicate key value violates unique constraint` | Running `database/init.sql` multiple times | The script uses `IF NOT EXISTS`—safe to re-run | +| `Function does not exist: auth.uid()` | Running outside Supabase | Replace `auth.uid()` with a session variable (e.g. `current_setting('request.jwt.claim.sub', true)`) + +--- + +## 10. Next Steps + +- Continue with the deployment options in [`docs/DEPLOYMENT.md`](./DEPLOYMENT.md) +- Follow the runtime checklist in [`docs/QUICK_START.md`](./QUICK_START.md) +- Keep the file inventory handy via [`docs/FILES_OVERVIEW.md`](./FILES_OVERVIEW.md) + +With the database foundation ready, you can integrate the Supabase client helpers from `src/lib/supabaseClient.js` and launch CyberDevStudio confidently. diff --git a/src/lib/supabaseClient.js b/src/lib/supabaseClient.js new file mode 100644 index 0000000..912427c --- /dev/null +++ b/src/lib/supabaseClient.js @@ -0,0 +1,347 @@ +import { createClient } from '@supabase/supabase-js'; + +const SUPABASE_URL = process.env.NEXT_PUBLIC_SUPABASE_URL || process.env.SUPABASE_URL || ''; +const SUPABASE_ANON_KEY = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || process.env.SUPABASE_ANON_KEY || ''; +const SUPABASE_SERVICE_ROLE_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY; + +let cachedClient; +let cachedServiceRoleClient; + +function ensureUrlAndKey(url, key, { serviceRole = false } = {}) { + if (!url) { + throw new Error('Supabase URL is missing. Set NEXT_PUBLIC_SUPABASE_URL or SUPABASE_URL.'); + } + if (!key) { + throw new Error( + serviceRole + ? 'Supabase service-role key is missing. Set SUPABASE_SERVICE_ROLE_KEY.' + : 'Supabase anon key is missing. Set NEXT_PUBLIC_SUPABASE_ANON_KEY or SUPABASE_ANON_KEY.' + ); + } +} + +/** + * Returns the singleton Supabase client configured for the `platform` schema. + * @param {object} [options] + * @param {import('@supabase/supabase-js').SupabaseClient} [options.override] + * @returns {import('@supabase/supabase-js').SupabaseClient} + */ +export function getSupabaseClient(options = {}) { + if (options.override) { + return options.override; + } + + if (!cachedClient) { + ensureUrlAndKey(SUPABASE_URL, SUPABASE_ANON_KEY); + cachedClient = createClient(SUPABASE_URL, SUPABASE_ANON_KEY, { + auth: { + persistSession: true, + autoRefreshToken: true, + }, + db: { + schema: 'platform', + }, + }); + } + + return cachedClient; +} + +/** + * Returns a service-role Supabase client (no session persistence). + * @returns {import('@supabase/supabase-js').SupabaseClient} + */ +export function getServiceRoleClient() { + ensureUrlAndKey(SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, { serviceRole: true }); + + if (!cachedServiceRoleClient) { + cachedServiceRoleClient = createClient(SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, { + auth: { + persistSession: false, + autoRefreshToken: false, + }, + db: { + schema: 'platform', + }, + }); + } + + return cachedServiceRoleClient; +} + +export const supabaseClient = getSupabaseClient(); + +async function resolveClient({ serviceRole = false, client } = {}) { + if (client) { + return client; + } + return serviceRole ? getServiceRoleClient() : getSupabaseClient(); +} + +/** Auth -------------------------------------------------------------------- */ +export async function signInWithEmail({ email, password }) { + const client = await resolveClient(); + const { data, error } = await client.auth.signInWithPassword({ email, password }); + if (error) throw error; + return data.user; +} + +export async function signInWithOtp({ email }) { + const client = await resolveClient(); + const { error } = await client.auth.signInWithOtp({ email }); + if (error) throw error; + return true; +} + +export async function signOut() { + const client = await resolveClient(); + const { error } = await client.auth.signOut(); + if (error) throw error; + return true; +} + +export async function getCurrentUser() { + const client = await resolveClient(); + const { data, error } = await client.auth.getUser(); + if (error) throw error; + return data.user ?? null; +} + +export async function refreshSession() { + const client = await resolveClient(); + const { data, error } = await client.auth.refreshSession(); + if (error) throw error; + return data.session; +} + +/** Profiles & users -------------------------------------------------------- */ +export async function upsertProfile(payload) { + const client = await resolveClient(); + const { data, error } = await client.from('users').upsert(payload).select().single(); + if (error) throw error; + return data; +} + +export async function getUserBalance(userId, { serviceRole = false } = {}) { + const client = await resolveClient({ serviceRole }); + const { data, error } = await client + .from('users') + .select('id, balance_tokens, role, metadata') + .eq('id', userId) + .maybeSingle(); + if (error) throw error; + return data; +} + +export async function grantTokens(userId, amount, reason = null) { + const client = await resolveClient({ serviceRole: true }); + return callRpc('grant_user_tokens', { + p_user_id: userId, + p_amount: amount, + p_reason: reason, + }, { client }); +} + +export async function fetchUsageOverview(userId, limit = 50) { + return callRpc('get_usage_overview', { + p_user_id: userId, + p_limit: limit, + }); +} + +/** Projects ---------------------------------------------------------------- */ +export async function listProjects({ ownerId } = {}) { + const client = await resolveClient(); + let query = client + .from('projects') + .select('id, name, description, visibility, metadata, created_at, updated_at', { count: 'exact' }) + .order('created_at', { ascending: false }); + + if (ownerId) { + query = query.eq('owner_id', ownerId); + } + + const { data, error } = await query; + if (error) throw error; + return data; +} + +export async function createProject(payload) { + const client = await resolveClient(); + const { data, error } = await client.from('projects').insert(payload).select().single(); + if (error) throw error; + return data; +} + +export async function updateProject(id, patch) { + const client = await resolveClient(); + const { data, error } = await client.from('projects').update(patch).eq('id', id).select().single(); + if (error) throw error; + return data; +} + +export async function deleteProject(id) { + const client = await resolveClient(); + const { error } = await client.from('projects').delete().eq('id', id); + if (error) throw error; + return true; +} + +/** Models ------------------------------------------------------------------ */ +export async function listModels({ onlyActive = false } = {}) { + const client = await resolveClient(); + let query = client + .from('models') + .select('id, name, provider, context_size, cost_per_1k_tokens, is_default, metadata') + .order('is_default', { ascending: false }) + .order('name'); + if (onlyActive) { + query = query.eq('metadata->>status', 'active'); + } + const { data, error } = await query; + if (error) throw error; + return data; +} + +export async function setDefaultModel(modelId) { + return callRpc('set_default_model', { p_model_id: modelId }, { serviceRole: true }); +} + +export async function recordTokenUsage({ userId, modelName, requestId, promptTokens, completionTokens, metadata = {} }) { + return callRpc('record_token_usage', { + p_user_id: userId, + p_model_name: modelName, + p_request_id: requestId, + p_prompt_tokens: promptTokens, + p_completion_tokens: completionTokens, + p_metadata: metadata, + }, { serviceRole: true }); +} + +export async function fetchTokenUsage({ userId, limit = 100 } = {}) { + const client = await resolveClient(); + let query = client + .from('token_usage') + .select('id, occurred_at, prompt_tokens, completion_tokens, cost_tokens, metadata, model_id') + .order('occurred_at', { ascending: false }) + .limit(limit); + + if (userId) { + query = query.eq('user_id', userId); + } + + const { data, error } = await query; + if (error) throw error; + return data; +} + +/** Sessions ---------------------------------------------------------------- */ +export async function getProjectSessions(projectId) { + const client = await resolveClient(); + const { data, error } = await client + .from('sessions') + .select('id, status, model_id, last_activity, metadata, created_at, user_id') + .eq('project_id', projectId) + .order('last_activity', { ascending: false }); + if (error) throw error; + return data; +} + +export async function closeSession(sessionId) { + const client = await resolveClient(); + const { data, error } = await client + .from('sessions') + .update({ status: 'closed', updated_at: new Date().toISOString() }) + .eq('id', sessionId) + .select() + .single(); + if (error) throw error; + return data; +} + +/** Telemetry --------------------------------------------------------------- */ +export async function logAgentEvent(payload) { + const client = await resolveClient({ serviceRole: true }); + const { data, error } = await client + .schema('telemetry') + .from('agent_events') + .insert(payload) + .select() + .single(); + if (error) throw error; + return data; +} + +export async function fetchExecutionMetrics({ projectId, limit = 50 } = {}) { + const client = await resolveClient({ serviceRole: true }); + let query = client + .schema('telemetry') + .from('execution_metrics') + .select('id, collected_at, cpu_percent, memory_mb, duration_ms, outcome, metadata, project_id') + .order('collected_at', { ascending: false }) + .limit(limit); + if (projectId) { + query = query.eq('project_id', projectId); + } + const { data, error } = await query; + if (error) throw error; + return data; +} + +/** Analytics --------------------------------------------------------------- */ +export async function refreshAnalyticsSummary() { + return callRpc('refresh_daily_token_summary', {}, { serviceRole: true }); +} + +export async function fetchDailyTokenSummary({ day, userId, limit = 30 } = {}) { + const client = await resolveClient({ serviceRole: true }); + let query = client + .schema('analytics') + .from('daily_token_summary') + .select('day, user_id, prompt_tokens, completion_tokens, cost_tokens') + .order('day', { ascending: false }) + .limit(limit); + if (day) { + query = query.eq('day', day); + } + if (userId) { + query = query.eq('user_id', userId); + } + const { data, error } = await query; + if (error) throw error; + return data; +} + +/** Utilities --------------------------------------------------------------- */ +export async function callRpc(fn, params = {}, options = {}) { + const client = await resolveClient(options); + const { data, error } = await client.rpc(fn, params); + if (error) throw error; + return data; +} + +export async function withServiceRole(fn) { + const client = await resolveClient({ serviceRole: true }); + return fn(client); +} + +export function resetSupabaseClients() { + cachedClient = undefined; + cachedServiceRoleClient = undefined; +} + +/** + * Convenience helper that wraps a fetch call with Supabase auth header if a session exists. + * Useful for calling CyberDevStudio APIs that require authenticated Supabase tokens. + */ +export async function withAuthFetch(input, init = {}) { + const client = await resolveClient(); + const session = await client.auth.getSession(); + const headers = new Headers(init.headers || {}); + if (session?.data?.session?.access_token) { + headers.set('Authorization', `Bearer ${session.data.session.access_token}`); + } + return fetch(input, { ...init, headers }); +} + +export default supabaseClient;