Compare commits
81 Commits
main
...
ba30d14041
| Author | SHA1 | Date | |
|---|---|---|---|
| ba30d14041 | |||
| 1a6ccc9aaa | |||
| 4c0b4fab48 | |||
| 87ea715c2a | |||
| c5c66ccbfc | |||
| 56f2c8d873 | |||
| 2d3b546bf9 | |||
| f013fc1a65 | |||
| a7ecd76e0a | |||
| f6a922921b | |||
| 0456606dc1 | |||
| b4fa3bf12a | |||
| 05be7f9ad0 | |||
| 28ad6c2b0b | |||
| 296a4f1950 | |||
| 361e653d37 | |||
| 430f4a1132 | |||
| 30f2bd0084 | |||
| 73d0b7017b | |||
| 8a03abac75 | |||
| def45dd0c6 | |||
| 56f6572086 | |||
| e6bb220979 | |||
| ad322543d0 | |||
| d3361dba59 | |||
| 4e26ec9884 | |||
| a732697855 | |||
| 44f84800e9 | |||
| 5c7753c0b8 | |||
| 91846a201e | |||
| 634e5119e1 | |||
| 84701730da | |||
| a657d17995 | |||
| a06e88d58b | |||
| 2145d36744 | |||
| b4bef65079 | |||
| 54cb94c30f | |||
| 54762f491c | |||
| 917d8fbeaf | |||
| 8efbb2f9e5 | |||
| d98498b705 | |||
| c70c6594de | |||
| a592319e3c | |||
| dda8264057 | |||
| 3313047ff1 | |||
| 36ffe2713f | |||
| a40df03521 | |||
| d2d2d29c34 | |||
| 8323bf68d2 | |||
| f63089aea8 | |||
| cbbb83a706 | |||
| 39f8e117e8 | |||
| bdef6c6b9f | |||
| 7f17cb8fee | |||
| ad22242ecc | |||
| 1112a1971b | |||
| c53295dfee | |||
| 745a33e05d | |||
| 83b680b2ea | |||
| 3e62a0f6ef | |||
| d312bdf968 | |||
| 7d3d3560ad | |||
| 404135a16a | |||
| 85a14f4c05 | |||
| a33b46b4af | |||
| eae0190184 | |||
| e6f5ba53bd | |||
| 8f480cd76f | |||
| 31361e2dcf | |||
| ae4ea46be2 | |||
| fab85b14a6 | |||
| a74b88a45c | |||
| 89fe54cc07 | |||
| 9ff790554d | |||
| ecc9339ede | |||
| fd8d0d637d | |||
| 35ae97d0a6 | |||
| e78ed687a6 | |||
| 8bdd308ea0 | |||
| 27c4f14233 | |||
| dce5e9598b |
+5
-2
@@ -38,8 +38,11 @@ INFOQUEST_API_KEY=your-infoquest-api-key
|
||||
# GitHub API Token
|
||||
# GITHUB_TOKEN=your-github-token
|
||||
|
||||
# Database (only needed when config.yaml has database.backend: postgres)
|
||||
# DATABASE_URL=postgresql://deerflow:password@localhost:5432/deerflow
|
||||
# Database (Stage 0+ default; required when config.yaml has database.backend: postgres)
|
||||
# Local dev — start with: docker compose -f docker/docker-compose-dev.yaml up -d postgres
|
||||
DATABASE_URL=postgresql+asyncpg://deerflow:deerflow_dev@localhost:5432/deerflow
|
||||
# Remote RDS example:
|
||||
# DATABASE_URL=postgresql+asyncpg://USER:PASSWORD@HOST:5432/DBNAME
|
||||
#
|
||||
# WECOM_BOT_ID=your-wecom-bot-id
|
||||
# WECOM_BOT_SECRET=your-wecom-bot-secret
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
name: Postgres Tests
|
||||
|
||||
# Stage 0 PR1 · runs the @pytest.mark.postgres subset against a real
|
||||
# Postgres backend (testcontainers spawns postgres:16-alpine on the
|
||||
# GitHub runner's docker daemon).
|
||||
#
|
||||
# Kept as a separate workflow from `Unit Tests` so:
|
||||
# - the existing fast unit test loop is unchanged
|
||||
# - PG tests can fail-soft during early Stage 0 rollout if needed
|
||||
# (set continue-on-error: true on the run step)
|
||||
# - infra cost is opt-in for forks
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ 'main' ]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened, ready_for_review]
|
||||
|
||||
concurrency:
|
||||
group: postgres-tests-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
backend-postgres-tests:
|
||||
if: github.event.pull_request.draft == false
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Verify Docker daemon (testcontainers needs it)
|
||||
run: docker version
|
||||
|
||||
- name: Install backend dependencies (with postgres-test extra)
|
||||
working-directory: backend
|
||||
run: uv sync --group dev --extra postgres-test
|
||||
|
||||
- name: Run @pytest.mark.postgres tests
|
||||
working-directory: backend
|
||||
# -m postgres selects only postgres-tagged tests; testcontainers
|
||||
# spins up postgres:16-alpine as the fixture's session-scoped
|
||||
# container. ~5-10s container startup + per-test DB CREATE/DROP.
|
||||
run: PYTHONPATH=. uv run pytest -m postgres -v
|
||||
@@ -60,3 +60,6 @@ config.yaml.bak
|
||||
/frontend/playwright-report/
|
||||
.gstack/
|
||||
.worktrees
|
||||
skills/gstack
|
||||
skills/superpowers
|
||||
CLAUDE.md
|
||||
|
||||
@@ -31,6 +31,7 @@ help:
|
||||
@echo " make start-daemon - Start prod services in background (daemon mode)"
|
||||
@echo " make stop - Stop all running services"
|
||||
@echo " make clean - Clean up processes and temporary files"
|
||||
@echo " make migrate-paths - Migrate legacy users/ tree into workspaces/ layout (DRY_RUN=1 to preview)"
|
||||
@echo ""
|
||||
@echo "Docker Production Commands:"
|
||||
@echo " make up - Build and start production Docker services (localhost:2026)"
|
||||
@@ -144,6 +145,14 @@ clean: stop
|
||||
@-rm -rf logs/*.log 2>/dev/null || true
|
||||
@echo "✓ Cleanup complete"
|
||||
|
||||
# Lift legacy per-user paths into the per-workspace layout (PR6).
|
||||
# Pass DRY_RUN=1 to log the migration plan without writing.
|
||||
# DEFAULT_WORKSPACE=<wid> claims un-assigned users (defaults to legacy_workspace).
|
||||
migrate-paths:
|
||||
@cd backend && PYTHONPATH=. uv run python scripts/migrate_paths_to_workspace.py \
|
||||
$(if $(filter 1 true,$(DRY_RUN)),--dry-run) \
|
||||
$(if $(DEFAULT_WORKSPACE),--default-workspace $(DEFAULT_WORKSPACE))
|
||||
|
||||
# ==========================================
|
||||
# Docker Development Commands
|
||||
# ==========================================
|
||||
|
||||
@@ -202,6 +202,38 @@ That prompt is intended for coding agents. It tells the agent to clone the repo
|
||||
|
||||
</details>
|
||||
|
||||
3. **Database backend (Stage 0+ defaults to Postgres)**
|
||||
|
||||
`config.example.yaml` ships with `database.backend: postgres` and `postgres_url: $DATABASE_URL`. Set `DATABASE_URL` in `.env`:
|
||||
|
||||
```bash
|
||||
DATABASE_URL=postgresql+asyncpg://deerflow:deerflow_dev@localhost:5432/deerflow
|
||||
```
|
||||
|
||||
Start the local Postgres dev container:
|
||||
|
||||
```bash
|
||||
docker compose -f docker/docker-compose-dev.yaml up -d postgres
|
||||
```
|
||||
|
||||
Or point `DATABASE_URL` at a remote RDS / Cloud SQL instance.
|
||||
|
||||
`make doctor` will report the configured backend, attempt an asyncpg connection, and surface actionable fix hints. `make dev` preflights Postgres reachability before starting services and aborts if `DATABASE_URL` is unreachable.
|
||||
|
||||
<details>
|
||||
<summary>Offline dev (SQLite fallback)</summary>
|
||||
|
||||
If you prefer no Postgres, edit `config.yaml`:
|
||||
|
||||
```yaml
|
||||
database:
|
||||
backend: sqlite
|
||||
sqlite_dir: .deer-flow/data
|
||||
```
|
||||
|
||||
SQLite is preserved as a valid backend for offline development. RLS / multi-node features (Stage 2+) require Postgres.
|
||||
</details>
|
||||
|
||||
### Running the Application
|
||||
|
||||
#### Deployment Sizing
|
||||
|
||||
@@ -108,6 +108,23 @@ async def _ensure_admin_user(app: FastAPI) -> None:
|
||||
|
||||
admin_id = str(row.id)
|
||||
|
||||
# Stage 0 PR4 backfill: pre-PR4 admins have no default_workspace_id.
|
||||
# Create their personal workspace + owner membership on next boot so
|
||||
# they can log in and pass the workspace gate without hand-rolling
|
||||
# SQL. Idempotent — ensure_default_workspace short-circuits when the
|
||||
# column is already set.
|
||||
try:
|
||||
admin_user = await provider.get_user(admin_id)
|
||||
if admin_user is not None and not admin_user.default_workspace_id:
|
||||
from app.gateway.routers.auth import ensure_default_workspace
|
||||
|
||||
ws_id = await ensure_default_workspace(admin_user)
|
||||
logger.info("Backfilled default workspace %s for admin %s", ws_id, admin_id)
|
||||
except Exception:
|
||||
# Don't fail startup if backfill stumbles — the user can still
|
||||
# log in (login_local calls the same helper on its hot path).
|
||||
logger.exception("Admin workspace backfill failed (non-fatal)")
|
||||
|
||||
# LangGraph store orphan migration — non-fatal.
|
||||
# This covers the "no-auth → with-auth" upgrade path for users
|
||||
# whose existing LangGraph thread metadata has no user_id set.
|
||||
@@ -158,6 +175,34 @@ async def _migrate_orphaned_threads(store, admin_user_id: str) -> int:
|
||||
return migrated
|
||||
|
||||
|
||||
def _check_path_migration_pending(app: FastAPI) -> None:
|
||||
"""Warn the operator if the PR4 legacy user-isolation layout still has content.
|
||||
|
||||
PR6 routes every new write into ``{base_dir}/workspaces/{wid}/...`` via
|
||||
``Paths``. Pre-PR6 installations have data at
|
||||
``{base_dir}/users/{uid}/...`` that needs ``make migrate-paths`` to lift
|
||||
it under a workspace. We emit a warning at boot rather than crashing so
|
||||
the gateway keeps serving (reads from the legacy tree still work via the
|
||||
user_id branch of ``Paths.thread_dir``), but with a loud signal to run
|
||||
the migration script.
|
||||
"""
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
legacy_users = get_paths().base_dir / "users"
|
||||
if not legacy_users.exists():
|
||||
return
|
||||
try:
|
||||
has_content = any(legacy_users.iterdir())
|
||||
except OSError:
|
||||
# Permission or transient FS issue — don't escalate; lifespan must succeed.
|
||||
return
|
||||
if has_content:
|
||||
logger.warning(
|
||||
"Legacy per-user layout detected at %s. Run `make migrate-paths` to lift it under the per-workspace layout (PR6).",
|
||||
legacy_users,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
"""Application lifespan handler."""
|
||||
@@ -174,6 +219,8 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
config = get_gateway_config()
|
||||
logger.info(f"Starting API Gateway on {config.host}:{config.port}")
|
||||
|
||||
_check_path_migration_pending(app)
|
||||
|
||||
# Initialize LangGraph runtime components (StreamBridge, RunManager, checkpointer, store)
|
||||
async with langgraph_runtime(app):
|
||||
logger.info("LangGraph runtime initialised")
|
||||
|
||||
@@ -21,6 +21,7 @@ class AuthErrorCode(StrEnum):
|
||||
PROVIDER_NOT_FOUND = "provider_not_found"
|
||||
NOT_AUTHENTICATED = "not_authenticated"
|
||||
SYSTEM_ALREADY_INITIALIZED = "system_already_initialized"
|
||||
WORKSPACE_REQUIRED = "workspace_required"
|
||||
|
||||
|
||||
class TokenError(StrEnum):
|
||||
@@ -29,6 +30,7 @@ class TokenError(StrEnum):
|
||||
EXPIRED = "expired"
|
||||
INVALID_SIGNATURE = "invalid_signature"
|
||||
MALFORMED = "malformed"
|
||||
WORKSPACE_MISSING = "workspace_missing"
|
||||
|
||||
|
||||
class AuthErrorResponse(BaseModel):
|
||||
@@ -42,4 +44,6 @@ def token_error_to_code(err: TokenError) -> AuthErrorCode:
|
||||
"""Map TokenError to AuthErrorCode — single source of truth."""
|
||||
if err == TokenError.EXPIRED:
|
||||
return AuthErrorCode.TOKEN_EXPIRED
|
||||
if err == TokenError.WORKSPACE_MISSING:
|
||||
return AuthErrorCode.WORKSPACE_REQUIRED
|
||||
return AuthErrorCode.TOKEN_INVALID
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""JWT token creation and verification."""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import jwt
|
||||
from pydantic import BaseModel
|
||||
@@ -10,30 +11,51 @@ from app.gateway.auth.errors import TokenError
|
||||
|
||||
|
||||
class TokenPayload(BaseModel):
|
||||
"""JWT token payload."""
|
||||
"""JWT token payload.
|
||||
|
||||
`wid` / `role` were added in Stage 0 PR4 and are optional at the
|
||||
model level so legacy 4-field tokens still parse into a value —
|
||||
callers (middleware, decode_token) decide what "missing wid" means.
|
||||
Post-PR4 production tokens always carry both fields.
|
||||
"""
|
||||
|
||||
sub: str # user_id
|
||||
wid: str | None = None # workspace_id (Stage 0 PR4)
|
||||
role: str | None = None # owner / admin / member (Stage 0 PR4)
|
||||
exp: datetime
|
||||
iat: datetime | None = None
|
||||
ver: int = 0 # token_version — must match User.token_version
|
||||
|
||||
|
||||
def create_access_token(user_id: str, expires_delta: timedelta | None = None, token_version: int = 0) -> str:
|
||||
def create_access_token(
|
||||
user_id: str,
|
||||
expires_delta: timedelta | None = None,
|
||||
token_version: int = 0,
|
||||
*,
|
||||
workspace_id: str | None = None,
|
||||
role: str | None = None,
|
||||
) -> str:
|
||||
"""Create a JWT access token.
|
||||
|
||||
Args:
|
||||
user_id: The user's UUID as string
|
||||
expires_delta: Optional custom expiry, defaults to 7 days
|
||||
token_version: User's current token_version for invalidation
|
||||
user_id: The user's UUID as string.
|
||||
expires_delta: Optional custom expiry, defaults to 7 days.
|
||||
token_version: User's current token_version for invalidation.
|
||||
workspace_id: Optional active workspace id; encoded as the ``wid`` claim.
|
||||
role: Optional workspace role; encoded as the ``role`` claim.
|
||||
|
||||
Returns:
|
||||
Encoded JWT string
|
||||
Encoded JWT string.
|
||||
"""
|
||||
config = get_auth_config()
|
||||
expiry = expires_delta or timedelta(days=config.token_expiry_days)
|
||||
|
||||
now = datetime.now(UTC)
|
||||
payload = {"sub": user_id, "exp": now + expiry, "iat": now, "ver": token_version}
|
||||
payload: dict[str, Any] = {"sub": user_id, "exp": now + expiry, "iat": now, "ver": token_version}
|
||||
if workspace_id is not None:
|
||||
payload["wid"] = workspace_id
|
||||
if role is not None:
|
||||
payload["role"] = role
|
||||
return jwt.encode(payload, config.jwt_secret, algorithm="HS256")
|
||||
|
||||
|
||||
@@ -46,10 +68,20 @@ def decode_token(token: str) -> TokenPayload | TokenError:
|
||||
config = get_auth_config()
|
||||
try:
|
||||
payload = jwt.decode(token, config.jwt_secret, algorithms=["HS256"])
|
||||
return TokenPayload(**payload)
|
||||
except jwt.ExpiredSignatureError:
|
||||
return TokenError.EXPIRED
|
||||
except jwt.InvalidSignatureError:
|
||||
return TokenError.INVALID_SIGNATURE
|
||||
except jwt.PyJWTError:
|
||||
return TokenError.MALFORMED
|
||||
|
||||
# Reject legacy pre-PR4 tokens that lack the wid claim. Reported as
|
||||
# WORKSPACE_MISSING (not MALFORMED) so middleware can surface a
|
||||
# specific 401 telling the frontend to re-issue via /select-workspace.
|
||||
if "wid" not in payload or payload.get("wid") is None:
|
||||
return TokenError.WORKSPACE_MISSING
|
||||
|
||||
try:
|
||||
return TokenPayload(**payload)
|
||||
except Exception:
|
||||
return TokenError.MALFORMED
|
||||
|
||||
@@ -31,6 +31,12 @@ class User(BaseModel):
|
||||
needs_setup: bool = Field(default=False, description="True for auto-created admin until setup completes")
|
||||
token_version: int = Field(default=0, description="Incremented on password change to invalidate old JWTs")
|
||||
|
||||
# Workspace linkage (Stage 0 PR4)
|
||||
default_workspace_id: str | None = Field(
|
||||
default=None,
|
||||
description="The workspace the user lands in by default after login. NULL → /select-workspace.",
|
||||
)
|
||||
|
||||
|
||||
class UserResponse(BaseModel):
|
||||
"""Response model for user info endpoint."""
|
||||
@@ -39,3 +45,38 @@ class UserResponse(BaseModel):
|
||||
email: str
|
||||
system_role: Literal["admin", "user"]
|
||||
needs_setup: bool = False
|
||||
|
||||
|
||||
class UserMeWorkspace(BaseModel):
|
||||
"""One workspace entry in ``GET /auth/me`` (Stage 0 PR4)."""
|
||||
|
||||
id: str
|
||||
name: str
|
||||
slug: str
|
||||
role: str
|
||||
|
||||
|
||||
class UserMeResponse(BaseModel):
|
||||
"""Response model for ``GET /auth/me`` — extends UserResponse with workspaces."""
|
||||
|
||||
id: str
|
||||
email: str
|
||||
system_role: Literal["admin", "user"]
|
||||
needs_setup: bool = False
|
||||
default_workspace_id: str | None = None
|
||||
workspaces: list[UserMeWorkspace] = []
|
||||
|
||||
|
||||
class ActiveWorkspace(BaseModel):
|
||||
"""Lightweight workspace proxy injected into the request-scoped contextvar.
|
||||
|
||||
Implements the structural ``CurrentWorkspace`` protocol expected by
|
||||
``deerflow.runtime.workspace_context``: only ``.id`` (str) and
|
||||
``.role`` (str) are required. We intentionally do *not* embed the
|
||||
full ``WorkspaceRow`` here — the middleware needs to set the
|
||||
contextvar on every request and an extra DB lookup just to populate
|
||||
a name/slug we don't use yet would be wasted work.
|
||||
"""
|
||||
|
||||
id: str
|
||||
role: str
|
||||
|
||||
@@ -46,6 +46,7 @@ class SQLiteUserRepository(UserRepository):
|
||||
oauth_id=row.oauth_id,
|
||||
needs_setup=row.needs_setup,
|
||||
token_version=row.token_version,
|
||||
default_workspace_id=row.default_workspace_id,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -60,6 +61,7 @@ class SQLiteUserRepository(UserRepository):
|
||||
oauth_id=user.oauth_id,
|
||||
needs_setup=user.needs_setup,
|
||||
token_version=user.token_version,
|
||||
default_workspace_id=user.default_workspace_id,
|
||||
)
|
||||
|
||||
# ── CRUD ──────────────────────────────────────────────────────────
|
||||
@@ -106,6 +108,7 @@ class SQLiteUserRepository(UserRepository):
|
||||
row.oauth_id = user.oauth_id
|
||||
row.needs_setup = user.needs_setup
|
||||
row.token_version = user.token_version
|
||||
row.default_workspace_id = user.default_workspace_id
|
||||
await session.commit()
|
||||
return user
|
||||
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Workspace slug helpers for the registration / initialize flow.
|
||||
|
||||
Stage 0 PR4 T4.10. Two responsibilities:
|
||||
|
||||
1. ``auto_slug_from_email(email)`` — pure transform from an email's
|
||||
local part to a base slug that matches the schema's
|
||||
``^[a-z0-9](-?[a-z0-9])*$`` pattern.
|
||||
2. ``next_available_slug(base, exists_check=...)`` — collision walker
|
||||
that appends ``-2``, ``-3``, … until ``exists_check`` reports the
|
||||
candidate is free. Kept separate from ``auto_slug_from_email`` so
|
||||
the pure function can be tested without a database.
|
||||
|
||||
Lives in the auth package (not in ``persistence``) because the input
|
||||
is the user's email — a registration-time concept that doesn't belong
|
||||
in a generic ``WorkspaceRepository``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import secrets
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
# Mirror the schema's slug rules from
|
||||
# ``deerflow.persistence.workspace.sql`` so callers of this module
|
||||
# never need to import private constants from persistence.
|
||||
_SLUG_MIN_LEN = 3
|
||||
_SLUG_MAX_LEN = 32
|
||||
|
||||
|
||||
def auto_slug_from_email(email: str) -> str:
|
||||
"""Map an email to a deterministic, schema-valid base slug.
|
||||
|
||||
Algorithm (from workspace-schema-design §3.1):
|
||||
|
||||
1. Take the local part (before ``@``).
|
||||
2. Replace ``+``, ``_``, ``.`` with ``-`` and lowercase.
|
||||
3. Strip everything that isn't ``[a-z0-9-]``.
|
||||
4. Collapse repeated ``-``; strip leading/trailing ``-``.
|
||||
5. Clamp to 32 chars.
|
||||
6. If the result is shorter than the schema minimum (3 chars) or
|
||||
empty, fall back to ``user-{token_hex(4)}`` so we always emit
|
||||
a valid slug.
|
||||
|
||||
The returned slug is the *base* — callers must run it through
|
||||
:func:`next_available_slug` before persisting to handle collisions.
|
||||
"""
|
||||
local = email.split("@", 1)[0]
|
||||
local = re.sub(r"[+_.]", "-", local).lower()
|
||||
local = re.sub(r"[^a-z0-9-]", "", local)
|
||||
local = re.sub(r"-+", "-", local).strip("-")
|
||||
slug = local[:_SLUG_MAX_LEN]
|
||||
if len(slug) < _SLUG_MIN_LEN:
|
||||
return f"user-{secrets.token_hex(4)}"
|
||||
return slug
|
||||
|
||||
|
||||
async def next_available_slug(
|
||||
base: str,
|
||||
*,
|
||||
exists_check: Callable[[str], Awaitable[bool]],
|
||||
) -> str:
|
||||
"""Return the first of ``base``, ``base-2``, ``base-3``, … that ``exists_check`` reports free.
|
||||
|
||||
Caller-supplied ``exists_check`` is awaited once per candidate so
|
||||
we can swap in a repository's ``get_by_slug`` without coupling
|
||||
this module to persistence imports.
|
||||
|
||||
When ``base + '-N'`` would exceed the 32-char schema limit, the
|
||||
base is truncated before the suffix is appended. The walker never
|
||||
returns an over-long slug.
|
||||
"""
|
||||
if not await exists_check(base):
|
||||
return base
|
||||
|
||||
n = 2
|
||||
while True:
|
||||
suffix = f"-{n}"
|
||||
max_base_len = _SLUG_MAX_LEN - len(suffix)
|
||||
candidate = f"{base[:max_base_len]}{suffix}"
|
||||
if not await exists_check(candidate):
|
||||
return candidate
|
||||
n += 1
|
||||
@@ -17,9 +17,11 @@ from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse
|
||||
from app.gateway.auth.models import ActiveWorkspace
|
||||
from app.gateway.authz import _ALL_PERMISSIONS, AuthContext
|
||||
from app.gateway.internal_auth import INTERNAL_AUTH_HEADER_NAME, get_internal_user, is_valid_internal_auth_token
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
from deerflow.runtime.workspace_context import reset_current_workspace, set_current_workspace
|
||||
|
||||
# Paths that never require authentication.
|
||||
_PUBLIC_PATH_PREFIXES: tuple[str, ...] = (
|
||||
@@ -119,8 +121,22 @@ class AuthMiddleware(BaseHTTPMiddleware):
|
||||
# JWT-decode + DB-lookup pipeline a second time per request).
|
||||
request.state.user = user
|
||||
request.state.auth = AuthContext(user=user, permissions=_ALL_PERMISSIONS)
|
||||
token = set_current_user(user)
|
||||
user_token = set_current_user(user)
|
||||
|
||||
# Inject workspace contextvar from the JWT's wid/role claims.
|
||||
# decode_token has already rejected legacy no-wid tokens upstream,
|
||||
# so by the time we get here payload.wid is guaranteed non-None
|
||||
# for cookie-authenticated requests. Internal-auth requests skip
|
||||
# the workspace contextvar (they don't have a workspace scope —
|
||||
# the internal user is a system actor).
|
||||
ws_token = None
|
||||
payload = getattr(request.state, "auth_payload", None)
|
||||
if payload is not None and payload.wid is not None:
|
||||
ws_token = set_current_workspace(ActiveWorkspace(id=payload.wid, role=payload.role or "owner"))
|
||||
|
||||
try:
|
||||
return await call_next(request)
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
if ws_token is not None:
|
||||
reset_current_workspace(ws_token)
|
||||
reset_current_user(user_token)
|
||||
|
||||
@@ -268,24 +268,27 @@ def require_permission(
|
||||
|
||||
# Owner check for thread-specific resources.
|
||||
#
|
||||
# 2.0-rc moved thread metadata into the SQL persistence layer
|
||||
# (``threads_meta`` table). We verify ownership via
|
||||
# ``ThreadMetaStore.check_access``: it returns True for
|
||||
# missing rows (untracked legacy thread) and for rows whose
|
||||
# ``user_id`` is NULL (shared / pre-auth data), so this is
|
||||
# strict-deny rather than strict-allow — only an *existing*
|
||||
# row with a *different* user_id triggers 404.
|
||||
# PR6: ``check_access`` now takes ``workspace_id`` as the third
|
||||
# positional argument; cross-workspace always denies regardless
|
||||
# of user_id match. We pull workspace_id from the contextvar
|
||||
# AuthMiddleware sets per request (and fall back to "default"
|
||||
# in no-auth dev mode so smoke flows keep working). Failures
|
||||
# convert to **404**, not 403, so the response never leaks the
|
||||
# existence of a thread that belongs to a different tenant.
|
||||
if owner_check:
|
||||
thread_id = kwargs.get("thread_id")
|
||||
if thread_id is None:
|
||||
raise ValueError("require_permission with owner_check=True requires 'thread_id' parameter")
|
||||
|
||||
from app.gateway.deps import get_thread_store
|
||||
from deerflow.runtime.workspace_context import get_effective_workspace_id
|
||||
|
||||
workspace_id = get_effective_workspace_id()
|
||||
thread_store = get_thread_store(request)
|
||||
allowed = await thread_store.check_access(
|
||||
thread_id,
|
||||
str(auth.user.id),
|
||||
workspace_id,
|
||||
require_existing=require_existing,
|
||||
)
|
||||
if not allowed:
|
||||
|
||||
@@ -220,6 +220,10 @@ async def get_current_user_from_request(request: Request):
|
||||
detail=AuthErrorResponse(code=AuthErrorCode.TOKEN_INVALID, message="Token revoked (password changed)").model_dump(),
|
||||
)
|
||||
|
||||
# Stash decoded payload on request.state so AuthMiddleware can read
|
||||
# wid/role for the workspace contextvar without a second decode.
|
||||
request.state.auth_payload = payload
|
||||
|
||||
return user
|
||||
|
||||
|
||||
|
||||
@@ -15,11 +15,60 @@ from app.gateway.auth import (
|
||||
)
|
||||
from app.gateway.auth.config import get_auth_config
|
||||
from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse
|
||||
from app.gateway.auth.models import UserMeResponse, UserMeWorkspace
|
||||
from app.gateway.auth.workspace_slug import auto_slug_from_email, next_available_slug
|
||||
from app.gateway.csrf_middleware import is_secure_request
|
||||
from app.gateway.deps import get_current_user_from_request, get_local_provider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def ensure_default_workspace(user) -> str:
|
||||
"""Create the user's personal workspace + owner membership, set default_workspace_id.
|
||||
|
||||
Returns the new workspace id. Idempotent for users who already
|
||||
have a default_workspace_id (used by both the registration flow
|
||||
and the lifespan backfill in app.py).
|
||||
"""
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.workspace import WorkspaceRepository
|
||||
from deerflow.persistence.workspace.sql import SLUG_BLACKLIST
|
||||
from deerflow.persistence.workspace_membership import WorkspaceMembershipRepository
|
||||
|
||||
if user.default_workspace_id:
|
||||
return user.default_workspace_id
|
||||
|
||||
sf = get_session_factory()
|
||||
ws_repo = WorkspaceRepository(sf)
|
||||
m_repo = WorkspaceMembershipRepository(sf)
|
||||
|
||||
base_slug = auto_slug_from_email(user.email)
|
||||
|
||||
async def slug_exists(s: str) -> bool:
|
||||
# Treat blacklisted slugs as "taken" so the walker skips them
|
||||
# instead of letting WorkspaceRepository.create raise after a
|
||||
# successful slug computation (the user picked a reserved name
|
||||
# like "admin@example.com" → base slug "admin").
|
||||
if s in SLUG_BLACKLIST:
|
||||
return True
|
||||
return (await ws_repo.get_by_slug(s)) is not None
|
||||
|
||||
unique_slug = await next_available_slug(base_slug, exists_check=slug_exists)
|
||||
|
||||
display_local = user.email.split("@", 1)[0]
|
||||
workspace = await ws_repo.create(
|
||||
name=f"{display_local}'s Workspace"[:64],
|
||||
slug=unique_slug,
|
||||
owner_id=str(user.id),
|
||||
)
|
||||
await m_repo.add(workspace_id=workspace["id"], user_id=str(user.id), role="owner")
|
||||
|
||||
user.default_workspace_id = workspace["id"]
|
||||
await get_local_provider().update_user(user)
|
||||
|
||||
return workspace["id"]
|
||||
|
||||
|
||||
router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
|
||||
|
||||
|
||||
@@ -292,7 +341,15 @@ async def login_local(
|
||||
)
|
||||
|
||||
_record_login_success(client_ip)
|
||||
token = create_access_token(str(user.id), token_version=user.token_version)
|
||||
# Ensure the user has a workspace (covers pre-PR4 users still in DB
|
||||
# whose default_workspace_id was never backfilled by the lifespan hook).
|
||||
workspace_id = await ensure_default_workspace(user)
|
||||
token = create_access_token(
|
||||
str(user.id),
|
||||
token_version=user.token_version,
|
||||
workspace_id=workspace_id,
|
||||
role="owner",
|
||||
)
|
||||
_set_session_cookie(response, token, request)
|
||||
|
||||
return LoginResponse(
|
||||
@@ -316,7 +373,14 @@ async def register(request: Request, response: Response, body: RegisterRequest):
|
||||
detail=AuthErrorResponse(code=AuthErrorCode.EMAIL_ALREADY_EXISTS, message="Email already registered").model_dump(),
|
||||
)
|
||||
|
||||
token = create_access_token(str(user.id), token_version=user.token_version)
|
||||
workspace_id = await ensure_default_workspace(user)
|
||||
|
||||
token = create_access_token(
|
||||
str(user.id),
|
||||
token_version=user.token_version,
|
||||
workspace_id=workspace_id,
|
||||
role="owner",
|
||||
)
|
||||
_set_session_cookie(response, token, request)
|
||||
|
||||
return UserResponse(id=str(user.id), email=user.email, system_role=user.system_role)
|
||||
@@ -368,18 +432,57 @@ async def change_password(request: Request, response: Response, body: ChangePass
|
||||
|
||||
await provider.update_user(user)
|
||||
|
||||
# Re-issue cookie with new token_version
|
||||
token = create_access_token(str(user.id), token_version=user.token_version)
|
||||
# Re-issue cookie with new token_version. wid + role must be carried
|
||||
# forward so the re-signed JWT still passes the AuthMiddleware
|
||||
# workspace gate; ensure_default_workspace fills in for the (rare)
|
||||
# case where the user predates PR4 and has not been backfilled.
|
||||
workspace_id = await ensure_default_workspace(user)
|
||||
token = create_access_token(
|
||||
str(user.id),
|
||||
token_version=user.token_version,
|
||||
workspace_id=workspace_id,
|
||||
role="owner",
|
||||
)
|
||||
_set_session_cookie(response, token, request)
|
||||
|
||||
return MessageResponse(message="Password changed successfully")
|
||||
|
||||
|
||||
@router.get("/me", response_model=UserResponse)
|
||||
@router.get("/me", response_model=UserMeResponse)
|
||||
async def get_me(request: Request):
|
||||
"""Get current authenticated user info."""
|
||||
"""Get current authenticated user info, including the workspaces they belong to."""
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.workspace import WorkspaceRepository
|
||||
from deerflow.persistence.workspace_membership import WorkspaceMembershipRepository
|
||||
|
||||
user = await get_current_user_from_request(request)
|
||||
return UserResponse(id=str(user.id), email=user.email, system_role=user.system_role, needs_setup=user.needs_setup)
|
||||
|
||||
sf = get_session_factory()
|
||||
workspaces: list[UserMeWorkspace] = []
|
||||
if sf is not None:
|
||||
ws_repo = WorkspaceRepository(sf)
|
||||
m_repo = WorkspaceMembershipRepository(sf)
|
||||
ws_rows = await ws_repo.list_by_user(user_id=str(user.id))
|
||||
memberships = await m_repo.list_by_user(user_id=str(user.id))
|
||||
role_by_ws = {m["workspace_id"]: m["role"] for m in memberships}
|
||||
workspaces = [
|
||||
UserMeWorkspace(
|
||||
id=w["id"],
|
||||
name=w["name"],
|
||||
slug=w["slug"],
|
||||
role=role_by_ws.get(w["id"], "member"),
|
||||
)
|
||||
for w in ws_rows
|
||||
]
|
||||
|
||||
return UserMeResponse(
|
||||
id=str(user.id),
|
||||
email=user.email,
|
||||
system_role=user.system_role,
|
||||
needs_setup=user.needs_setup,
|
||||
default_workspace_id=user.default_workspace_id,
|
||||
workspaces=workspaces,
|
||||
)
|
||||
|
||||
|
||||
_SETUP_STATUS_COOLDOWN: dict[str, float] = {}
|
||||
@@ -452,7 +555,14 @@ async def initialize_admin(request: Request, response: Response, body: Initializ
|
||||
detail=AuthErrorResponse(code=AuthErrorCode.SYSTEM_ALREADY_INITIALIZED, message="System already initialized").model_dump(),
|
||||
)
|
||||
|
||||
token = create_access_token(str(user.id), token_version=user.token_version)
|
||||
workspace_id = await ensure_default_workspace(user)
|
||||
|
||||
token = create_access_token(
|
||||
str(user.id),
|
||||
token_version=user.token_version,
|
||||
workspace_id=workspace_id,
|
||||
role="owner",
|
||||
)
|
||||
_set_session_cookie(response, token, request)
|
||||
|
||||
return UserResponse(id=str(user.id), email=user.email, system_role=user.system_role)
|
||||
|
||||
@@ -11,6 +11,7 @@ from langgraph.runtime import Runtime
|
||||
from deerflow.agents.thread_state import ThreadDataState
|
||||
from deerflow.config.paths import Paths, get_paths
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.runtime.workspace_context import get_effective_workspace_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -24,10 +25,15 @@ class ThreadDataMiddlewareState(AgentState):
|
||||
class ThreadDataMiddleware(AgentMiddleware[ThreadDataMiddlewareState]):
|
||||
"""Create thread data directories for each thread execution.
|
||||
|
||||
Creates the following directory structure:
|
||||
- {base_dir}/threads/{thread_id}/user-data/workspace
|
||||
- {base_dir}/threads/{thread_id}/user-data/uploads
|
||||
- {base_dir}/threads/{thread_id}/user-data/outputs
|
||||
PR6 routes thread storage through the workspace dimension. When a
|
||||
workspace contextvar is set (production via AuthMiddleware; tests via
|
||||
the autouse fixture), directories live at
|
||||
``{base_dir}/workspaces/{wid}/threads/{thread_id}/user-data/{workspace,uploads,outputs}``.
|
||||
In no-auth dev mode ``get_effective_workspace_id()`` returns
|
||||
``"default"`` so the layout stays valid; the ``user_id`` falls through
|
||||
to the same default constant. Either way thread state lives below a
|
||||
workspace bucket, never directly under ``{base_dir}/threads`` (legacy)
|
||||
or ``{base_dir}/users`` (PR4 layout).
|
||||
|
||||
Lifecycle Management:
|
||||
- With lazy_init=True (default): Only compute paths, directories created on-demand
|
||||
@@ -49,34 +55,18 @@ class ThreadDataMiddleware(AgentMiddleware[ThreadDataMiddlewareState]):
|
||||
self._paths = Paths(base_dir) if base_dir else get_paths()
|
||||
self._lazy_init = lazy_init
|
||||
|
||||
def _get_thread_paths(self, thread_id: str, user_id: str | None = None) -> dict[str, str]:
|
||||
"""Get the paths for a thread's data directories.
|
||||
|
||||
Args:
|
||||
thread_id: The thread ID.
|
||||
user_id: Optional user ID for per-user path isolation.
|
||||
|
||||
Returns:
|
||||
Dictionary with workspace_path, uploads_path, and outputs_path.
|
||||
"""
|
||||
def _get_thread_paths(self, thread_id: str, *, workspace_id: str, user_id: str) -> dict[str, str]:
|
||||
return {
|
||||
"workspace_path": str(self._paths.sandbox_work_dir(thread_id, user_id=user_id)),
|
||||
"uploads_path": str(self._paths.sandbox_uploads_dir(thread_id, user_id=user_id)),
|
||||
"outputs_path": str(self._paths.sandbox_outputs_dir(thread_id, user_id=user_id)),
|
||||
"workspace_path": str(self._paths.sandbox_work_dir(thread_id, workspace_id=workspace_id)),
|
||||
"uploads_path": str(self._paths.sandbox_uploads_dir(thread_id, workspace_id=workspace_id)),
|
||||
"outputs_path": str(self._paths.sandbox_outputs_dir(thread_id, workspace_id=workspace_id)),
|
||||
"user_id": user_id,
|
||||
"workspace_id": workspace_id,
|
||||
}
|
||||
|
||||
def _create_thread_directories(self, thread_id: str, user_id: str | None = None) -> dict[str, str]:
|
||||
"""Create the thread data directories.
|
||||
|
||||
Args:
|
||||
thread_id: The thread ID.
|
||||
user_id: Optional user ID for per-user path isolation.
|
||||
|
||||
Returns:
|
||||
Dictionary with the created directory paths.
|
||||
"""
|
||||
self._paths.ensure_thread_dirs(thread_id, user_id=user_id)
|
||||
return self._get_thread_paths(thread_id, user_id=user_id)
|
||||
def _create_thread_directories(self, thread_id: str, *, workspace_id: str, user_id: str) -> dict[str, str]:
|
||||
self._paths.ensure_thread_dirs(thread_id, workspace_id=workspace_id)
|
||||
return self._get_thread_paths(thread_id, workspace_id=workspace_id, user_id=user_id)
|
||||
|
||||
@override
|
||||
def before_agent(self, state: ThreadDataMiddlewareState, runtime: Runtime) -> dict | None:
|
||||
@@ -90,14 +80,15 @@ class ThreadDataMiddleware(AgentMiddleware[ThreadDataMiddlewareState]):
|
||||
raise ValueError("Thread ID is required in runtime context or config.configurable")
|
||||
|
||||
user_id = get_effective_user_id()
|
||||
workspace_id = get_effective_workspace_id()
|
||||
|
||||
if self._lazy_init:
|
||||
# Lazy initialization: only compute paths, don't create directories
|
||||
paths = self._get_thread_paths(thread_id, user_id=user_id)
|
||||
paths = self._get_thread_paths(thread_id, workspace_id=workspace_id, user_id=user_id)
|
||||
else:
|
||||
# Eager initialization: create directories immediately
|
||||
paths = self._create_thread_directories(thread_id, user_id=user_id)
|
||||
logger.debug("Created thread data directories for thread %s", thread_id)
|
||||
paths = self._create_thread_directories(thread_id, workspace_id=workspace_id, user_id=user_id)
|
||||
logger.debug("Created thread data directories for thread %s under workspace %s", thread_id, workspace_id)
|
||||
|
||||
messages = list(state.get("messages", []))
|
||||
last_message = messages[-1] if messages else None
|
||||
|
||||
@@ -10,6 +10,7 @@ VIRTUAL_PATH_PREFIX = "/mnt/user-data"
|
||||
|
||||
_SAFE_THREAD_ID_RE = re.compile(r"^[A-Za-z0-9_\-]+$")
|
||||
_SAFE_USER_ID_RE = re.compile(r"^[A-Za-z0-9_\-]+$")
|
||||
_SAFE_WORKSPACE_ID_RE = re.compile(r"^[A-Za-z0-9_\-]+$")
|
||||
|
||||
|
||||
def _default_local_base_dir() -> Path:
|
||||
@@ -31,6 +32,13 @@ def _validate_user_id(user_id: str) -> str:
|
||||
return user_id
|
||||
|
||||
|
||||
def _validate_workspace_id(workspace_id: str) -> str:
|
||||
"""Validate a workspace ID before using it in filesystem paths."""
|
||||
if not _SAFE_WORKSPACE_ID_RE.match(workspace_id):
|
||||
raise ValueError(f"Invalid workspace_id {workspace_id!r}: only alphanumeric characters, hyphens, and underscores are allowed.")
|
||||
return workspace_id
|
||||
|
||||
|
||||
def _join_host_path(base: str, *parts: str) -> str:
|
||||
"""Join host filesystem path segments while preserving native style.
|
||||
|
||||
@@ -148,116 +156,129 @@ class Paths:
|
||||
"""Legacy per-agent memory file: `{base_dir}/agents/{name}/memory.json`."""
|
||||
return self.agent_dir(name) / "memory.json"
|
||||
|
||||
def user_dir(self, user_id: str) -> Path:
|
||||
"""Directory for a specific user: `{base_dir}/users/{user_id}/`."""
|
||||
def workspace_dir(self, workspace_id: str) -> Path:
|
||||
"""Directory for a specific workspace: `{base_dir}/workspaces/{workspace_id}/`.
|
||||
|
||||
PR6 introduces this as the top-level isolation dimension. Per-user
|
||||
state and per-thread state both live underneath their workspace so
|
||||
a user with access to two workspaces never sees state bleed between
|
||||
them on the filesystem.
|
||||
"""
|
||||
return self.base_dir / "workspaces" / _validate_workspace_id(workspace_id)
|
||||
|
||||
def user_dir(self, user_id: str, *, workspace_id: str | None = None) -> Path:
|
||||
"""Directory for a specific user.
|
||||
|
||||
When ``workspace_id`` is provided (PR6+):
|
||||
``{base_dir}/workspaces/{wid}/users/{user_id}/``
|
||||
|
||||
Otherwise (legacy layout):
|
||||
``{base_dir}/users/{user_id}/``
|
||||
"""
|
||||
if workspace_id is not None:
|
||||
return self.workspace_dir(workspace_id) / "users" / _validate_user_id(user_id)
|
||||
return self.base_dir / "users" / _validate_user_id(user_id)
|
||||
|
||||
def user_memory_file(self, user_id: str) -> Path:
|
||||
"""Per-user memory file: `{base_dir}/users/{user_id}/memory.json`."""
|
||||
return self.user_dir(user_id) / "memory.json"
|
||||
def user_memory_file(self, user_id: str, *, workspace_id: str | None = None) -> Path:
|
||||
"""Per-user memory file under the active workspace (legacy without)."""
|
||||
return self.user_dir(user_id, workspace_id=workspace_id) / "memory.json"
|
||||
|
||||
def user_agents_dir(self, user_id: str) -> Path:
|
||||
"""Per-user root for that user's custom agents: `{base_dir}/users/{user_id}/agents/`."""
|
||||
return self.user_dir(user_id) / "agents"
|
||||
def user_agents_dir(self, user_id: str, *, workspace_id: str | None = None) -> Path:
|
||||
"""Per-user root for custom agents under the active workspace."""
|
||||
return self.user_dir(user_id, workspace_id=workspace_id) / "agents"
|
||||
|
||||
def user_agent_dir(self, user_id: str, agent_name: str) -> Path:
|
||||
"""Per-user per-agent directory: `{base_dir}/users/{user_id}/agents/{name}/`."""
|
||||
return self.user_agents_dir(user_id) / agent_name.lower()
|
||||
def user_agent_dir(self, user_id: str, agent_name: str, *, workspace_id: str | None = None) -> Path:
|
||||
"""Per-user per-agent directory under the active workspace."""
|
||||
return self.user_agents_dir(user_id, workspace_id=workspace_id) / agent_name.lower()
|
||||
|
||||
def user_agent_memory_file(self, user_id: str, agent_name: str) -> Path:
|
||||
"""Per-user per-agent memory: `{base_dir}/users/{user_id}/agents/{name}/memory.json`."""
|
||||
return self.user_agent_dir(user_id, agent_name) / "memory.json"
|
||||
def user_agent_memory_file(self, user_id: str, agent_name: str, *, workspace_id: str | None = None) -> Path:
|
||||
"""Per-user per-agent memory file under the active workspace."""
|
||||
return self.user_agent_dir(user_id, agent_name, workspace_id=workspace_id) / "memory.json"
|
||||
|
||||
def thread_dir(self, thread_id: str, *, user_id: str | None = None) -> Path:
|
||||
def thread_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> Path:
|
||||
"""
|
||||
Host path for a thread's data.
|
||||
|
||||
When *user_id* is provided:
|
||||
`{base_dir}/users/{user_id}/threads/{thread_id}/`
|
||||
Otherwise (legacy layout):
|
||||
`{base_dir}/threads/{thread_id}/`
|
||||
Precedence — workspace beats user, both beat legacy:
|
||||
|
||||
This directory contains a `user-data/` subdirectory that is mounted
|
||||
as `/mnt/user-data/` inside the sandbox.
|
||||
* ``workspace_id`` given (PR6+):
|
||||
``{base_dir}/workspaces/{wid}/threads/{thread_id}/``
|
||||
* ``user_id`` only (legacy after user-isolation migration):
|
||||
``{base_dir}/users/{user_id}/threads/{thread_id}/``
|
||||
* neither (very legacy, pre-isolation):
|
||||
``{base_dir}/threads/{thread_id}/``
|
||||
|
||||
The contained ``user-data/`` subdirectory is mounted as
|
||||
``/mnt/user-data/`` inside the sandbox regardless of which form is
|
||||
chosen — only the host-side parent differs.
|
||||
|
||||
Raises:
|
||||
ValueError: If `thread_id` or `user_id` contains unsafe characters (path
|
||||
separators or `..`) that could cause directory traversal.
|
||||
ValueError: If any of the supplied ids contains unsafe characters.
|
||||
"""
|
||||
if workspace_id is not None:
|
||||
return self.workspace_dir(workspace_id) / "threads" / _validate_thread_id(thread_id)
|
||||
if user_id is not None:
|
||||
return self.user_dir(user_id) / "threads" / _validate_thread_id(thread_id)
|
||||
return self.base_dir / "users" / _validate_user_id(user_id) / "threads" / _validate_thread_id(thread_id)
|
||||
return self.base_dir / "threads" / _validate_thread_id(thread_id)
|
||||
|
||||
def sandbox_work_dir(self, thread_id: str, *, user_id: str | None = None) -> Path:
|
||||
def sandbox_work_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> Path:
|
||||
"""
|
||||
Host path for the agent's workspace directory.
|
||||
Host: `{base_dir}/threads/{thread_id}/user-data/workspace/`
|
||||
Sandbox: `/mnt/user-data/workspace/`
|
||||
Host: ``{thread_dir}/user-data/workspace/``
|
||||
Sandbox: ``/mnt/user-data/workspace/``
|
||||
"""
|
||||
return self.thread_dir(thread_id, user_id=user_id) / "user-data" / "workspace"
|
||||
return self.thread_dir(thread_id, workspace_id=workspace_id, user_id=user_id) / "user-data" / "workspace"
|
||||
|
||||
def sandbox_uploads_dir(self, thread_id: str, *, user_id: str | None = None) -> Path:
|
||||
"""
|
||||
Host path for user-uploaded files.
|
||||
Host: `{base_dir}/threads/{thread_id}/user-data/uploads/`
|
||||
Sandbox: `/mnt/user-data/uploads/`
|
||||
"""
|
||||
return self.thread_dir(thread_id, user_id=user_id) / "user-data" / "uploads"
|
||||
def sandbox_uploads_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> Path:
|
||||
"""Host path for user-uploaded files; sandbox: ``/mnt/user-data/uploads/``."""
|
||||
return self.thread_dir(thread_id, workspace_id=workspace_id, user_id=user_id) / "user-data" / "uploads"
|
||||
|
||||
def sandbox_outputs_dir(self, thread_id: str, *, user_id: str | None = None) -> Path:
|
||||
"""
|
||||
Host path for agent-generated artifacts.
|
||||
Host: `{base_dir}/threads/{thread_id}/user-data/outputs/`
|
||||
Sandbox: `/mnt/user-data/outputs/`
|
||||
"""
|
||||
return self.thread_dir(thread_id, user_id=user_id) / "user-data" / "outputs"
|
||||
def sandbox_outputs_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> Path:
|
||||
"""Host path for agent-generated artifacts; sandbox: ``/mnt/user-data/outputs/``."""
|
||||
return self.thread_dir(thread_id, workspace_id=workspace_id, user_id=user_id) / "user-data" / "outputs"
|
||||
|
||||
def acp_workspace_dir(self, thread_id: str, *, user_id: str | None = None) -> Path:
|
||||
def acp_workspace_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> Path:
|
||||
"""
|
||||
Host path for the ACP workspace of a specific thread.
|
||||
Host: `{base_dir}/threads/{thread_id}/acp-workspace/`
|
||||
Sandbox: `/mnt/acp-workspace/`
|
||||
Host path for the ACP workspace of a specific thread; sandbox: ``/mnt/acp-workspace/``.
|
||||
|
||||
Each thread gets its own isolated ACP workspace so that concurrent
|
||||
sessions cannot read each other's ACP agent outputs.
|
||||
"""
|
||||
return self.thread_dir(thread_id, user_id=user_id) / "acp-workspace"
|
||||
return self.thread_dir(thread_id, workspace_id=workspace_id, user_id=user_id) / "acp-workspace"
|
||||
|
||||
def sandbox_user_data_dir(self, thread_id: str, *, user_id: str | None = None) -> Path:
|
||||
"""
|
||||
Host path for the user-data root.
|
||||
Host: `{base_dir}/threads/{thread_id}/user-data/`
|
||||
Sandbox: `/mnt/user-data/`
|
||||
"""
|
||||
return self.thread_dir(thread_id, user_id=user_id) / "user-data"
|
||||
def sandbox_user_data_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> Path:
|
||||
"""Host path for the user-data root; sandbox: ``/mnt/user-data/``."""
|
||||
return self.thread_dir(thread_id, workspace_id=workspace_id, user_id=user_id) / "user-data"
|
||||
|
||||
def host_thread_dir(self, thread_id: str, *, user_id: str | None = None) -> str:
|
||||
def host_thread_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> str:
|
||||
"""Host path for a thread directory, preserving Windows path syntax."""
|
||||
if workspace_id is not None:
|
||||
return _join_host_path(self._host_base_dir_str(), "workspaces", _validate_workspace_id(workspace_id), "threads", _validate_thread_id(thread_id))
|
||||
if user_id is not None:
|
||||
return _join_host_path(self._host_base_dir_str(), "users", _validate_user_id(user_id), "threads", _validate_thread_id(thread_id))
|
||||
return _join_host_path(self._host_base_dir_str(), "threads", _validate_thread_id(thread_id))
|
||||
|
||||
def host_sandbox_user_data_dir(self, thread_id: str, *, user_id: str | None = None) -> str:
|
||||
def host_sandbox_user_data_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> str:
|
||||
"""Host path for a thread's user-data root."""
|
||||
return _join_host_path(self.host_thread_dir(thread_id, user_id=user_id), "user-data")
|
||||
return _join_host_path(self.host_thread_dir(thread_id, workspace_id=workspace_id, user_id=user_id), "user-data")
|
||||
|
||||
def host_sandbox_work_dir(self, thread_id: str, *, user_id: str | None = None) -> str:
|
||||
def host_sandbox_work_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> str:
|
||||
"""Host path for the workspace mount source."""
|
||||
return _join_host_path(self.host_sandbox_user_data_dir(thread_id, user_id=user_id), "workspace")
|
||||
return _join_host_path(self.host_sandbox_user_data_dir(thread_id, workspace_id=workspace_id, user_id=user_id), "workspace")
|
||||
|
||||
def host_sandbox_uploads_dir(self, thread_id: str, *, user_id: str | None = None) -> str:
|
||||
def host_sandbox_uploads_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> str:
|
||||
"""Host path for the uploads mount source."""
|
||||
return _join_host_path(self.host_sandbox_user_data_dir(thread_id, user_id=user_id), "uploads")
|
||||
return _join_host_path(self.host_sandbox_user_data_dir(thread_id, workspace_id=workspace_id, user_id=user_id), "uploads")
|
||||
|
||||
def host_sandbox_outputs_dir(self, thread_id: str, *, user_id: str | None = None) -> str:
|
||||
def host_sandbox_outputs_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> str:
|
||||
"""Host path for the outputs mount source."""
|
||||
return _join_host_path(self.host_sandbox_user_data_dir(thread_id, user_id=user_id), "outputs")
|
||||
return _join_host_path(self.host_sandbox_user_data_dir(thread_id, workspace_id=workspace_id, user_id=user_id), "outputs")
|
||||
|
||||
def host_acp_workspace_dir(self, thread_id: str, *, user_id: str | None = None) -> str:
|
||||
def host_acp_workspace_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> str:
|
||||
"""Host path for the ACP workspace mount source."""
|
||||
return _join_host_path(self.host_thread_dir(thread_id, user_id=user_id), "acp-workspace")
|
||||
return _join_host_path(self.host_thread_dir(thread_id, workspace_id=workspace_id, user_id=user_id), "acp-workspace")
|
||||
|
||||
def ensure_thread_dirs(self, thread_id: str, *, user_id: str | None = None) -> None:
|
||||
def ensure_thread_dirs(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> None:
|
||||
"""Create all standard sandbox directories for a thread.
|
||||
|
||||
Directories are created with mode 0o777 so that sandbox containers
|
||||
@@ -271,24 +292,28 @@ class Paths:
|
||||
ACP agent invocation.
|
||||
"""
|
||||
for d in [
|
||||
self.sandbox_work_dir(thread_id, user_id=user_id),
|
||||
self.sandbox_uploads_dir(thread_id, user_id=user_id),
|
||||
self.sandbox_outputs_dir(thread_id, user_id=user_id),
|
||||
self.acp_workspace_dir(thread_id, user_id=user_id),
|
||||
self.sandbox_work_dir(thread_id, workspace_id=workspace_id, user_id=user_id),
|
||||
self.sandbox_uploads_dir(thread_id, workspace_id=workspace_id, user_id=user_id),
|
||||
self.sandbox_outputs_dir(thread_id, workspace_id=workspace_id, user_id=user_id),
|
||||
self.acp_workspace_dir(thread_id, workspace_id=workspace_id, user_id=user_id),
|
||||
]:
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
d.chmod(0o777)
|
||||
|
||||
def delete_thread_dir(self, thread_id: str, *, user_id: str | None = None) -> None:
|
||||
"""Delete all persisted data for a thread.
|
||||
|
||||
The operation is idempotent: missing thread directories are ignored.
|
||||
"""
|
||||
thread_dir = self.thread_dir(thread_id, user_id=user_id)
|
||||
def delete_thread_dir(self, thread_id: str, *, workspace_id: str | None = None, user_id: str | None = None) -> None:
|
||||
"""Delete all persisted data for a thread. Idempotent."""
|
||||
thread_dir = self.thread_dir(thread_id, workspace_id=workspace_id, user_id=user_id)
|
||||
if thread_dir.exists():
|
||||
shutil.rmtree(thread_dir)
|
||||
|
||||
def resolve_virtual_path(self, thread_id: str, virtual_path: str, *, user_id: str | None = None) -> Path:
|
||||
def resolve_virtual_path(
|
||||
self,
|
||||
thread_id: str,
|
||||
virtual_path: str,
|
||||
*,
|
||||
workspace_id: str | None = None,
|
||||
user_id: str | None = None,
|
||||
) -> Path:
|
||||
"""Resolve a sandbox virtual path to the actual host filesystem path.
|
||||
|
||||
Args:
|
||||
@@ -296,7 +321,8 @@ class Paths:
|
||||
virtual_path: Virtual path as seen inside the sandbox, e.g.
|
||||
``/mnt/user-data/outputs/report.pdf``.
|
||||
Leading slashes are stripped before matching.
|
||||
user_id: Optional user ID for user-scoped path resolution.
|
||||
workspace_id: Optional workspace ID for workspace-scoped resolution.
|
||||
user_id: Optional user ID for legacy user-scoped resolution.
|
||||
|
||||
Returns:
|
||||
The resolved absolute host filesystem path.
|
||||
@@ -314,7 +340,7 @@ class Paths:
|
||||
raise ValueError(f"Path must start with /{prefix}")
|
||||
|
||||
relative = stripped[len(prefix) :].lstrip("/")
|
||||
base = self.sandbox_user_data_dir(thread_id, user_id=user_id).resolve()
|
||||
base = self.sandbox_user_data_dir(thread_id, workspace_id=workspace_id, user_id=user_id).resolve()
|
||||
actual = (base / relative).resolve()
|
||||
|
||||
try:
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, Text, UniqueConstraint
|
||||
from sqlalchemy import DateTime, ForeignKey, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -13,20 +13,22 @@ from deerflow.persistence.base import Base
|
||||
class FeedbackRow(Base):
|
||||
__tablename__ = "feedback"
|
||||
|
||||
__table_args__ = (UniqueConstraint("thread_id", "run_id", "user_id", name="uq_feedback_thread_run_user"),)
|
||||
__table_args__ = (
|
||||
UniqueConstraint("thread_id", "run_id", "user_id", name="uq_feedback_thread_run_user"),
|
||||
{"comment": "用户对运行结果的反馈(点赞/点踩 + 文字评论),(thread, run, user) 唯一"},
|
||||
)
|
||||
|
||||
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
message_id: Mapped[str | None] = mapped_column(String(64))
|
||||
# message_id is an optional RunEventStore event identifier —
|
||||
# allows feedback to target a specific message or the entire run
|
||||
|
||||
rating: Mapped[int] = mapped_column(nullable=False)
|
||||
# +1 (thumbs-up) or -1 (thumbs-down)
|
||||
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
# Optional text feedback from the user
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True, comment="反馈主键")
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True, comment="关联的运行 ID(runs.run_id)")
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True, comment="关联的会话 ID(threads_meta.thread_id)")
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True, comment="反馈作者;为 NULL 表示历史无主数据")
|
||||
workspace_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("workspaces.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
comment="所属 workspace。PR5 引入时 nullable 用于回填;alembic 0003 + PR6 仓储接入完成后 NOT NULL",
|
||||
)
|
||||
message_id: Mapped[str | None] = mapped_column(String(64), comment="可选的 RunEventStore 事件 ID;为 NULL 表示针对整次运行而非单条消息")
|
||||
rating: Mapped[int] = mapped_column(nullable=False, comment="评分:+1 点赞,-1 点踩")
|
||||
comment: Mapped[str | None] = mapped_column(Text, comment="可选的文字评论")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), comment="创建时间(UTC)")
|
||||
|
||||
@@ -13,6 +13,13 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
from deerflow.runtime.workspace_context import AUTO as WORKSPACE_AUTO
|
||||
from deerflow.runtime.workspace_context import (
|
||||
_AutoSentinel as _WorkspaceAutoSentinel,
|
||||
)
|
||||
from deerflow.runtime.workspace_context import (
|
||||
resolve_workspace_id,
|
||||
)
|
||||
|
||||
|
||||
class FeedbackRepository:
|
||||
@@ -34,6 +41,7 @@ class FeedbackRepository:
|
||||
thread_id: str,
|
||||
rating: int,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
message_id: str | None = None,
|
||||
comment: str | None = None,
|
||||
) -> dict:
|
||||
@@ -41,11 +49,13 @@ class FeedbackRepository:
|
||||
if rating not in (1, -1):
|
||||
raise ValueError(f"rating must be +1 or -1, got {rating}")
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.create")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="FeedbackRepository.create")
|
||||
row = FeedbackRow(
|
||||
feedback_id=str(uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
workspace_id=resolved_workspace_id,
|
||||
message_id=message_id,
|
||||
rating=rating,
|
||||
comment=comment,
|
||||
@@ -62,12 +72,16 @@ class FeedbackRepository:
|
||||
feedback_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> dict | None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.get")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="FeedbackRepository.get")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(FeedbackRow, feedback_id)
|
||||
if row is None:
|
||||
return None
|
||||
if resolved_workspace_id is not None and row.workspace_id != resolved_workspace_id:
|
||||
return None
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
@@ -79,9 +93,13 @@ class FeedbackRepository:
|
||||
*,
|
||||
limit: int = 100,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_run")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="FeedbackRepository.list_by_run")
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
|
||||
if resolved_workspace_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.workspace_id == resolved_workspace_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
|
||||
@@ -95,9 +113,13 @@ class FeedbackRepository:
|
||||
*,
|
||||
limit: int = 100,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="FeedbackRepository.list_by_thread")
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
|
||||
if resolved_workspace_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.workspace_id == resolved_workspace_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
|
||||
@@ -110,12 +132,16 @@ class FeedbackRepository:
|
||||
feedback_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> bool:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="FeedbackRepository.delete")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(FeedbackRow, feedback_id)
|
||||
if row is None:
|
||||
return False
|
||||
if resolved_workspace_id is not None and row.workspace_id != resolved_workspace_id:
|
||||
return False
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return False
|
||||
await session.delete(row)
|
||||
@@ -129,18 +155,22 @@ class FeedbackRepository:
|
||||
thread_id: str,
|
||||
rating: int,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
comment: str | None = None,
|
||||
) -> dict:
|
||||
"""Create or update feedback for (thread_id, run_id, user_id). rating must be +1 or -1."""
|
||||
if rating not in (1, -1):
|
||||
raise ValueError(f"rating must be +1 or -1, got {rating}")
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.upsert")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="FeedbackRepository.upsert")
|
||||
async with self._sf() as session:
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
)
|
||||
if resolved_workspace_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.workspace_id == resolved_workspace_id)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is not None:
|
||||
@@ -153,6 +183,7 @@ class FeedbackRepository:
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
workspace_id=resolved_workspace_id,
|
||||
rating=rating,
|
||||
comment=comment,
|
||||
created_at=datetime.now(UTC),
|
||||
@@ -168,15 +199,19 @@ class FeedbackRepository:
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> bool:
|
||||
"""Delete the current user's feedback for a run. Returns True if a record was deleted."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete_by_run")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="FeedbackRepository.delete_by_run")
|
||||
async with self._sf() as session:
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
)
|
||||
if resolved_workspace_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.workspace_id == resolved_workspace_id)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
@@ -190,10 +225,14 @@ class FeedbackRepository:
|
||||
thread_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> dict[str, dict]:
|
||||
"""Return feedback grouped by run_id for a thread: {run_id: feedback_dict}."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread_grouped")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="FeedbackRepository.list_by_thread_grouped")
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
|
||||
if resolved_workspace_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.workspace_id == resolved_workspace_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
async with self._sf() as session:
|
||||
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
"""users.default_workspace_id column + FK to workspaces
|
||||
|
||||
Revision ID: 0001_users_default_workspace
|
||||
Revises: None
|
||||
Create Date: 2026-05-12
|
||||
|
||||
First Alembic revision for the DeerFlow application schema. Adds
|
||||
`users.default_workspace_id` so a newly-registered user can be sent
|
||||
back to their default workspace on next login without consulting the
|
||||
memberships table.
|
||||
|
||||
Existing deployments created `users` via `metadata.create_all()` without
|
||||
this column; running `alembic upgrade head` on those DBs will simply add
|
||||
the column (ON DELETE SET NULL FK), no data backfill needed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# Alembic identifiers.
|
||||
revision: str = "0001_users_default_workspace"
|
||||
down_revision: str | None = None
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("users") as batch:
|
||||
batch.add_column(sa.Column("default_workspace_id", sa.String(36), nullable=True))
|
||||
batch.create_foreign_key(
|
||||
"fk_users_default_workspace",
|
||||
"workspaces",
|
||||
["default_workspace_id"],
|
||||
["id"],
|
||||
ondelete="SET NULL",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("users") as batch:
|
||||
batch.drop_constraint("fk_users_default_workspace", type_="foreignkey")
|
||||
batch.drop_column("default_workspace_id")
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
"""Business tables: nullable workspace_id + FK to workspaces
|
||||
|
||||
Revision ID: 0002_business_tables_workspace
|
||||
Revises: 0001_users_default_workspace
|
||||
Create Date: 2026-05-13
|
||||
|
||||
Stage 0 PR5 step 1/2 (the second step lives in revision 0003).
|
||||
|
||||
This revision adds a *nullable* ``workspace_id`` column to the four
|
||||
business tables that need tenancy scoping:
|
||||
|
||||
* ``threads_meta``
|
||||
* ``runs``
|
||||
* ``feedback``
|
||||
* ``run_events``
|
||||
|
||||
The column is nullable here on purpose — running ``upgrade`` on a
|
||||
database with existing rows leaves those rows with ``workspace_id = NULL``
|
||||
until ``scripts/backfill_workspace_id.py`` populates them. Once the
|
||||
backfill finishes, revision 0003 flips the column to ``NOT NULL`` and
|
||||
adds the threads_meta ``(workspace_id, thread_id)`` UNIQUE index.
|
||||
|
||||
A composite index ``idx_threads_meta_workspace_user_updated`` is added
|
||||
on ``threads_meta`` to support the common "list a workspace's threads
|
||||
for a user, newest first" access pattern that PR6 routers will rely on.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0002_business_tables_workspace"
|
||||
down_revision: str | None = "0001_users_default_workspace"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
# Tables that receive the new column. The order matters only for human
|
||||
# readability in migration logs — there are no inter-table data deps
|
||||
# during ALTER, and the FK references workspaces (introduced in PR3) which
|
||||
# is already present at this point.
|
||||
_BUSINESS_TABLES = ("threads_meta", "runs", "feedback", "run_events")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for table in _BUSINESS_TABLES:
|
||||
with op.batch_alter_table(table) as batch:
|
||||
batch.add_column(sa.Column("workspace_id", sa.String(36), nullable=True))
|
||||
batch.create_foreign_key(
|
||||
f"fk_{table}_workspace_id",
|
||||
"workspaces",
|
||||
["workspace_id"],
|
||||
["id"],
|
||||
ondelete="CASCADE",
|
||||
)
|
||||
|
||||
op.create_index(
|
||||
"idx_threads_meta_workspace_user_updated",
|
||||
"threads_meta",
|
||||
["workspace_id", "user_id", "updated_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("idx_threads_meta_workspace_user_updated", table_name="threads_meta")
|
||||
for table in reversed(_BUSINESS_TABLES):
|
||||
with op.batch_alter_table(table) as batch:
|
||||
batch.drop_constraint(f"fk_{table}_workspace_id", type_="foreignkey")
|
||||
batch.drop_column("workspace_id")
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
"""Business tables: workspace_id NOT NULL + UNIQUE(workspace_id, thread_id)
|
||||
|
||||
Revision ID: 0003_business_tables_workspace_not_null
|
||||
Revises: 0002_business_tables_workspace
|
||||
Create Date: 2026-05-13
|
||||
|
||||
Stage 0 PR5 step 2/2. Flips ``workspace_id`` on the four business
|
||||
tables to ``NOT NULL`` and adds the threads_meta ``(workspace_id,
|
||||
thread_id)`` UNIQUE index promised in workspace-schema-design §4.
|
||||
|
||||
**Refuses to upgrade** if any of the four tables still has rows with
|
||||
``workspace_id IS NULL`` — the operator must run
|
||||
``scripts/backfill_workspace_id.py`` first. Doing the NOT NULL ALTER
|
||||
with stragglers in place would either fail (Postgres) or silently
|
||||
corrupt SQLite tables via ``batch_alter_table`` rebuilds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0003_business_tables_workspace_not_null"
|
||||
down_revision: str | None = "0002_business_tables_workspace"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
_BUSINESS_TABLES = ("threads_meta", "runs", "feedback", "run_events")
|
||||
|
||||
|
||||
class _BackfillRequiredError(RuntimeError):
|
||||
"""Raised when null workspace_id rows remain at the start of upgrade."""
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
for table in _BUSINESS_TABLES:
|
||||
# Quoted identifier is fine here — table names are constants in this
|
||||
# module, no operator input reaches the SQL.
|
||||
count = conn.execute(sa.text(f"SELECT count(*) FROM {table} WHERE workspace_id IS NULL")).scalar() or 0
|
||||
if count > 0:
|
||||
raise _BackfillRequiredError(
|
||||
f"Cannot ALTER {table}.workspace_id to NOT NULL: {count} row(s) still have workspace_id=NULL. Run `python scripts/backfill_workspace_id.py` first.",
|
||||
)
|
||||
|
||||
for table in _BUSINESS_TABLES:
|
||||
with op.batch_alter_table(table) as batch:
|
||||
batch.alter_column("workspace_id", existing_type=sa.String(36), nullable=False)
|
||||
|
||||
op.create_index(
|
||||
"idx_threads_meta_workspace_thread",
|
||||
"threads_meta",
|
||||
["workspace_id", "thread_id"],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("idx_threads_meta_workspace_thread", table_name="threads_meta")
|
||||
for table in reversed(_BUSINESS_TABLES):
|
||||
with op.batch_alter_table(table) as batch:
|
||||
batch.alter_column("workspace_id", existing_type=sa.String(36), nullable=True)
|
||||
@@ -8,6 +8,8 @@ The actual ORM classes have moved to entity-specific subpackages:
|
||||
- ``deerflow.persistence.run``
|
||||
- ``deerflow.persistence.feedback``
|
||||
- ``deerflow.persistence.user``
|
||||
- ``deerflow.persistence.workspace`` (Stage 0 PR3)
|
||||
- ``deerflow.persistence.workspace_membership`` (Stage 0 PR3)
|
||||
|
||||
``RunEventRow`` remains in ``deerflow.persistence.models.run_event`` because
|
||||
its storage implementation lives in ``deerflow.runtime.events.store.db`` and
|
||||
@@ -19,5 +21,15 @@ from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.workspace.model import WorkspaceRow
|
||||
from deerflow.persistence.workspace_membership.model import WorkspaceMembershipRow
|
||||
|
||||
__all__ = ["FeedbackRow", "RunEventRow", "RunRow", "ThreadMetaRow", "UserRow"]
|
||||
__all__ = [
|
||||
"FeedbackRow",
|
||||
"RunEventRow",
|
||||
"RunRow",
|
||||
"ThreadMetaRow",
|
||||
"UserRow",
|
||||
"WorkspaceMembershipRow",
|
||||
"WorkspaceRow",
|
||||
]
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Index, String, Text, UniqueConstraint
|
||||
from sqlalchemy import JSON, DateTime, ForeignKey, Index, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -13,23 +13,31 @@ from deerflow.persistence.base import Base
|
||||
class RunEventRow(Base):
|
||||
__tablename__ = "run_events"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
# Owner of the conversation this event belongs to. Nullable for data
|
||||
# created before auth was introduced; populated by auth middleware on
|
||||
# new writes and by the boot-time orphan migration on existing rows.
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
event_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
category: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
# "message" | "trace" | "lifecycle"
|
||||
content: Mapped[str] = mapped_column(Text, default="")
|
||||
event_metadata: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
seq: Mapped[int] = mapped_column(nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True, comment="自增主键")
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, comment="所属会话 ID(threads_meta.thread_id)")
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, comment="所属运行 ID(runs.run_id)")
|
||||
user_id: Mapped[str | None] = mapped_column(
|
||||
String(64),
|
||||
nullable=True,
|
||||
index=True,
|
||||
comment="会话所有者;为 NULL 表示鉴权引入之前的历史数据,新写入由 auth 中间件填充,启动期 orphan 迁移会回填存量",
|
||||
)
|
||||
workspace_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("workspaces.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
comment="所属 workspace。PR5 引入时 nullable 用于回填;alembic 0003 + PR6 仓储接入完成后 NOT NULL",
|
||||
)
|
||||
event_type: Mapped[str] = mapped_column(String(32), nullable=False, comment="事件子类型(具体含义由 category 决定,如 ai_message_chunk、tool_call、run_started)")
|
||||
category: Mapped[str] = mapped_column(String(16), nullable=False, comment='事件大类:"message" 消息 / "trace" 追踪 / "lifecycle" 生命周期')
|
||||
content: Mapped[str] = mapped_column(Text, default="", comment="事件文本内容(消息体、错误、状态字符串等)")
|
||||
event_metadata: Mapped[dict] = mapped_column(JSON, default=dict, comment="事件结构化元数据(JSON),随 event_type 而异")
|
||||
seq: Mapped[int] = mapped_column(nullable=False, comment="在 thread 内的全局递增序号;与 thread_id 组合唯一")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), comment="创建时间(UTC)")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("thread_id", "seq", name="uq_events_thread_seq"),
|
||||
Index("ix_events_thread_cat_seq", "thread_id", "category", "seq"),
|
||||
Index("ix_events_run", "thread_id", "run_id", "seq"),
|
||||
{"comment": "运行事件流(消息/追踪/生命周期事件按 seq 顺序追加,是消息回放与审计的真源)"},
|
||||
)
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Index, String, Text
|
||||
from sqlalchemy import JSON, DateTime, ForeignKey, Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -13,37 +13,49 @@ from deerflow.persistence.base import Base
|
||||
class RunRow(Base):
|
||||
__tablename__ = "runs"
|
||||
|
||||
run_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
assistant_id: Mapped[str | None] = mapped_column(String(128))
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
status: Mapped[str] = mapped_column(String(20), default="pending")
|
||||
# "pending" | "running" | "success" | "error" | "timeout" | "interrupted"
|
||||
run_id: Mapped[str] = mapped_column(String(64), primary_key=True, comment="运行主键")
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True, comment="所属会话 ID(threads_meta.thread_id)")
|
||||
assistant_id: Mapped[str | None] = mapped_column(String(128), comment="使用的 Assistant ID(自定义智能体名);为 NULL 表示默认 lead agent")
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True, comment="发起本次运行的用户 ID")
|
||||
workspace_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("workspaces.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
comment="所属 workspace。PR5 引入时 nullable 用于回填;alembic 0003 + PR6 仓储接入完成后 NOT NULL",
|
||||
)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(20),
|
||||
default="pending",
|
||||
comment='运行状态:"pending" / "running" / "success" / "error" / "timeout" / "interrupted"',
|
||||
)
|
||||
model_name: Mapped[str | None] = mapped_column(String(128), comment="本次运行的主模型名(来自 config.yaml.models[*].name)")
|
||||
multitask_strategy: Mapped[str] = mapped_column(
|
||||
String(20),
|
||||
default="reject",
|
||||
comment='并发策略:同一 thread 已有运行时怎么处理("reject" / "interrupt" / "rollback" / "enqueue")',
|
||||
)
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict, comment="运行级元数据(JSON),如 channel/source 等")
|
||||
kwargs_json: Mapped[dict] = mapped_column(JSON, default=dict, comment="提交运行时的额外参数(JSON),如 thinking_enabled、tool 配置等")
|
||||
error: Mapped[str | None] = mapped_column(Text, comment="运行失败时的错误文本;成功时为 NULL")
|
||||
|
||||
model_name: Mapped[str | None] = mapped_column(String(128))
|
||||
multitask_strategy: Mapped[str] = mapped_column(String(20), default="reject")
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
kwargs_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
message_count: Mapped[int] = mapped_column(default=0, comment="本次运行产生的消息总数(便利字段,避免列表页查 RunEventStore)")
|
||||
first_human_message: Mapped[str | None] = mapped_column(Text, comment="首条用户消息文本预览(用于列表展示)")
|
||||
last_ai_message: Mapped[str | None] = mapped_column(Text, comment="末条 AI 消息文本预览(用于列表展示)")
|
||||
|
||||
# Convenience fields (for listing pages without querying RunEventStore)
|
||||
message_count: Mapped[int] = mapped_column(default=0)
|
||||
first_human_message: Mapped[str | None] = mapped_column(Text)
|
||||
last_ai_message: Mapped[str | None] = mapped_column(Text)
|
||||
total_input_tokens: Mapped[int] = mapped_column(default=0, comment="累计输入 token 数(运行结束时由 RunJournal 落盘)")
|
||||
total_output_tokens: Mapped[int] = mapped_column(default=0, comment="累计输出 token 数")
|
||||
total_tokens: Mapped[int] = mapped_column(default=0, comment="累计 token 总数 = input + output")
|
||||
llm_call_count: Mapped[int] = mapped_column(default=0, comment="累计 LLM 调用次数")
|
||||
lead_agent_tokens: Mapped[int] = mapped_column(default=0, comment="主 agent 自身消耗的 token 数")
|
||||
subagent_tokens: Mapped[int] = mapped_column(default=0, comment="子 agent(task 工具委派)消耗的 token 数")
|
||||
middleware_tokens: Mapped[int] = mapped_column(default=0, comment="中间件(如 summarization、title)消耗的 token 数")
|
||||
|
||||
# Token usage (accumulated in-memory by RunJournal, written on run completion)
|
||||
total_input_tokens: Mapped[int] = mapped_column(default=0)
|
||||
total_output_tokens: Mapped[int] = mapped_column(default=0)
|
||||
total_tokens: Mapped[int] = mapped_column(default=0)
|
||||
llm_call_count: Mapped[int] = mapped_column(default=0)
|
||||
lead_agent_tokens: Mapped[int] = mapped_column(default=0)
|
||||
subagent_tokens: Mapped[int] = mapped_column(default=0)
|
||||
middleware_tokens: Mapped[int] = mapped_column(default=0)
|
||||
follow_up_to_run_id: Mapped[str | None] = mapped_column(String(64), comment="续接的上一次运行 ID(用于'重新生成'/'继续'等链式调用)")
|
||||
|
||||
# Follow-up association
|
||||
follow_up_to_run_id: Mapped[str | None] = mapped_column(String(64))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), comment="创建时间(UTC)")
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC), comment="最近更新时间(UTC,写入时自动更新)")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
__table_args__ = (Index("ix_runs_thread_status", "thread_id", "status"),)
|
||||
__table_args__ = (
|
||||
Index("ix_runs_thread_status", "thread_id", "status"),
|
||||
{"comment": "运行(一次完整 agent 执行)的元数据 + 累计 token 指标"},
|
||||
)
|
||||
|
||||
@@ -17,6 +17,13 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.runtime.runs.store.base import RunStore
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
from deerflow.runtime.workspace_context import AUTO as WORKSPACE_AUTO
|
||||
from deerflow.runtime.workspace_context import (
|
||||
_AutoSentinel as _WorkspaceAutoSentinel,
|
||||
)
|
||||
from deerflow.runtime.workspace_context import (
|
||||
resolve_workspace_id,
|
||||
)
|
||||
|
||||
|
||||
class RunRepository(RunStore):
|
||||
@@ -70,6 +77,7 @@ class RunRepository(RunStore):
|
||||
thread_id,
|
||||
assistant_id=None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
status="pending",
|
||||
multitask_strategy="reject",
|
||||
metadata=None,
|
||||
@@ -79,12 +87,14 @@ class RunRepository(RunStore):
|
||||
follow_up_to_run_id=None,
|
||||
):
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="RunRepository.put")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="RunRepository.put")
|
||||
now = datetime.now(UTC)
|
||||
row = RunRow(
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
assistant_id=assistant_id,
|
||||
user_id=resolved_user_id,
|
||||
workspace_id=resolved_workspace_id,
|
||||
status=status,
|
||||
multitask_strategy=multitask_strategy,
|
||||
metadata_json=self._safe_json(metadata) or {},
|
||||
@@ -103,12 +113,16 @@ class RunRepository(RunStore):
|
||||
run_id,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
):
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="RunRepository.get")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="RunRepository.get")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RunRow, run_id)
|
||||
if row is None:
|
||||
return None
|
||||
if resolved_workspace_id is not None and row.workspace_id != resolved_workspace_id:
|
||||
return None
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
@@ -118,10 +132,14 @@ class RunRepository(RunStore):
|
||||
thread_id,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
limit=100,
|
||||
):
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="RunRepository.list_by_thread")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="RunRepository.list_by_thread")
|
||||
stmt = select(RunRow).where(RunRow.thread_id == thread_id)
|
||||
if resolved_workspace_id is not None:
|
||||
stmt = stmt.where(RunRow.workspace_id == resolved_workspace_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(RunRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(RunRow.created_at.desc()).limit(limit)
|
||||
@@ -142,12 +160,16 @@ class RunRepository(RunStore):
|
||||
run_id,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
):
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="RunRepository.delete")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="RunRepository.delete")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(RunRow, run_id)
|
||||
if row is None:
|
||||
return
|
||||
if resolved_workspace_id is not None and row.workspace_id != resolved_workspace_id:
|
||||
return
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return
|
||||
await session.delete(row)
|
||||
|
||||
@@ -4,12 +4,18 @@ Implementations:
|
||||
- ThreadMetaRepository: SQL-backed (sqlite / postgres via SQLAlchemy)
|
||||
- MemoryThreadMetaStore: wraps LangGraph BaseStore (memory mode)
|
||||
|
||||
All mutating and querying methods accept a ``user_id`` parameter with
|
||||
three-state semantics (see :mod:`deerflow.runtime.user_context`):
|
||||
All mutating and querying methods accept both a ``user_id`` parameter
|
||||
(member-scoped owner check) and a ``workspace_id`` parameter (tenant
|
||||
scope). Both follow three-state semantics:
|
||||
|
||||
- ``AUTO`` (default): resolve from the request-scoped contextvar.
|
||||
- Explicit ``str``: use the provided value verbatim.
|
||||
- Explicit ``None``: bypass owner filtering (migration/CLI only).
|
||||
- Explicit ``None``: bypass that filter (migration / CLI only).
|
||||
|
||||
The workspace scope is the **outer** boundary: a row in workspace A is
|
||||
unreachable from any user_id under workspace B. ``check_access`` returns
|
||||
False on cross-workspace mismatch so the route layer can convert it into
|
||||
a 404 instead of leaking thread existence across tenants.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -17,6 +23,8 @@ from __future__ import annotations
|
||||
import abc
|
||||
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel
|
||||
from deerflow.runtime.workspace_context import AUTO as WORKSPACE_AUTO
|
||||
from deerflow.runtime.workspace_context import _AutoSentinel as _WorkspaceAutoSentinel
|
||||
|
||||
|
||||
class ThreadMetaStore(abc.ABC):
|
||||
@@ -27,13 +35,20 @@ class ThreadMetaStore(abc.ABC):
|
||||
*,
|
||||
assistant_id: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
display_name: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
) -> dict:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def get(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> dict | None:
|
||||
async def get(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> dict | None:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
@@ -45,32 +60,72 @@ class ThreadMetaStore(abc.ABC):
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> list[dict]:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def update_display_name(self, thread_id: str, display_name: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
async def update_display_name(
|
||||
self,
|
||||
thread_id: str,
|
||||
display_name: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def update_status(self, thread_id: str, status: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
async def update_status(
|
||||
self,
|
||||
thread_id: str,
|
||||
status: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def update_metadata(self, thread_id: str, metadata: dict, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
async def update_metadata(
|
||||
self,
|
||||
thread_id: str,
|
||||
metadata: dict,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> None:
|
||||
"""Merge ``metadata`` into the thread's metadata field.
|
||||
|
||||
Existing keys are overwritten by the new values; keys absent from
|
||||
``metadata`` are preserved. No-op if the thread does not exist
|
||||
or the owner check fails.
|
||||
or the user/workspace check fails.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def check_access(self, thread_id: str, user_id: str, *, require_existing: bool = False) -> bool:
|
||||
"""Check if ``user_id`` has access to ``thread_id``."""
|
||||
async def check_access(
|
||||
self,
|
||||
thread_id: str,
|
||||
user_id: str,
|
||||
workspace_id: str,
|
||||
*,
|
||||
require_existing: bool = False,
|
||||
) -> bool:
|
||||
"""Check whether ``user_id`` (in ``workspace_id``) can access ``thread_id``.
|
||||
|
||||
Cross-workspace access returns ``False`` unconditionally so the
|
||||
decorator layer can convert it into a 404 — never leak the
|
||||
existence of a thread that belongs to a different tenant.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def delete(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
async def delete(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@@ -13,6 +13,13 @@ from langgraph.store.base import BaseStore
|
||||
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
from deerflow.runtime.workspace_context import AUTO as WORKSPACE_AUTO
|
||||
from deerflow.runtime.workspace_context import (
|
||||
_AutoSentinel as _WorkspaceAutoSentinel,
|
||||
)
|
||||
from deerflow.runtime.workspace_context import (
|
||||
resolve_workspace_id,
|
||||
)
|
||||
from deerflow.utils.time import coerce_iso, now_iso
|
||||
|
||||
THREADS_NS: tuple[str, ...] = ("threads",)
|
||||
@@ -26,15 +33,19 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
self,
|
||||
thread_id: str,
|
||||
user_id: str | None | _AutoSentinel,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel,
|
||||
method_name: str,
|
||||
) -> dict | None:
|
||||
"""Fetch a record and verify ownership. Returns a mutable copy, or None."""
|
||||
resolved = resolve_user_id(user_id, method_name=method_name)
|
||||
"""Fetch a record and verify workspace + ownership. Returns a mutable copy, or None."""
|
||||
resolved_user = resolve_user_id(user_id, method_name=method_name)
|
||||
resolved_workspace = resolve_workspace_id(workspace_id, method_name=method_name)
|
||||
item = await self._store.aget(THREADS_NS, thread_id)
|
||||
if item is None:
|
||||
return None
|
||||
record = dict(item.value)
|
||||
if resolved is not None and record.get("user_id") != resolved:
|
||||
if resolved_workspace is not None and record.get("workspace_id") != resolved_workspace:
|
||||
return None
|
||||
if resolved_user is not None and record.get("user_id") != resolved_user:
|
||||
return None
|
||||
return record
|
||||
|
||||
@@ -44,15 +55,18 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
*,
|
||||
assistant_id: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
display_name: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
) -> dict:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="MemoryThreadMetaStore.create")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="MemoryThreadMetaStore.create")
|
||||
now = now_iso()
|
||||
record: dict[str, Any] = {
|
||||
"thread_id": thread_id,
|
||||
"assistant_id": assistant_id,
|
||||
"user_id": resolved_user_id,
|
||||
"workspace_id": resolved_workspace_id,
|
||||
"display_name": display_name,
|
||||
"status": "idle",
|
||||
"metadata": metadata or {},
|
||||
@@ -63,8 +77,14 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
return record
|
||||
|
||||
async def get(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> dict | None:
|
||||
return await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.get")
|
||||
async def get(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> dict | None:
|
||||
return await self._get_owned_record(thread_id, user_id, workspace_id, "MemoryThreadMetaStore.get")
|
||||
|
||||
async def search(
|
||||
self,
|
||||
@@ -74,13 +94,17 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="MemoryThreadMetaStore.search")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="MemoryThreadMetaStore.search")
|
||||
filter_dict: dict[str, Any] = {}
|
||||
if metadata:
|
||||
filter_dict.update(metadata)
|
||||
if status:
|
||||
filter_dict["status"] = status
|
||||
if resolved_workspace_id is not None:
|
||||
filter_dict["workspace_id"] = resolved_workspace_id
|
||||
if resolved_user_id is not None:
|
||||
filter_dict["user_id"] = resolved_user_id
|
||||
|
||||
@@ -92,33 +116,64 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
)
|
||||
return [self._item_to_dict(item) for item in items]
|
||||
|
||||
async def check_access(self, thread_id: str, user_id: str, *, require_existing: bool = False) -> bool:
|
||||
async def check_access(
|
||||
self,
|
||||
thread_id: str,
|
||||
user_id: str,
|
||||
workspace_id: str,
|
||||
*,
|
||||
require_existing: bool = False,
|
||||
) -> bool:
|
||||
item = await self._store.aget(THREADS_NS, thread_id)
|
||||
if item is None:
|
||||
return not require_existing
|
||||
record_workspace_id = item.value.get("workspace_id")
|
||||
if record_workspace_id is not None and record_workspace_id != workspace_id:
|
||||
return False
|
||||
record_user_id = item.value.get("user_id")
|
||||
if record_user_id is None:
|
||||
return True
|
||||
return record_user_id == user_id
|
||||
|
||||
async def update_display_name(self, thread_id: str, display_name: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_display_name")
|
||||
async def update_display_name(
|
||||
self,
|
||||
thread_id: str,
|
||||
display_name: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, workspace_id, "MemoryThreadMetaStore.update_display_name")
|
||||
if record is None:
|
||||
return
|
||||
record["display_name"] = display_name
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
|
||||
async def update_status(self, thread_id: str, status: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_status")
|
||||
async def update_status(
|
||||
self,
|
||||
thread_id: str,
|
||||
status: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, workspace_id, "MemoryThreadMetaStore.update_status")
|
||||
if record is None:
|
||||
return
|
||||
record["status"] = status
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
|
||||
async def update_metadata(self, thread_id: str, metadata: dict, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_metadata")
|
||||
async def update_metadata(
|
||||
self,
|
||||
thread_id: str,
|
||||
metadata: dict,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, workspace_id, "MemoryThreadMetaStore.update_metadata")
|
||||
if record is None:
|
||||
return
|
||||
merged = dict(record.get("metadata") or {})
|
||||
@@ -127,8 +182,14 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
|
||||
async def delete(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.delete")
|
||||
async def delete(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, workspace_id, "MemoryThreadMetaStore.delete")
|
||||
if record is None:
|
||||
return
|
||||
await self._store.adelete(THREADS_NS, thread_id)
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, String
|
||||
from sqlalchemy import JSON, DateTime, ForeignKey, Index, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -13,11 +13,25 @@ from deerflow.persistence.base import Base
|
||||
class ThreadMetaRow(Base):
|
||||
__tablename__ = "threads_meta"
|
||||
|
||||
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
assistant_id: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
display_name: Mapped[str | None] = mapped_column(String(256))
|
||||
status: Mapped[str] = mapped_column(String(20), default="idle")
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True, comment="会话主键(LangGraph thread_id)")
|
||||
assistant_id: Mapped[str | None] = mapped_column(String(128), index=True, comment="关联的 Assistant ID(自定义智能体名);为 NULL 表示默认 lead agent")
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True, comment="会话所有者;为 NULL 表示历史无主数据")
|
||||
workspace_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("workspaces.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
comment="所属 workspace。PR5 引入时 nullable 用于回填;alembic 0003 + PR6 仓储接入完成后 NOT NULL",
|
||||
)
|
||||
display_name: Mapped[str | None] = mapped_column(String(256), comment="会话显示名(自动生成的标题或用户手改)")
|
||||
status: Mapped[str] = mapped_column(String(20), default="idle", comment='会话状态:"idle" 空闲 / "busy" 正在产出')
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict, comment="任意扩展元数据(JSON)")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), comment="创建时间(UTC)")
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC), comment="最近更新时间(UTC,写入时自动更新)")
|
||||
|
||||
__table_args__ = (
|
||||
# Workspace-scoped "list threads of this user, newest first" index;
|
||||
# added by alembic 0002. Mirrored on the ORM side so create_all()
|
||||
# produces the same shape on fresh dev databases.
|
||||
Index("idx_threads_meta_workspace_user_updated", "workspace_id", "user_id", "updated_at"),
|
||||
{"comment": "会话元数据(每个 LangGraph thread 的概要信息)"},
|
||||
)
|
||||
|
||||
@@ -11,6 +11,13 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
from deerflow.runtime.workspace_context import AUTO as WORKSPACE_AUTO
|
||||
from deerflow.runtime.workspace_context import (
|
||||
_AutoSentinel as _WorkspaceAutoSentinel,
|
||||
)
|
||||
from deerflow.runtime.workspace_context import (
|
||||
resolve_workspace_id,
|
||||
)
|
||||
|
||||
|
||||
class ThreadMetaRepository(ThreadMetaStore):
|
||||
@@ -33,17 +40,21 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
*,
|
||||
assistant_id: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
display_name: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
) -> dict:
|
||||
# Auto-resolve user_id from contextvar when AUTO; explicit None
|
||||
# creates an orphan row (used by migration scripts).
|
||||
# Auto-resolve both user_id and workspace_id from contextvars when
|
||||
# AUTO; explicit None creates an orphan row (used by migration
|
||||
# scripts that intentionally bypass scope).
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.create")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="ThreadMetaRepository.create")
|
||||
now = datetime.now(UTC)
|
||||
row = ThreadMetaRow(
|
||||
thread_id=thread_id,
|
||||
assistant_id=assistant_id,
|
||||
user_id=resolved_user_id,
|
||||
workspace_id=resolved_workspace_id,
|
||||
display_name=display_name,
|
||||
metadata_json=metadata or {},
|
||||
created_at=now,
|
||||
@@ -60,43 +71,52 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
thread_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> dict | None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.get")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="ThreadMetaRepository.get")
|
||||
stmt = select(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id)
|
||||
if resolved_workspace_id is not None:
|
||||
stmt = stmt.where(ThreadMetaRow.workspace_id == resolved_workspace_id)
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ThreadMetaRow, thread_id)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
# Enforce owner filter unless explicitly bypassed (user_id=None).
|
||||
# Owner filter still applies inside the workspace scope.
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def check_access(self, thread_id: str, user_id: str, *, require_existing: bool = False) -> bool:
|
||||
"""Check if ``user_id`` has access to ``thread_id``.
|
||||
async def check_access(
|
||||
self,
|
||||
thread_id: str,
|
||||
user_id: str,
|
||||
workspace_id: str,
|
||||
*,
|
||||
require_existing: bool = False,
|
||||
) -> bool:
|
||||
"""Check if ``user_id`` in ``workspace_id`` has access to ``thread_id``.
|
||||
|
||||
Two modes — one row, two distinct semantics depending on what
|
||||
the caller is about to do:
|
||||
Three filters layered, from outside in:
|
||||
|
||||
- ``require_existing=False`` (default, permissive):
|
||||
Returns True for: row missing (untracked legacy thread),
|
||||
``row.user_id`` is None (shared / pre-auth data),
|
||||
or ``row.user_id == user_id``. Use for **read-style**
|
||||
decorators where treating an untracked thread as accessible
|
||||
preserves backward-compat.
|
||||
|
||||
- ``require_existing=True`` (strict):
|
||||
Returns True **only** when the row exists AND
|
||||
(``row.user_id == user_id`` OR ``row.user_id is None``).
|
||||
Use for **destructive / mutating** decorators (DELETE, PATCH,
|
||||
state-update) so a thread that has *already been deleted*
|
||||
cannot be re-targeted by any caller — closing the
|
||||
delete-idempotence cross-user gap where the row vanishing
|
||||
made every other user appear to "own" it.
|
||||
- Cross-workspace is **always** denied (returns False), even when
|
||||
the row exists and ``user_id`` matches. The decorator layer
|
||||
converts a False into a 404 so cross-tenant access never leaks
|
||||
the existence of a thread.
|
||||
- Missing row honours ``require_existing``: False by default
|
||||
(permissive — untracked legacy threads still readable), True
|
||||
for destructive routes (DELETE / PATCH) so a re-targeted ghost
|
||||
row cannot be claimed.
|
||||
- Within the workspace, ``row.user_id IS NULL`` keeps the legacy
|
||||
"shared / pre-auth" semantics — readable by anyone in the
|
||||
workspace. ``row.user_id == user_id`` is the normal case.
|
||||
"""
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ThreadMetaRow, thread_id)
|
||||
if row is None:
|
||||
return not require_existing
|
||||
if row.workspace_id is not None and row.workspace_id != workspace_id:
|
||||
return False
|
||||
if row.user_id is None:
|
||||
return True
|
||||
return row.user_id == user_id
|
||||
@@ -109,14 +129,19 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> list[dict]:
|
||||
"""Search threads with optional metadata and status filters.
|
||||
|
||||
Owner filter is enforced by default: caller must be in a user
|
||||
context. Pass ``user_id=None`` to bypass (migration/CLI).
|
||||
Both workspace and owner filters are enforced by default. Pass
|
||||
``workspace_id=None`` and / or ``user_id=None`` to bypass for
|
||||
migration / CLI paths.
|
||||
"""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.search")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="ThreadMetaRepository.search")
|
||||
stmt = select(ThreadMetaRow).order_by(ThreadMetaRow.updated_at.desc())
|
||||
if resolved_workspace_id is not None:
|
||||
stmt = stmt.where(ThreadMetaRow.workspace_id == resolved_workspace_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(ThreadMetaRow.user_id == resolved_user_id)
|
||||
if status:
|
||||
@@ -138,12 +163,22 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
async def _check_ownership(self, session: AsyncSession, thread_id: str, resolved_user_id: str | None) -> bool:
|
||||
"""Return True if the row exists and is owned (or filter bypassed)."""
|
||||
if resolved_user_id is None:
|
||||
return True # explicit bypass
|
||||
async def _check_ownership(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
thread_id: str,
|
||||
resolved_user_id: str | None,
|
||||
resolved_workspace_id: str | None,
|
||||
) -> bool:
|
||||
"""Return True if the row exists, is in scope, and is owned (or filter bypassed)."""
|
||||
row = await session.get(ThreadMetaRow, thread_id)
|
||||
return row is not None and row.user_id == resolved_user_id
|
||||
if row is None:
|
||||
return False
|
||||
if resolved_workspace_id is not None and row.workspace_id != resolved_workspace_id:
|
||||
return False
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def update_display_name(
|
||||
self,
|
||||
@@ -151,11 +186,13 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
display_name: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> None:
|
||||
"""Update the display_name (title) for a thread."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.update_display_name")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="ThreadMetaRepository.update_display_name")
|
||||
async with self._sf() as session:
|
||||
if not await self._check_ownership(session, thread_id, resolved_user_id):
|
||||
if not await self._check_ownership(session, thread_id, resolved_user_id, resolved_workspace_id):
|
||||
return
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(display_name=display_name, updated_at=datetime.now(UTC)))
|
||||
await session.commit()
|
||||
@@ -166,10 +203,12 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
status: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.update_status")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="ThreadMetaRepository.update_status")
|
||||
async with self._sf() as session:
|
||||
if not await self._check_ownership(session, thread_id, resolved_user_id):
|
||||
if not await self._check_ownership(session, thread_id, resolved_user_id, resolved_workspace_id):
|
||||
return
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(status=status, updated_at=datetime.now(UTC)))
|
||||
await session.commit()
|
||||
@@ -180,18 +219,22 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
metadata: dict,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> None:
|
||||
"""Merge ``metadata`` into ``metadata_json``.
|
||||
|
||||
Read-modify-write inside a single session/transaction so concurrent
|
||||
callers see consistent state. No-op if the row does not exist or
|
||||
the user_id check fails.
|
||||
the workspace / user check fails.
|
||||
"""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.update_metadata")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="ThreadMetaRepository.update_metadata")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ThreadMetaRow, thread_id)
|
||||
if row is None:
|
||||
return
|
||||
if resolved_workspace_id is not None and row.workspace_id != resolved_workspace_id:
|
||||
return
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return
|
||||
merged = dict(row.metadata_json or {})
|
||||
@@ -205,12 +248,16 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
thread_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
) -> None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.delete")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="ThreadMetaRepository.delete")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ThreadMetaRow, thread_id)
|
||||
if row is None:
|
||||
return
|
||||
if resolved_workspace_id is not None and row.workspace_id != resolved_workspace_id:
|
||||
return
|
||||
if resolved_user_id is not None and row.user_id != resolved_user_id:
|
||||
return
|
||||
await session.delete(row)
|
||||
|
||||
@@ -13,7 +13,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Index, String, text
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, String, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -22,31 +22,26 @@ from deerflow.persistence.base import Base
|
||||
class UserRow(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
# UUIDs are stored as 36-char strings for cross-backend portability.
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
|
||||
email: Mapped[str] = mapped_column(String(320), unique=True, nullable=False, index=True)
|
||||
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# "admin" | "user" — kept as plain string to avoid ALTER TABLE pain
|
||||
# when new roles are introduced.
|
||||
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user")
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, comment="用户主键,UUID 字符串(36 字符),跨数据库可移植")
|
||||
email: Mapped[str] = mapped_column(String(320), unique=True, nullable=False, index=True, comment="登录邮箱,全局唯一")
|
||||
password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True, comment="本地账户的密码哈希;OAuth-only 用户为 NULL")
|
||||
system_role: Mapped[str] = mapped_column(String(16), nullable=False, default="user", comment='系统角色:"admin" 或 "user";用字符串以便未来扩展角色而不必 ALTER TABLE')
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
comment="账户创建时间(UTC)",
|
||||
)
|
||||
oauth_provider: Mapped[str | None] = mapped_column(String(32), nullable=True, comment="OAuth 提供商名(如 google/github);本地账户为 NULL")
|
||||
oauth_id: Mapped[str | None] = mapped_column(String(128), nullable=True, comment="OAuth 提供商内的用户 ID;与 oauth_provider 组合需唯一")
|
||||
needs_setup: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, comment="是否需要完成首次设置(admin 自动创建后改密码/邮箱)")
|
||||
token_version: Mapped[int] = mapped_column(nullable=False, default=0, comment="JWT 令牌版本号;自增即吊销该用户所有旧令牌")
|
||||
default_workspace_id: Mapped[str | None] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("workspaces.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
comment="登录后默认进入的 workspace;NULL 时强制走 picker(user 多 workspace 场景)",
|
||||
)
|
||||
|
||||
# OAuth linkage (optional). A partial unique index enforces one
|
||||
# account per (provider, oauth_id) pair, leaving NULL/NULL rows
|
||||
# unconstrained so plain password accounts can coexist.
|
||||
oauth_provider: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
oauth_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# Auth lifecycle flags
|
||||
needs_setup: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
token_version: Mapped[int] = mapped_column(nullable=False, default=0)
|
||||
|
||||
__table_args__ = (
|
||||
Index(
|
||||
@@ -56,4 +51,5 @@ class UserRow(Base):
|
||||
unique=True,
|
||||
sqlite_where=text("oauth_provider IS NOT NULL AND oauth_id IS NOT NULL"),
|
||||
),
|
||||
{"comment": "用户账户表(本地密码登录 + OAuth 联合登录)"},
|
||||
)
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Workspace persistence — ORM model + repository.
|
||||
|
||||
A workspace is the multi-tenant scope unit. Every user has at least
|
||||
one (auto-created on registration; their personal workspace where
|
||||
they are sole owner). Team plans get multi-member workspaces.
|
||||
|
||||
Stage 0 PR3 introduces the schema + repository; PR4 wires it into
|
||||
the registration / login flow; PR5+ ALTER existing business tables
|
||||
to FK back to ``workspaces.id``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workspace.model import WorkspaceRow
|
||||
from deerflow.persistence.workspace.sql import WorkspaceRepository, WorkspaceValidationError
|
||||
|
||||
__all__ = ["WorkspaceRepository", "WorkspaceRow", "WorkspaceValidationError"]
|
||||
@@ -0,0 +1,58 @@
|
||||
"""ORM model for workspaces (multi-tenant scope)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class WorkspaceRow(Base):
|
||||
__tablename__ = "workspaces"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
primary_key=True,
|
||||
comment="工作空间主键,UUID 字符串(36 字符),与 users.id 类型对齐",
|
||||
)
|
||||
name: Mapped[str] = mapped_column(
|
||||
String(64),
|
||||
nullable=False,
|
||||
comment="工作空间显示名(用户注册时默认 <email 前缀>'s Workspace)",
|
||||
)
|
||||
slug: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
comment="URL 标识(^[a-z0-9](-?[a-z0-9])*$,3-32 字符);全局唯一,DB 存小写",
|
||||
)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16),
|
||||
nullable=False,
|
||||
default="active",
|
||||
comment='状态:"active"(正常)/ "suspended"(平台 admin 暂停)/ "deleted"(软删)',
|
||||
)
|
||||
owner_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("users.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
comment="所有者 user_id;与 workspace_memberships 中 role='owner' 行严格一致(事务保证);删除 owner 时 RESTRICT 阻拦(必须先转让所有权)",
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
comment="创建时间(UTC)",
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
comment="最近更新时间(UTC,写入时自动更新)",
|
||||
)
|
||||
|
||||
__table_args__ = ({"comment": ("工作空间表(多租户隔离粒度单位)。每个用户注册时自动建一个 1 人 workspace,owner 即注册者。团队订阅时 workspace 可有多个 member。")},)
|
||||
@@ -0,0 +1,230 @@
|
||||
"""SQLAlchemy-backed workspace repository.
|
||||
|
||||
CRUD + slug lookup for ``workspaces``. Membership-aware methods
|
||||
(``get``, ``list_by_user``) JOIN against ``workspace_memberships`` so
|
||||
callers cannot read workspaces they don't belong to.
|
||||
|
||||
Three-state ``user_id`` parameter (same convention as
|
||||
:class:`ThreadMetaRepository`):
|
||||
- ``AUTO`` → read from contextvar; raise if unset
|
||||
- explicit ``str`` → override contextvar (admin / tests)
|
||||
- explicit ``None`` → no filter (migration / CLI only)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workspace.model import WorkspaceRow
|
||||
from deerflow.persistence.workspace_membership.model import WorkspaceMembershipRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
# slug 字符集 / 长度(与 workspace-schema-design §2.1 锁定一致)。
|
||||
_SLUG_PATTERN = re.compile(r"^[a-z0-9](-?[a-z0-9])*$")
|
||||
_SLUG_MIN_LEN = 3
|
||||
_SLUG_MAX_LEN = 32
|
||||
|
||||
# slug 黑名单(应用层校验,不写 DB constraint)。包含 ADR-007 §4 保留 slug
|
||||
# + 路径 + Next.js 保留 + 业务保留词。
|
||||
SLUG_BLACKLIST = frozenset(
|
||||
{
|
||||
"admin",
|
||||
"api",
|
||||
"auth",
|
||||
"login",
|
||||
"signup",
|
||||
"accept-invite",
|
||||
"pricing",
|
||||
"docs",
|
||||
"status",
|
||||
"platform",
|
||||
"system",
|
||||
"health",
|
||||
"static",
|
||||
"public",
|
||||
"favicon.ico",
|
||||
"robots.txt",
|
||||
"sitemap.xml",
|
||||
"_next",
|
||||
".well-known",
|
||||
"settings",
|
||||
"billing",
|
||||
"onboarding",
|
||||
"select-workspace",
|
||||
}
|
||||
)
|
||||
|
||||
# 允许的 status 集合。
|
||||
_VALID_STATUSES = frozenset({"active", "suspended", "deleted"})
|
||||
|
||||
|
||||
class WorkspaceValidationError(ValueError):
|
||||
"""Raised when workspace input fails application-layer validation
|
||||
(slug format / blacklist / status enum)."""
|
||||
|
||||
|
||||
def _validate_slug(slug: str) -> None:
|
||||
"""Raise :class:`WorkspaceValidationError` if slug is invalid."""
|
||||
if not isinstance(slug, str):
|
||||
raise WorkspaceValidationError(f"slug must be a string, got {type(slug).__name__}")
|
||||
if not (_SLUG_MIN_LEN <= len(slug) <= _SLUG_MAX_LEN):
|
||||
raise WorkspaceValidationError(f"slug length must be between {_SLUG_MIN_LEN} and {_SLUG_MAX_LEN}, got {len(slug)}")
|
||||
if not _SLUG_PATTERN.fullmatch(slug):
|
||||
raise WorkspaceValidationError(f"slug {slug!r} does not match required pattern ^[a-z0-9](-?[a-z0-9])*$")
|
||||
if slug in SLUG_BLACKLIST:
|
||||
raise WorkspaceValidationError(f"slug {slug!r} is reserved")
|
||||
|
||||
|
||||
def _validate_status(status: str) -> None:
|
||||
if status not in _VALID_STATUSES:
|
||||
raise WorkspaceValidationError(f"status {status!r} is not in allowed set {_VALID_STATUSES!r}")
|
||||
|
||||
|
||||
class WorkspaceRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: WorkspaceRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"name": row.name,
|
||||
"slug": row.slug,
|
||||
"status": row.status,
|
||||
"owner_id": row.owner_id,
|
||||
"created_at": row.created_at.isoformat() if row.created_at else None,
|
||||
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
||||
}
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
name: str,
|
||||
slug: str,
|
||||
owner_id: str,
|
||||
workspace_id: str | None = None,
|
||||
status: str = "active",
|
||||
) -> dict[str, Any]:
|
||||
"""Create a new workspace.
|
||||
|
||||
``workspace_id`` is optional — UUID v4 is generated if omitted.
|
||||
Caller is responsible for creating the matching ``owner`` row
|
||||
in ``workspace_memberships`` (this is typically done in the same
|
||||
transaction by the registration flow; we deliberately don't bundle
|
||||
it here to keep the repository single-responsibility).
|
||||
|
||||
Raises :class:`WorkspaceValidationError` for invalid slug / status.
|
||||
Raises :class:`sqlalchemy.exc.IntegrityError` for slug collision
|
||||
or invalid owner_id FK.
|
||||
"""
|
||||
_validate_slug(slug)
|
||||
_validate_status(status)
|
||||
wid = workspace_id or str(uuid.uuid4())
|
||||
now = datetime.now(UTC)
|
||||
row = WorkspaceRow(
|
||||
id=wid,
|
||||
name=name,
|
||||
slug=slug,
|
||||
status=status,
|
||||
owner_id=owner_id,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get(
|
||||
self,
|
||||
workspace_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Return workspace row IFF caller is a member.
|
||||
|
||||
``user_id=None`` bypasses the membership filter (migration / CLI).
|
||||
Returns ``None`` if the workspace does not exist or the caller is
|
||||
not a member.
|
||||
"""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="WorkspaceRepository.get")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkspaceRow, workspace_id)
|
||||
if row is None:
|
||||
return None
|
||||
if resolved_user_id is None:
|
||||
# Explicit bypass (migration / admin path).
|
||||
return self._row_to_dict(row)
|
||||
# Membership check via separate SELECT (cheap; index covers it).
|
||||
membership = await session.execute(
|
||||
select(WorkspaceMembershipRow).where(
|
||||
WorkspaceMembershipRow.workspace_id == workspace_id,
|
||||
WorkspaceMembershipRow.user_id == resolved_user_id,
|
||||
)
|
||||
)
|
||||
if membership.scalar_one_or_none() is None:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get_by_slug(self, slug: str) -> dict[str, Any] | None:
|
||||
"""Public slug lookup — does NOT check membership.
|
||||
|
||||
Used by path-based routing (``/{slug}/...``) where we need to
|
||||
resolve slug → workspace_id BEFORE we know if caller belongs.
|
||||
Membership check happens downstream in the route handler.
|
||||
"""
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(select(WorkspaceRow).where(WorkspaceRow.slug == slug))
|
||||
row = result.scalar_one_or_none()
|
||||
return self._row_to_dict(row) if row else None
|
||||
|
||||
async def list_by_user(
|
||||
self,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return all workspaces caller is a member of, ordered by joined_at desc.
|
||||
|
||||
``user_id=None`` lists ALL workspaces (migration / admin path).
|
||||
"""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="WorkspaceRepository.list_by_user")
|
||||
async with self._sf() as session:
|
||||
stmt = select(WorkspaceRow).order_by(WorkspaceRow.created_at.desc())
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.join(
|
||||
WorkspaceMembershipRow,
|
||||
WorkspaceMembershipRow.workspace_id == WorkspaceRow.id,
|
||||
).where(WorkspaceMembershipRow.user_id == resolved_user_id)
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
async def update_status(self, workspace_id: str, status: str) -> None:
|
||||
"""Platform-admin operation: change workspace status (active/suspended/deleted).
|
||||
|
||||
No membership check — this is for platform-level operations. Audit
|
||||
logging belongs at the route layer.
|
||||
"""
|
||||
_validate_status(status)
|
||||
async with self._sf() as session:
|
||||
await session.execute(update(WorkspaceRow).where(WorkspaceRow.id == workspace_id).values(status=status, updated_at=datetime.now(UTC)))
|
||||
await session.commit()
|
||||
|
||||
async def delete(self, workspace_id: str) -> None:
|
||||
"""Hard-delete a workspace. CASCADE drops all memberships.
|
||||
|
||||
Intentionally no membership check — caller (platform admin route)
|
||||
must enforce authorization. Stage 0 doesn't expose this to end
|
||||
users; Stage 2+ adds it behind owner_only permission.
|
||||
"""
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkspaceRow, workspace_id)
|
||||
if row is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Workspace membership persistence — ORM model + repository.
|
||||
|
||||
Tracks who is a member of which workspace and what their role is
|
||||
within that workspace. Composite primary key ``(workspace_id, user_id)``;
|
||||
``role`` follows the three-state ``owner`` / ``admin`` / ``member`` model
|
||||
(Stage 0 only writes ``'owner'``; Stage 2 RBAC rollout opens admin/member).
|
||||
|
||||
A workspace MUST have exactly one ``owner`` — enforced by a partial
|
||||
unique index. Owner transfer is a two-row transactional swap.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workspace_membership.model import WorkspaceMembershipRow
|
||||
from deerflow.persistence.workspace_membership.sql import (
|
||||
MembershipValidationError,
|
||||
WorkspaceMembershipRepository,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MembershipValidationError",
|
||||
"WorkspaceMembershipRepository",
|
||||
"WorkspaceMembershipRow",
|
||||
]
|
||||
@@ -0,0 +1,62 @@
|
||||
"""ORM model for workspace memberships (which user is in which workspace, with what role)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, Index, String, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class WorkspaceMembershipRow(Base):
|
||||
__tablename__ = "workspace_memberships"
|
||||
|
||||
workspace_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("workspaces.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
comment="所属 workspace;workspace 删除时级联清掉成员记录",
|
||||
)
|
||||
user_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("users.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
comment="成员 user_id;用户删除时级联清掉成员记录",
|
||||
)
|
||||
role: Mapped[str] = mapped_column(
|
||||
String(16),
|
||||
nullable=False,
|
||||
comment='角色字符串:Stage 0 仅写 "owner";Stage 2 RBAC 打开 "admin"/"member"。用 String(16) 而非 enum 以便未来加 "viewer"/"auditor" 不动 schema',
|
||||
)
|
||||
invited_by: Mapped[str | None] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("users.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
comment="邀请人 user_id(Stage 2 invitation 流程才写);邀请人被删时此字段清空(不影响成员记录本身)",
|
||||
)
|
||||
joined_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
comment="加入 workspace 的时间(UTC)",
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
# 倒查索引:列出 user 所属的所有 workspace(/auth/me 用)。
|
||||
# 复合主键 (workspace_id, user_id) 的前导列是 workspace_id,
|
||||
# 所以"按 user_id 查"需要单独的索引。
|
||||
Index("idx_workspace_memberships_user", "user_id", "workspace_id"),
|
||||
# 一个 workspace 严格 1 个 owner —— partial unique on role='owner'。
|
||||
# SQLite + Postgres 都支持 WHERE 子句的 partial unique;双驱动
|
||||
# 维护两套等价 where 表达式。
|
||||
Index(
|
||||
"idx_one_owner_per_workspace",
|
||||
"workspace_id",
|
||||
unique=True,
|
||||
sqlite_where=text("role = 'owner'"),
|
||||
postgresql_where=text("role = 'owner'"),
|
||||
),
|
||||
{"comment": ("工作空间成员表(multi-tenant RBAC)。复合 PK (workspace_id, user_id);每个 workspace 必有恰好 1 个 owner(partial unique 约束保证)。Stage 0 仅写 owner;Stage 2 RBAC 打开 admin/member。")},
|
||||
)
|
||||
@@ -0,0 +1,149 @@
|
||||
"""SQLAlchemy-backed workspace membership repository.
|
||||
|
||||
Manages the ``workspace_memberships`` join table. Stage 0 only writes
|
||||
``role='owner'`` (single-user workspaces); the repository accepts the
|
||||
full Stage 2 RBAC role enum so the schema is forward-compatible.
|
||||
|
||||
Owner transfer is intentionally NOT modelled here as a single method —
|
||||
it requires a two-row transactional swap with careful retry semantics,
|
||||
and belongs in the auth router (PR4+) where it can be wrapped in a
|
||||
permission check.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workspace_membership.model import WorkspaceMembershipRow
|
||||
|
||||
# 允许的 role 字面值。Stage 0 实际仅写 owner;admin/member 留给 Stage 2 RBAC。
|
||||
_VALID_ROLES = frozenset({"owner", "admin", "member"})
|
||||
|
||||
|
||||
class MembershipValidationError(ValueError):
|
||||
"""Raised when role is not in the allowed enum."""
|
||||
|
||||
|
||||
def _validate_role(role: str) -> None:
|
||||
if role not in _VALID_ROLES:
|
||||
raise MembershipValidationError(f"role {role!r} is not in allowed set {_VALID_ROLES!r}")
|
||||
|
||||
|
||||
class WorkspaceMembershipRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: WorkspaceMembershipRow) -> dict[str, Any]:
|
||||
return {
|
||||
"workspace_id": row.workspace_id,
|
||||
"user_id": row.user_id,
|
||||
"role": row.role,
|
||||
"invited_by": row.invited_by,
|
||||
"joined_at": row.joined_at.isoformat() if row.joined_at else None,
|
||||
}
|
||||
|
||||
async def add(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
user_id: str,
|
||||
role: str,
|
||||
invited_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Insert a new membership row.
|
||||
|
||||
Raises:
|
||||
- :class:`MembershipValidationError` for unknown role values
|
||||
- :class:`sqlalchemy.exc.IntegrityError` for:
|
||||
- duplicate (workspace_id, user_id) — composite PK collision
|
||||
- second ``owner`` in the same workspace — partial unique index
|
||||
- invalid workspace_id / user_id / invited_by FK
|
||||
"""
|
||||
_validate_role(role)
|
||||
row = WorkspaceMembershipRow(
|
||||
workspace_id=workspace_id,
|
||||
user_id=user_id,
|
||||
role=role,
|
||||
invited_by=invited_by,
|
||||
joined_at=datetime.now(UTC),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def remove(self, *, workspace_id: str, user_id: str) -> bool:
|
||||
"""Delete a membership row; returns True if a row was deleted.
|
||||
|
||||
Owner removal is allowed at the repository layer — auth router
|
||||
(PR4) layers the "cannot remove last owner" rule on top.
|
||||
"""
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(WorkspaceMembershipRow).where(
|
||||
WorkspaceMembershipRow.workspace_id == workspace_id,
|
||||
WorkspaceMembershipRow.user_id == user_id,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return (result.rowcount or 0) > 0
|
||||
|
||||
async def list_by_user(self, *, user_id: str) -> list[dict[str, Any]]:
|
||||
"""Return all memberships for ``user_id``, ordered by joined_at desc.
|
||||
|
||||
No contextvar resolution here — caller is responsible for passing
|
||||
the correct user_id. Used by ``/auth/me`` to list workspaces a
|
||||
user belongs to.
|
||||
"""
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(select(WorkspaceMembershipRow).where(WorkspaceMembershipRow.user_id == user_id).order_by(WorkspaceMembershipRow.joined_at.desc()))
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
async def list_by_workspace(self, *, workspace_id: str) -> list[dict[str, Any]]:
|
||||
"""Return all members of a workspace, ordered by joined_at asc."""
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(select(WorkspaceMembershipRow).where(WorkspaceMembershipRow.workspace_id == workspace_id).order_by(WorkspaceMembershipRow.joined_at.asc()))
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
async def get_role(self, *, workspace_id: str, user_id: str) -> str | None:
|
||||
"""Return the role string, or None if user is not a member."""
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(WorkspaceMembershipRow.role).where(
|
||||
WorkspaceMembershipRow.workspace_id == workspace_id,
|
||||
WorkspaceMembershipRow.user_id == user_id,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def change_role(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
user_id: str,
|
||||
new_role: str,
|
||||
) -> bool:
|
||||
"""Update a member's role; returns True iff a row was updated.
|
||||
|
||||
Validates ``new_role`` against the allowed enum. Owner-transfer
|
||||
flow needs to swap two rows atomically — do that with a manual
|
||||
transaction in the caller; this method is for non-owner changes.
|
||||
"""
|
||||
_validate_role(new_role)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkspaceMembershipRow)
|
||||
.where(
|
||||
WorkspaceMembershipRow.workspace_id == workspace_id,
|
||||
WorkspaceMembershipRow.user_id == user_id,
|
||||
)
|
||||
.values(role=new_role)
|
||||
)
|
||||
await session.commit()
|
||||
return (result.rowcount or 0) > 0
|
||||
@@ -20,6 +20,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from langgraph.types import Checkpointer
|
||||
@@ -114,7 +115,14 @@ async def _async_checkpointer_from_database(db_config) -> AsyncIterator[Checkpoi
|
||||
if not db_config.postgres_url:
|
||||
raise ValueError("database.postgres_url is required for the postgres backend")
|
||||
|
||||
async with AsyncPostgresSaver.from_conn_string(db_config.postgres_url) as saver:
|
||||
# LangGraph's AsyncPostgresSaver wraps psycopg directly and expects a
|
||||
# libpq-style conninfo (`postgresql://...`). DeerFlow's own SQLAlchemy
|
||||
# engine uses the same `postgres_url` but needs the `+asyncpg` dialect
|
||||
# prefix. Strip the dialect prefix here so the same env var/config
|
||||
# value satisfies both paths.
|
||||
lg_conn_str = re.sub(r"^postgresql\+\w+://", "postgresql://", db_config.postgres_url)
|
||||
|
||||
async with AsyncPostgresSaver.from_conn_string(lg_conn_str) as saver:
|
||||
await saver.setup()
|
||||
yield saver
|
||||
return
|
||||
|
||||
@@ -17,6 +17,14 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.runtime.events.store.base import RunEventStore
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, get_current_user, resolve_user_id
|
||||
from deerflow.runtime.workspace_context import AUTO as WORKSPACE_AUTO
|
||||
from deerflow.runtime.workspace_context import (
|
||||
_AutoSentinel as _WorkspaceAutoSentinel,
|
||||
)
|
||||
from deerflow.runtime.workspace_context import (
|
||||
get_current_workspace,
|
||||
resolve_workspace_id,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -86,6 +94,19 @@ class DbRunEventStore(RunEventStore):
|
||||
user = get_current_user()
|
||||
return str(user.id) if user is not None else None
|
||||
|
||||
@staticmethod
|
||||
def _workspace_id_from_context() -> str | None:
|
||||
"""Soft read of workspace_id from contextvar for write paths.
|
||||
|
||||
Mirrors :meth:`_user_id_from_context`. Returns ``None`` (no stamp)
|
||||
when no workspace is in context — typical for background worker
|
||||
writes that fire outside an HTTP request. The DB column is
|
||||
nullable through PR5 and becomes NOT NULL only after the alembic
|
||||
0003 migration runs (verified by the backfill path).
|
||||
"""
|
||||
workspace = get_current_workspace()
|
||||
return str(workspace.id) if workspace is not None else None
|
||||
|
||||
async def put(self, *, thread_id, run_id, event_type, category, content="", metadata=None, created_at=None): # noqa: D401
|
||||
"""Write a single event — low-frequency path only.
|
||||
|
||||
@@ -98,6 +119,7 @@ class DbRunEventStore(RunEventStore):
|
||||
content, metadata = self._truncate_trace(category, content, metadata)
|
||||
db_content, metadata = self._content_to_db(content, metadata)
|
||||
user_id = self._user_id_from_context()
|
||||
workspace_id = self._workspace_id_from_context()
|
||||
async with self._sf() as session:
|
||||
async with session.begin():
|
||||
# Use FOR UPDATE to serialize seq assignment within a thread.
|
||||
@@ -109,6 +131,7 @@ class DbRunEventStore(RunEventStore):
|
||||
thread_id=thread_id,
|
||||
run_id=run_id,
|
||||
user_id=user_id,
|
||||
workspace_id=workspace_id,
|
||||
event_type=event_type,
|
||||
category=category,
|
||||
content=db_content,
|
||||
@@ -123,6 +146,7 @@ class DbRunEventStore(RunEventStore):
|
||||
if not events:
|
||||
return []
|
||||
user_id = self._user_id_from_context()
|
||||
workspace_id = self._workspace_id_from_context()
|
||||
async with self._sf() as session:
|
||||
async with session.begin():
|
||||
# Get max seq for the thread (assume all events in batch belong to same thread).
|
||||
@@ -143,6 +167,7 @@ class DbRunEventStore(RunEventStore):
|
||||
thread_id=e["thread_id"],
|
||||
run_id=e["run_id"],
|
||||
user_id=e.get("user_id", user_id),
|
||||
workspace_id=e.get("workspace_id", workspace_id),
|
||||
event_type=e["event_type"],
|
||||
category=category,
|
||||
content=db_content,
|
||||
@@ -162,9 +187,13 @@ class DbRunEventStore(RunEventStore):
|
||||
before_seq=None,
|
||||
after_seq=None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
):
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="DbRunEventStore.list_messages")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="DbRunEventStore.list_messages")
|
||||
stmt = select(RunEventRow).where(RunEventRow.thread_id == thread_id, RunEventRow.category == "message")
|
||||
if resolved_workspace_id is not None:
|
||||
stmt = stmt.where(RunEventRow.workspace_id == resolved_workspace_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(RunEventRow.user_id == resolved_user_id)
|
||||
if before_seq is not None:
|
||||
@@ -194,9 +223,13 @@ class DbRunEventStore(RunEventStore):
|
||||
event_types=None,
|
||||
limit=500,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
):
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="DbRunEventStore.list_events")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="DbRunEventStore.list_events")
|
||||
stmt = select(RunEventRow).where(RunEventRow.thread_id == thread_id, RunEventRow.run_id == run_id)
|
||||
if resolved_workspace_id is not None:
|
||||
stmt = stmt.where(RunEventRow.workspace_id == resolved_workspace_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(RunEventRow.user_id == resolved_user_id)
|
||||
if event_types:
|
||||
@@ -215,13 +248,17 @@ class DbRunEventStore(RunEventStore):
|
||||
before_seq=None,
|
||||
after_seq=None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
):
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="DbRunEventStore.list_messages_by_run")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="DbRunEventStore.list_messages_by_run")
|
||||
stmt = select(RunEventRow).where(
|
||||
RunEventRow.thread_id == thread_id,
|
||||
RunEventRow.run_id == run_id,
|
||||
RunEventRow.category == "message",
|
||||
)
|
||||
if resolved_workspace_id is not None:
|
||||
stmt = stmt.where(RunEventRow.workspace_id == resolved_workspace_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(RunEventRow.user_id == resolved_user_id)
|
||||
if before_seq is not None:
|
||||
@@ -246,9 +283,13 @@ class DbRunEventStore(RunEventStore):
|
||||
thread_id,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
):
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="DbRunEventStore.count_messages")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="DbRunEventStore.count_messages")
|
||||
stmt = select(func.count()).select_from(RunEventRow).where(RunEventRow.thread_id == thread_id, RunEventRow.category == "message")
|
||||
if resolved_workspace_id is not None:
|
||||
stmt = stmt.where(RunEventRow.workspace_id == resolved_workspace_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(RunEventRow.user_id == resolved_user_id)
|
||||
async with self._sf() as session:
|
||||
@@ -259,10 +300,14 @@ class DbRunEventStore(RunEventStore):
|
||||
thread_id,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
):
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="DbRunEventStore.delete_by_thread")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="DbRunEventStore.delete_by_thread")
|
||||
async with self._sf() as session:
|
||||
count_conditions = [RunEventRow.thread_id == thread_id]
|
||||
if resolved_workspace_id is not None:
|
||||
count_conditions.append(RunEventRow.workspace_id == resolved_workspace_id)
|
||||
if resolved_user_id is not None:
|
||||
count_conditions.append(RunEventRow.user_id == resolved_user_id)
|
||||
count_stmt = select(func.count()).select_from(RunEventRow).where(*count_conditions)
|
||||
@@ -278,10 +323,14 @@ class DbRunEventStore(RunEventStore):
|
||||
run_id,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
workspace_id: str | None | _WorkspaceAutoSentinel = WORKSPACE_AUTO,
|
||||
):
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="DbRunEventStore.delete_by_run")
|
||||
resolved_workspace_id = resolve_workspace_id(workspace_id, method_name="DbRunEventStore.delete_by_run")
|
||||
async with self._sf() as session:
|
||||
count_conditions = [RunEventRow.thread_id == thread_id, RunEventRow.run_id == run_id]
|
||||
if resolved_workspace_id is not None:
|
||||
count_conditions.append(RunEventRow.workspace_id == resolved_workspace_id)
|
||||
if resolved_user_id is not None:
|
||||
count_conditions.append(RunEventRow.user_id == resolved_user_id)
|
||||
count_stmt = select(func.count()).select_from(RunEventRow).where(*count_conditions)
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Request-scoped workspace context for multi-tenant authorization.
|
||||
|
||||
Sibling of :mod:`deerflow.runtime.user_context`. Holds a
|
||||
:class:`~contextvars.ContextVar` that the gateway's auth middleware sets
|
||||
after JWT verification (PR4 will wire this up). Repository methods read
|
||||
the contextvar via a sentinel default parameter, letting routers stay
|
||||
free of ``workspace_id`` boilerplate.
|
||||
|
||||
Three-state semantics for the repository ``workspace_id`` parameter:
|
||||
|
||||
- ``AUTO`` (sentinel, default): read from contextvar; raise
|
||||
:class:`RuntimeError` if unset.
|
||||
- Explicit ``str``: use the provided value, overriding contextvar.
|
||||
- Explicit ``None``: no WHERE clause — used only by migration scripts
|
||||
and admin CLIs that intentionally bypass workspace isolation.
|
||||
|
||||
Concept boundary
|
||||
----------------
|
||||
A workspace is the multi-tenant scope: a single-user free account is
|
||||
its own 1-person workspace; a team subscription is a multi-member
|
||||
workspace. The user_id contextvar narrows further to "which member of
|
||||
the workspace", letting some operations be member-scoped while others
|
||||
(skill install, billing, etc.) are workspace-scoped.
|
||||
|
||||
Dependency direction
|
||||
--------------------
|
||||
``persistence`` (lower layer) reads from this module; ``gateway.auth``
|
||||
(higher layer) writes to it. ``CurrentWorkspace`` is defined here as a
|
||||
:class:`typing.Protocol` so that ``persistence`` never needs to import
|
||||
the concrete ``Workspace`` row class from ``deerflow.persistence.workspace``.
|
||||
Any object with ``.id: str`` and ``.role: str`` attributes structurally
|
||||
satisfies the protocol.
|
||||
|
||||
Asyncio semantics
|
||||
-----------------
|
||||
Identical to ``user_context``: ``ContextVar`` is task-local under asyncio.
|
||||
``asyncio.create_task`` inherits the parent task's workspace context;
|
||||
threading.Timer does **not** (callers spawning timers must capture
|
||||
``get_effective_workspace_id()`` at enqueue time, the same way
|
||||
:mod:`deerflow.agents.memory.queue` captures ``user_id``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextvars import ContextVar, Token
|
||||
from typing import Final, Protocol, runtime_checkable
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class CurrentWorkspace(Protocol):
|
||||
"""Structural type for the current active workspace.
|
||||
|
||||
Any object with ``.id: str`` and ``.role: str`` attributes satisfies
|
||||
this protocol. Concrete implementations live in
|
||||
``app.gateway.auth.models`` (PR4 will add them).
|
||||
|
||||
``role`` is the *caller's* role within this workspace
|
||||
(``'owner'`` / ``'admin'`` / ``'member'``), not the workspace's
|
||||
own metadata. Stage 0 sees ``'owner'`` only — Stage 2 RBAC rollout
|
||||
opens up the other values.
|
||||
"""
|
||||
|
||||
id: str
|
||||
role: str
|
||||
|
||||
|
||||
_current_workspace: Final[ContextVar[CurrentWorkspace | None]] = ContextVar("deerflow_current_workspace", default=None)
|
||||
|
||||
|
||||
def set_current_workspace(workspace: CurrentWorkspace) -> Token[CurrentWorkspace | None]:
|
||||
"""Set the current workspace for this async task.
|
||||
|
||||
Returns a reset token that should be passed to
|
||||
:func:`reset_current_workspace` in a ``finally`` block to restore
|
||||
the previous context.
|
||||
"""
|
||||
return _current_workspace.set(workspace)
|
||||
|
||||
|
||||
def reset_current_workspace(token: Token[CurrentWorkspace | None]) -> None:
|
||||
"""Restore the context to the state captured by ``token``."""
|
||||
_current_workspace.reset(token)
|
||||
|
||||
|
||||
def get_current_workspace() -> CurrentWorkspace | None:
|
||||
"""Return the current workspace, or ``None`` if unset.
|
||||
|
||||
Safe to call in any context. Used by code paths that can proceed
|
||||
without a workspace (migration scripts, public endpoints).
|
||||
"""
|
||||
return _current_workspace.get()
|
||||
|
||||
|
||||
def require_current_workspace() -> CurrentWorkspace:
|
||||
"""Return the current workspace, or raise :class:`RuntimeError`.
|
||||
|
||||
Used by repository code that must not be called outside a
|
||||
request-authenticated context. The error message is phrased so
|
||||
that a caller debugging a stack trace can locate the offending
|
||||
code path.
|
||||
"""
|
||||
workspace = _current_workspace.get()
|
||||
if workspace is None:
|
||||
raise RuntimeError("repository accessed without workspace context")
|
||||
return workspace
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Effective workspace_id helpers (filesystem isolation)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DEFAULT_WORKSPACE_ID: Final[str] = "default"
|
||||
|
||||
|
||||
def get_effective_workspace_id() -> str:
|
||||
"""Return the current workspace id as a string, or DEFAULT_WORKSPACE_ID if unset.
|
||||
|
||||
Unlike :func:`require_current_workspace` this never raises — it is
|
||||
designed for filesystem-path resolution where a valid workspace
|
||||
bucket is always needed (PR6 will switch
|
||||
``Paths.thread_dir(workspace_id=...)`` to read from here).
|
||||
"""
|
||||
workspace = _current_workspace.get()
|
||||
if workspace is None:
|
||||
return DEFAULT_WORKSPACE_ID
|
||||
return str(workspace.id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sentinel-based workspace_id resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Repository methods accept a ``workspace_id`` keyword-only argument that
|
||||
# defaults to ``AUTO``. The three possible values drive distinct
|
||||
# behaviours; see the docstring on :func:`resolve_workspace_id`.
|
||||
|
||||
|
||||
class _AutoSentinel:
|
||||
"""Singleton marker meaning 'resolve workspace_id from contextvar'."""
|
||||
|
||||
_instance: _AutoSentinel | None = None
|
||||
|
||||
def __new__(cls) -> _AutoSentinel:
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
return cls._instance
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "<AUTO>"
|
||||
|
||||
|
||||
AUTO: Final[_AutoSentinel] = _AutoSentinel()
|
||||
|
||||
|
||||
def resolve_workspace_id(
|
||||
value: str | None | _AutoSentinel,
|
||||
*,
|
||||
method_name: str = "repository method",
|
||||
) -> str | None:
|
||||
"""Resolve the workspace_id parameter passed to a repository method.
|
||||
|
||||
Three-state semantics:
|
||||
|
||||
- :data:`AUTO` (default): read from contextvar; raise
|
||||
:class:`RuntimeError` if no workspace is in context. This is the
|
||||
common case for request-scoped calls.
|
||||
- Explicit ``str``: use the provided id verbatim, overriding any
|
||||
contextvar value. Useful for tests and admin-override flows.
|
||||
- Explicit ``None``: no filter — the repository should skip the
|
||||
workspace_id WHERE clause entirely. Reserved for migration scripts
|
||||
and CLI tools that intentionally bypass workspace isolation.
|
||||
"""
|
||||
if isinstance(value, _AutoSentinel):
|
||||
workspace = _current_workspace.get()
|
||||
if workspace is None:
|
||||
raise RuntimeError(
|
||||
f"{method_name} called with workspace_id=AUTO but no workspace context is set; pass an explicit workspace_id, set the contextvar via auth middleware, or opt out with workspace_id=None for migration/CLI paths."
|
||||
)
|
||||
# Coerce to ``str`` at the boundary; persistence stores
|
||||
# ``workspace_id`` as ``String(36)`` (UUID v4 text).
|
||||
return str(workspace.id)
|
||||
return value
|
||||
@@ -46,6 +46,10 @@ postgres = [
|
||||
"psycopg[binary]>=3.3.3",
|
||||
"psycopg-pool>=3.3.0",
|
||||
]
|
||||
postgres-test = [
|
||||
"deerflow-harness[postgres]",
|
||||
"testcontainers[postgres]>=4.0",
|
||||
]
|
||||
pymupdf = ["pymupdf4llm>=0.0.17"]
|
||||
|
||||
[build-system]
|
||||
|
||||
@@ -25,6 +25,7 @@ dependencies = [
|
||||
|
||||
[project.optional-dependencies]
|
||||
postgres = ["deerflow-harness[postgres]"]
|
||||
postgres-test = ["deerflow-harness[postgres-test]"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
@@ -36,7 +37,9 @@ dev = [
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
markers = [
|
||||
"no_auto_user: disable the conftest autouse contextvar fixture for this test",
|
||||
"no_auto_user: disable the conftest autouse user contextvar fixture for this test",
|
||||
"no_auto_workspace: disable the conftest autouse workspace contextvar fixture for this test",
|
||||
"postgres: requires a Postgres testcontainer (Docker daemon); skipped if unavailable",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
"""Backfill ``workspace_id`` on PR5 business tables.
|
||||
|
||||
Three-step backfill (each idempotent — re-running picks up where a crash
|
||||
left off because every step's WHERE clause filters already-processed rows):
|
||||
|
||||
1. For each user without ``default_workspace_id``: create a personal
|
||||
workspace + ``owner`` membership + write back the user's
|
||||
``default_workspace_id``.
|
||||
2. ``UPDATE`` each of ``threads_meta`` / ``runs`` / ``feedback`` /
|
||||
``run_events`` setting ``workspace_id`` from the row's owner's
|
||||
``users.default_workspace_id``. Only touches rows where
|
||||
``workspace_id IS NULL`` and ``user_id IS NOT NULL``.
|
||||
3. Any rows still with ``workspace_id IS NULL`` (truly orphan — they had
|
||||
``user_id = NULL`` to begin with) are assigned the *legacy* workspace
|
||||
UUID ``00000000-0000-0000-0000-000000000000``. The script creates
|
||||
that workspace on demand, owned by the platform admin.
|
||||
|
||||
Usage::
|
||||
|
||||
PYTHONPATH=. python scripts/backfill_workspace_id.py [--dry-run]
|
||||
|
||||
T5.4 only ships the skeleton — the three step bodies are filled in by
|
||||
T5.5 / T5.6 / T5.7 along with their per-step tests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from app.gateway.auth.workspace_slug import auto_slug_from_email, next_available_slug
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.workspace import WorkspaceRepository
|
||||
from deerflow.persistence.workspace.sql import SLUG_BLACKLIST
|
||||
from deerflow.persistence.workspace_membership import WorkspaceMembershipRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The legacy workspace anchor. Stage 0 LOCK'd UUID — chosen as the standard
|
||||
# nil UUID so SQL log scans can spot it instantly.
|
||||
LEGACY_WORKSPACE_ID = "00000000-0000-0000-0000-000000000000"
|
||||
LEGACY_WORKSPACE_SLUG = "legacy"
|
||||
LEGACY_WORKSPACE_NAME = "Legacy Workspace"
|
||||
|
||||
# The four business tables that gained ``workspace_id`` in alembic 0002.
|
||||
_BUSINESS_TABLES: tuple[str, ...] = ("threads_meta", "runs", "feedback", "run_events")
|
||||
|
||||
# Map table-name to ORM class so we can build a portable correlated UPDATE
|
||||
# using SQLAlchemy expression language (SQLite < 3.33 lacks UPDATE-FROM
|
||||
# but supports correlated subqueries on every version we ship).
|
||||
_TABLE_MODELS: dict[str, type[Base]] = {
|
||||
"threads_meta": ThreadMetaRow,
|
||||
"runs": RunRow,
|
||||
"feedback": FeedbackRow,
|
||||
"run_events": RunEventRow,
|
||||
}
|
||||
|
||||
|
||||
async def _step1_create_workspaces_for_users(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
*,
|
||||
dry_run: bool,
|
||||
) -> int:
|
||||
"""Create one workspace + owner membership for each user missing default_workspace_id.
|
||||
|
||||
Returns the count of users a workspace was created for. Idempotent —
|
||||
users that already have ``default_workspace_id`` are skipped so a
|
||||
crashed run can resume safely.
|
||||
"""
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(select(UserRow.id, UserRow.email).where(UserRow.default_workspace_id.is_(None)))
|
||||
candidates = [(row.id, row.email) for row in result]
|
||||
|
||||
if not candidates:
|
||||
return 0
|
||||
|
||||
ws_repo = WorkspaceRepository(session_factory)
|
||||
m_repo = WorkspaceMembershipRepository(session_factory)
|
||||
created = 0
|
||||
for user_id, email in candidates:
|
||||
base_slug = auto_slug_from_email(email)
|
||||
|
||||
async def slug_exists(s: str) -> bool:
|
||||
if s in SLUG_BLACKLIST:
|
||||
return True
|
||||
return (await ws_repo.get_by_slug(s)) is not None
|
||||
|
||||
unique_slug = await next_available_slug(base_slug, exists_check=slug_exists)
|
||||
|
||||
if dry_run:
|
||||
logger.info("WOULD create workspace for user=%s email=%s slug=%s", user_id, email, unique_slug)
|
||||
created += 1
|
||||
continue
|
||||
|
||||
display_local = email.split("@", 1)[0]
|
||||
workspace = await ws_repo.create(
|
||||
name=f"{display_local}'s Workspace"[:64],
|
||||
slug=unique_slug,
|
||||
owner_id=user_id,
|
||||
)
|
||||
await m_repo.add(workspace_id=workspace["id"], user_id=user_id, role="owner")
|
||||
async with session_factory() as session:
|
||||
await session.execute(update(UserRow).where(UserRow.id == user_id).values(default_workspace_id=workspace["id"]))
|
||||
await session.commit()
|
||||
created += 1
|
||||
logger.info("Created workspace %s (slug=%s) for user=%s", workspace["id"], unique_slug, user_id)
|
||||
|
||||
return created
|
||||
|
||||
|
||||
async def _step2_update_table_from_users(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
table: str,
|
||||
*,
|
||||
dry_run: bool,
|
||||
) -> int:
|
||||
"""UPDATE *table* setting workspace_id from owner's users.default_workspace_id.
|
||||
|
||||
Uses a correlated subquery (portable across SQLite + Postgres). Filters
|
||||
``workspace_id IS NULL AND user_id IS NOT NULL`` so already-set rows
|
||||
and truly orphan rows are skipped (Step 3 handles the latter).
|
||||
|
||||
Returns the number of rows updated (or that *would* be updated under
|
||||
``dry_run``).
|
||||
"""
|
||||
model = _TABLE_MODELS[table]
|
||||
workspace_col = model.workspace_id
|
||||
user_col = model.user_id
|
||||
|
||||
# Subquery: pull the user's default_workspace_id for each row.
|
||||
correlated_default = select(UserRow.default_workspace_id).where(UserRow.id == user_col).scalar_subquery()
|
||||
|
||||
if dry_run:
|
||||
# Count rows whose owner has a default_workspace_id assigned — only
|
||||
# those would get touched by the actual UPDATE.
|
||||
count_stmt = select(func.count()).select_from(model).join(UserRow, UserRow.id == user_col).where(workspace_col.is_(None), user_col.is_not(None), UserRow.default_workspace_id.is_not(None))
|
||||
async with session_factory() as session:
|
||||
count = (await session.execute(count_stmt)).scalar_one() or 0
|
||||
logger.info("WOULD update %d rows in %s from users.default_workspace_id", count, table)
|
||||
return int(count)
|
||||
|
||||
stmt = update(model).where(workspace_col.is_(None), user_col.is_not(None)).values(workspace_id=correlated_default)
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(stmt)
|
||||
await session.commit()
|
||||
rowcount = result.rowcount or 0
|
||||
logger.info("Updated %d rows in %s from users.default_workspace_id", rowcount, table)
|
||||
return int(rowcount)
|
||||
|
||||
|
||||
async def _ensure_legacy_workspace(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
*,
|
||||
dry_run: bool,
|
||||
) -> bool:
|
||||
"""Create the ``legacy_workspace`` anchor row idempotently.
|
||||
|
||||
The anchor is needed before Step 3 can point orphan rows at it. We
|
||||
pick the platform admin (``system_role='admin'``) as owner; if no
|
||||
admin exists yet we fall back to the oldest user. If the database
|
||||
has no users at all we refuse to continue — running this script on
|
||||
an unbootstrapped DB would create a workspace with no owner and the
|
||||
FK to ``users`` would fail anyway.
|
||||
|
||||
Returns True if the workspace was just created (or would be, under
|
||||
``dry_run``). False if it already existed.
|
||||
"""
|
||||
from deerflow.persistence.workspace.model import WorkspaceRow
|
||||
|
||||
async with session_factory() as session:
|
||||
existing = await session.get(WorkspaceRow, LEGACY_WORKSPACE_ID)
|
||||
if existing is not None:
|
||||
return False
|
||||
|
||||
async with session_factory() as session:
|
||||
admin_id = (await session.execute(select(UserRow.id).where(UserRow.system_role == "admin").order_by(UserRow.created_at).limit(1))).scalar_one_or_none()
|
||||
if admin_id is None:
|
||||
admin_id = (await session.execute(select(UserRow.id).order_by(UserRow.created_at).limit(1))).scalar_one_or_none()
|
||||
|
||||
if admin_id is None:
|
||||
raise RuntimeError(
|
||||
"Cannot create legacy_workspace: no users exist. Bootstrap an admin via /auth/initialize before running backfill.",
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
logger.info("WOULD create legacy_workspace (id=%s) owned by user=%s", LEGACY_WORKSPACE_ID, admin_id)
|
||||
return True
|
||||
|
||||
ws_repo = WorkspaceRepository(session_factory)
|
||||
await ws_repo.create(
|
||||
workspace_id=LEGACY_WORKSPACE_ID,
|
||||
name=LEGACY_WORKSPACE_NAME,
|
||||
slug=LEGACY_WORKSPACE_SLUG,
|
||||
owner_id=admin_id,
|
||||
)
|
||||
m_repo = WorkspaceMembershipRepository(session_factory)
|
||||
await m_repo.add(workspace_id=LEGACY_WORKSPACE_ID, user_id=admin_id, role="owner")
|
||||
logger.info("Created legacy_workspace (id=%s) owned by user=%s", LEGACY_WORKSPACE_ID, admin_id)
|
||||
return True
|
||||
|
||||
|
||||
async def _step3_assign_legacy_workspace(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
table: str,
|
||||
*,
|
||||
dry_run: bool,
|
||||
) -> int:
|
||||
"""Assign LEGACY_WORKSPACE_ID to *table* rows still missing workspace_id.
|
||||
|
||||
Callers should ensure :func:`_ensure_legacy_workspace` has run first;
|
||||
the orchestrator does this between Step 2 and Step 3. Orphan rows are
|
||||
rows whose ``user_id`` was already NULL (or pointed at a deleted user)
|
||||
so Step 2's correlated subquery left them untouched.
|
||||
"""
|
||||
model = _TABLE_MODELS[table]
|
||||
workspace_col = model.workspace_id
|
||||
|
||||
if dry_run:
|
||||
count_stmt = select(func.count()).select_from(model).where(workspace_col.is_(None))
|
||||
async with session_factory() as session:
|
||||
count = (await session.execute(count_stmt)).scalar_one() or 0
|
||||
logger.info("WOULD assign %d orphan row(s) in %s to legacy_workspace", count, table)
|
||||
return int(count)
|
||||
|
||||
stmt = update(model).where(workspace_col.is_(None)).values(workspace_id=LEGACY_WORKSPACE_ID)
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(stmt)
|
||||
await session.commit()
|
||||
rowcount = result.rowcount or 0
|
||||
logger.info("Assigned %d orphan row(s) in %s to legacy_workspace", rowcount, table)
|
||||
return int(rowcount)
|
||||
|
||||
|
||||
async def backfill(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Run all three backfill steps; return a per-step row-count report.
|
||||
|
||||
Order matters: Step 1 must populate ``users.default_workspace_id``
|
||||
before Step 2 can correlate business rows back through ``users``.
|
||||
"""
|
||||
report: dict[str, Any] = {"dry_run": dry_run}
|
||||
|
||||
report["users_workspaces_created"] = await _step1_create_workspaces_for_users(session_factory, dry_run=dry_run)
|
||||
for table in _BUSINESS_TABLES:
|
||||
report[f"{table}_from_users"] = await _step2_update_table_from_users(session_factory, table, dry_run=dry_run)
|
||||
report["legacy_workspace_created"] = await _ensure_legacy_workspace(session_factory, dry_run=dry_run)
|
||||
for table in _BUSINESS_TABLES:
|
||||
report[f"{table}_legacy"] = await _step3_assign_legacy_workspace(session_factory, table, dry_run=dry_run)
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def _build_session_factory_from_config() -> async_sessionmaker[AsyncSession]:
|
||||
"""Build an async session factory from the active config.yaml.
|
||||
|
||||
Avoids importing on module load so unit tests can stub
|
||||
``session_factory`` directly without booting the full config pipeline.
|
||||
"""
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.persistence.engine import get_session_factory, init_engine_from_config
|
||||
|
||||
asyncio.run(init_engine_from_config(get_app_config().database))
|
||||
sf = get_session_factory()
|
||||
if sf is None:
|
||||
raise RuntimeError(
|
||||
"database.backend=memory: nothing to backfill. Switch config.yaml to sqlite/postgres first.",
|
||||
)
|
||||
return sf
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Backfill workspace_id on Stage 0 business tables (idempotent).")
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Print the rows each step would touch without writing.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
|
||||
sf = _build_session_factory_from_config()
|
||||
report = asyncio.run(backfill(sf, dry_run=args.dry_run))
|
||||
|
||||
logger.info("Backfill report (dry_run=%s):", args.dry_run)
|
||||
for key, value in report.items():
|
||||
if key == "dry_run":
|
||||
continue
|
||||
logger.info(" %s: %s", key, value)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,212 @@
|
||||
"""One-time migration: lift per-user thread / memory / agent dirs into per-workspace layout.
|
||||
|
||||
PR4 introduced ``{base_dir}/users/{user_id}/...`` for per-user isolation. PR6
|
||||
adds the multi-tenant top dimension: ``{base_dir}/workspaces/{wid}/...``,
|
||||
with per-user state nested under each workspace
|
||||
(``{base_dir}/workspaces/{wid}/users/{uid}/memory.json`` etc.). This script
|
||||
walks the old PR4 layout, looks up each user's ``default_workspace_id``
|
||||
from the ``users`` table, and rewrites the path.
|
||||
|
||||
Usage:
|
||||
PYTHONPATH=. python scripts/migrate_paths_to_workspace.py [--dry-run] [--default-workspace WID]
|
||||
|
||||
The script is idempotent — re-running it after a successful migration is a no-op.
|
||||
A ``--dry-run`` invocation must not write anything; it only logs what would happen.
|
||||
|
||||
Mapping rules:
|
||||
|
||||
- ``{base_dir}/users/{uid}/threads/{tid}/`` -> ``{base_dir}/workspaces/{wid}/threads/{tid}/``
|
||||
- ``{base_dir}/users/{uid}/memory.json`` -> ``{base_dir}/workspaces/{wid}/users/{uid}/memory.json``
|
||||
- ``{base_dir}/users/{uid}/agents/{name}/`` -> ``{base_dir}/workspaces/{wid}/users/{uid}/agents/{name}/``
|
||||
|
||||
``{wid}`` is read from ``users.default_workspace_id``. Users without one
|
||||
fall through to ``--default-workspace`` (defaults to the special bucket
|
||||
``"legacy_workspace"`` so they can be triaged manually). Pre-existing
|
||||
destinations are preserved; legacy copies are moved to
|
||||
``{base_dir}/migration-conflicts/<...>`` for human review.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import shutil
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.config.paths import Paths, get_paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LEGACY_WORKSPACE_FALLBACK = "legacy_workspace"
|
||||
|
||||
|
||||
def _load_user_workspaces(paths: Paths) -> dict[str, str | None]:
|
||||
"""Read ``user_id -> default_workspace_id`` from the local sqlite DB.
|
||||
|
||||
Returns an empty dict when the database does not exist (fresh install
|
||||
/ Postgres-only deployments). The Postgres path is out of scope for
|
||||
this script — the operator should run it with ``--default-workspace``
|
||||
set to the target workspace and skip the DB lookup.
|
||||
"""
|
||||
db_path = paths.base_dir / "deer-flow.db"
|
||||
if not db_path.exists():
|
||||
logger.info("No sqlite database at %s — every user will use the fallback workspace.", db_path)
|
||||
return {}
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
try:
|
||||
cursor = conn.execute("SELECT id, default_workspace_id FROM users")
|
||||
except sqlite3.OperationalError as e:
|
||||
logger.warning("Failed to query users.default_workspace_id: %s", e)
|
||||
return {}
|
||||
return {row[0]: row[1] for row in cursor.fetchall()}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _resolve_workspace(user_id: str, user_workspaces: dict[str, str | None], fallback: str) -> str:
|
||||
wid = user_workspaces.get(user_id)
|
||||
return wid or fallback
|
||||
|
||||
|
||||
def _move(src: Path, dest: Path, conflict_root: Path, *, dry_run: bool, label: str) -> str:
|
||||
"""Move ``src`` to ``dest``; on conflict, divert legacy under ``conflict_root``.
|
||||
|
||||
Returns a short string describing what happened (for the report).
|
||||
"""
|
||||
if dest.exists():
|
||||
conflict_dest = conflict_root / label / src.name
|
||||
if not dry_run:
|
||||
conflict_dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(src), str(conflict_dest))
|
||||
logger.warning("Conflict for %s/%s: legacy copy diverted to %s", label, src.name, conflict_dest)
|
||||
return f"conflict -> {conflict_dest}"
|
||||
|
||||
if not dry_run:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(src), str(dest))
|
||||
return f"moved -> {dest}"
|
||||
|
||||
|
||||
def migrate_user_tree(
|
||||
paths: Paths,
|
||||
user_id: str,
|
||||
workspace_id: str,
|
||||
*,
|
||||
dry_run: bool,
|
||||
) -> list[dict]:
|
||||
"""Lift one user's tree under a workspace. Returns per-asset report rows."""
|
||||
report: list[dict] = []
|
||||
legacy_user_dir = paths.base_dir / "users" / user_id
|
||||
if not legacy_user_dir.exists():
|
||||
return report
|
||||
|
||||
conflict_root = paths.base_dir / "migration-conflicts" / "workspace-migration"
|
||||
|
||||
# 1. threads/{tid}/ -> workspaces/{wid}/threads/{tid}/
|
||||
legacy_threads = legacy_user_dir / "threads"
|
||||
if legacy_threads.exists():
|
||||
for thread_dir in sorted(legacy_threads.iterdir()):
|
||||
if not thread_dir.is_dir():
|
||||
continue
|
||||
dest = paths.thread_dir(thread_dir.name, workspace_id=workspace_id)
|
||||
action = _move(thread_dir, dest, conflict_root, dry_run=dry_run, label=f"threads/{workspace_id}")
|
||||
report.append({"asset": "thread", "user_id": user_id, "workspace_id": workspace_id, "name": thread_dir.name, "action": action})
|
||||
if not dry_run and legacy_threads.exists() and not any(legacy_threads.iterdir()):
|
||||
legacy_threads.rmdir()
|
||||
|
||||
# 2. memory.json -> workspaces/{wid}/users/{uid}/memory.json
|
||||
legacy_mem = legacy_user_dir / "memory.json"
|
||||
if legacy_mem.exists():
|
||||
dest = paths.user_memory_file(user_id, workspace_id=workspace_id)
|
||||
action = _move(legacy_mem, dest, conflict_root, dry_run=dry_run, label=f"users/{user_id}/memory")
|
||||
report.append({"asset": "memory", "user_id": user_id, "workspace_id": workspace_id, "name": "memory.json", "action": action})
|
||||
|
||||
# 3. agents/{name}/ -> workspaces/{wid}/users/{uid}/agents/{name}/
|
||||
legacy_agents = legacy_user_dir / "agents"
|
||||
if legacy_agents.exists():
|
||||
for agent_dir in sorted(legacy_agents.iterdir()):
|
||||
if not agent_dir.is_dir():
|
||||
continue
|
||||
dest = paths.user_agent_dir(user_id, agent_dir.name, workspace_id=workspace_id)
|
||||
action = _move(agent_dir, dest, conflict_root, dry_run=dry_run, label=f"users/{user_id}/agents")
|
||||
report.append({"asset": "agent", "user_id": user_id, "workspace_id": workspace_id, "name": agent_dir.name, "action": action})
|
||||
if not dry_run and legacy_agents.exists() and not any(legacy_agents.iterdir()):
|
||||
legacy_agents.rmdir()
|
||||
|
||||
# 4. If the user dir is now empty, remove it so lifespan warnings clear.
|
||||
if not dry_run and legacy_user_dir.exists() and not any(legacy_user_dir.iterdir()):
|
||||
legacy_user_dir.rmdir()
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def migrate(
|
||||
paths: Paths,
|
||||
*,
|
||||
user_workspaces: dict[str, str | None],
|
||||
fallback_workspace: str,
|
||||
dry_run: bool,
|
||||
) -> list[dict]:
|
||||
"""Top-level entry point: iterate every user under ``{base_dir}/users``."""
|
||||
legacy_users = paths.base_dir / "users"
|
||||
if not legacy_users.exists():
|
||||
logger.info("No legacy ``users/`` directory under %s — nothing to migrate.", paths.base_dir)
|
||||
return []
|
||||
|
||||
report: list[dict] = []
|
||||
for user_dir in sorted(legacy_users.iterdir()):
|
||||
if not user_dir.is_dir():
|
||||
continue
|
||||
user_id = user_dir.name
|
||||
workspace_id = _resolve_workspace(user_id, user_workspaces, fallback_workspace)
|
||||
logger.info("Migrating user %s -> workspace %s", user_id, workspace_id)
|
||||
report.extend(migrate_user_tree(paths, user_id, workspace_id, dry_run=dry_run))
|
||||
|
||||
if not dry_run and legacy_users.exists() and not any(legacy_users.iterdir()):
|
||||
legacy_users.rmdir()
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Lift legacy per-user paths into per-workspace layout (PR6).")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Log actions without making changes.")
|
||||
parser.add_argument(
|
||||
"--default-workspace",
|
||||
default=LEGACY_WORKSPACE_FALLBACK,
|
||||
metavar="WID",
|
||||
help=(f"Workspace id to use for users without a ``default_workspace_id`` in the DB. Defaults to ``{LEGACY_WORKSPACE_FALLBACK}`` (matches the orphan-row bucket used by PR5 backfill)."),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
|
||||
paths = get_paths()
|
||||
logger.info("Base directory: %s", paths.base_dir)
|
||||
logger.info("Dry run: %s", args.dry_run)
|
||||
logger.info("Fallback workspace: %s", args.default_workspace)
|
||||
|
||||
user_workspaces = _load_user_workspaces(paths)
|
||||
logger.info("Loaded %d user->workspace mappings from DB", len(user_workspaces))
|
||||
|
||||
report = migrate(
|
||||
paths,
|
||||
user_workspaces=user_workspaces,
|
||||
fallback_workspace=args.default_workspace,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
|
||||
if not report:
|
||||
logger.info("Nothing to migrate.")
|
||||
return
|
||||
|
||||
logger.info("Migration report (%d entries):", len(report))
|
||||
for entry in report:
|
||||
logger.info(" asset=%s user=%s workspace=%s name=%s action=%s", entry["asset"], entry["user_id"], entry["workspace_id"], entry["name"], entry["action"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -36,8 +36,16 @@ from fastapi import FastAPI, Request, Response
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
from app.gateway.auth.models import User
|
||||
from app.gateway.auth.models import ActiveWorkspace, User
|
||||
from app.gateway.authz import AuthContext, Permissions
|
||||
from deerflow.runtime.user_context import (
|
||||
reset_current_user,
|
||||
set_current_user,
|
||||
)
|
||||
from deerflow.runtime.workspace_context import (
|
||||
reset_current_workspace,
|
||||
set_current_workspace,
|
||||
)
|
||||
|
||||
# Default permission set granted to the stub user. Mirrors `_ALL_PERMISSIONS`
|
||||
# in authz.py — kept inline so the tests don't import a private symbol.
|
||||
@@ -67,22 +75,55 @@ class _StubAuthMiddleware(BaseHTTPMiddleware):
|
||||
Mirrors what production ``AuthMiddleware`` does after the JWT decode
|
||||
+ DB lookup short-circuit, so ``@require_permission`` finds an
|
||||
authenticated context and skips its own re-authentication path.
|
||||
|
||||
Optionally stamps the workspace contextvar too — needed by the PR6
|
||||
decorator path that calls ``get_effective_workspace_id()`` before
|
||||
delegating to ``check_access``.
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp, user_factory: Callable[[], User]) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
app: ASGIApp,
|
||||
user_factory: Callable[[], User],
|
||||
workspace_factory: Callable[[], ActiveWorkspace | None] | None = None,
|
||||
override_user_contextvar: bool = False,
|
||||
) -> None:
|
||||
super().__init__(app)
|
||||
self._user_factory = user_factory
|
||||
self._workspace_factory = workspace_factory
|
||||
# Tests that only need ``request.state.auth`` (the @require_permission
|
||||
# path) keep the autouse user contextvar — flipping it to a per-call
|
||||
# UUID would break legacy tests whose routes resolve paths via
|
||||
# ``get_effective_user_id()``. Cross-user / cross-workspace tests opt
|
||||
# in by setting this flag so the contextvar matches the request user.
|
||||
self._override_user_contextvar = override_user_contextvar
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
user = self._user_factory()
|
||||
request.state.user = user
|
||||
request.state.auth = AuthContext(user=user, permissions=list(_STUB_PERMISSIONS))
|
||||
return await call_next(request)
|
||||
|
||||
user_token = set_current_user(user) if self._override_user_contextvar else None
|
||||
ws_token = None
|
||||
if self._workspace_factory is not None:
|
||||
workspace = self._workspace_factory()
|
||||
if workspace is not None:
|
||||
request.state.workspace = workspace
|
||||
ws_token = set_current_workspace(workspace)
|
||||
try:
|
||||
return await call_next(request)
|
||||
finally:
|
||||
if ws_token is not None:
|
||||
reset_current_workspace(ws_token)
|
||||
if user_token is not None:
|
||||
reset_current_user(user_token)
|
||||
|
||||
|
||||
def make_authed_test_app(
|
||||
*,
|
||||
user_factory: Callable[[], User] | None = None,
|
||||
workspace_factory: Callable[[], ActiveWorkspace | None] | None = None,
|
||||
override_user_contextvar: bool = False,
|
||||
owner_check_passes: bool = True,
|
||||
) -> FastAPI:
|
||||
"""Build a FastAPI test app with stub auth + permissive thread_store.
|
||||
@@ -103,7 +144,12 @@ def make_authed_test_app(
|
||||
"""
|
||||
factory = user_factory or _make_stub_user
|
||||
app = FastAPI()
|
||||
app.add_middleware(_StubAuthMiddleware, user_factory=factory)
|
||||
app.add_middleware(
|
||||
_StubAuthMiddleware,
|
||||
user_factory=factory,
|
||||
workspace_factory=workspace_factory,
|
||||
override_user_contextvar=override_user_contextvar,
|
||||
)
|
||||
|
||||
repo = MagicMock()
|
||||
repo.check_access = AsyncMock(return_value=owner_check_passes)
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# PR7 — boundary scan allowlist for direct LangGraph checkpoint/saver imports.
|
||||
#
|
||||
# Paths are relative to backend/. A path here means: "this file is permitted to
|
||||
# import langgraph.checkpoint.* at runtime". Adding to this list requires a
|
||||
# matching justification in the PR that introduces the new importer.
|
||||
#
|
||||
# Consumed by tests/test_workspace_boundary.py.
|
||||
#
|
||||
# Imports inside `if TYPE_CHECKING:` blocks are exempt automatically — they do
|
||||
# not pull the symbol into runtime — so type-only references (e.g. annotations
|
||||
# on `BaseCheckpointSaver` parameters) do NOT need to be listed here.
|
||||
|
||||
langgraph_checkpoint_importers = [
|
||||
# Gateway thread plumbing: constructs an empty checkpoint when initialising
|
||||
# a new thread's state via the LangGraph runtime.
|
||||
"app/gateway/routers/threads.py",
|
||||
|
||||
# Harness checkpointer factories: the single authorised place to construct
|
||||
# InMemorySaver / SqliteSaver / PostgresSaver implementations. Everywhere
|
||||
# else must obtain a checkpointer via `app.gateway.deps.get_checkpointer`
|
||||
# or `deerflow.runtime.checkpointer` helpers.
|
||||
"packages/harness/deerflow/runtime/checkpointer/async_provider.py",
|
||||
"packages/harness/deerflow/runtime/checkpointer/provider.py",
|
||||
|
||||
# Background run worker: uses `empty_checkpoint` to seed state for runs
|
||||
# resumed from a missing/expired checkpoint id.
|
||||
"packages/harness/deerflow/runtime/runs/worker.py",
|
||||
]
|
||||
@@ -15,6 +15,13 @@ import pytest
|
||||
# Make 'app' and 'deerflow' importable from any working directory
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "scripts"))
|
||||
# Make 'fixtures.*' importable as plugin modules from this conftest.
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
# Register fixture plugin modules so tests can request fixtures by name
|
||||
# without ad-hoc imports. ``fixtures.postgres`` provides
|
||||
# ``postgres_container`` (session-scoped) and ``postgres_url`` (per-test).
|
||||
pytest_plugins = ["fixtures.postgres"]
|
||||
|
||||
# Break the circular import chain that exists in production code:
|
||||
# deerflow.subagents.__init__
|
||||
@@ -38,6 +45,95 @@ _executor_mock.get_background_task_result = MagicMock()
|
||||
sys.modules["deerflow.subagents.executor"] = _executor_mock
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auto-seed test workspace + user when Base.metadata.create_all() runs
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# PR6 makes every business-row INSERT carry ``workspace_id`` (resolved
|
||||
# from the autouse workspace contextvar = "test-workspace-autouse"). The
|
||||
# Stage 0 schema has a NOT NULL FK from those rows to ``workspaces`` and
|
||||
# from ``workspaces.owner_id`` to ``users``. Without the seed below,
|
||||
# every legacy repo test would fail with a FOREIGN KEY error the moment
|
||||
# it tries to insert a thread.
|
||||
#
|
||||
# We register an ``after_create`` hook on ``Base.metadata`` so that
|
||||
# whenever ``init_engine`` finishes ``create_all()`` (the auto-create
|
||||
# path used by tests and dev), the two anchor rows are present. Alembic
|
||||
# migration tests don't trigger create_all so they are unaffected and
|
||||
# keep exercising real FK constraints in isolation.
|
||||
|
||||
|
||||
def _register_test_seed_listener() -> None:
|
||||
"""Attach an after_create hook that seeds the autouse user + workspace."""
|
||||
try:
|
||||
from sqlalchemy import event, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.workspace.model import WorkspaceRow
|
||||
except ImportError:
|
||||
return
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
def _seed(_target, connection, **kw): # noqa: ARG001
|
||||
tables = {t.name for t in kw.get("tables", []) or []}
|
||||
if "users" not in tables or "workspaces" not in tables:
|
||||
return
|
||||
|
||||
dialect = connection.dialect.name
|
||||
now = datetime.now(UTC)
|
||||
|
||||
# Seed both rows with a consistent ``default_workspace_id`` so the
|
||||
# PR5 backfill script (which scans ``users.default_workspace_id IS
|
||||
# NULL``) does not pick up the test fixtures as candidates.
|
||||
# Insert user first with NULL default_workspace_id (chicken-and-egg
|
||||
# with workspaces.owner_id FK), then workspace, then UPDATE the user
|
||||
# row to point at the workspace so the PR5 backfill script does not
|
||||
# pick up the autouse user as a candidate.
|
||||
user_values = {
|
||||
"id": "test-user-autouse",
|
||||
"email": "test-user-autouse@local",
|
||||
"password_hash": None,
|
||||
"system_role": "user",
|
||||
"created_at": now,
|
||||
"oauth_provider": None,
|
||||
"oauth_id": None,
|
||||
"needs_setup": False,
|
||||
"token_version": 0,
|
||||
"default_workspace_id": None,
|
||||
}
|
||||
workspace_values = {
|
||||
"id": "test-workspace-autouse",
|
||||
"name": "Autouse Test Workspace",
|
||||
"slug": "autouse-test",
|
||||
"status": "active",
|
||||
"owner_id": "test-user-autouse",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
|
||||
if dialect == "sqlite":
|
||||
user_stmt = sqlite_insert(UserRow.__table__).values(**user_values).on_conflict_do_nothing(index_elements=["id"])
|
||||
ws_stmt = sqlite_insert(WorkspaceRow.__table__).values(**workspace_values).on_conflict_do_nothing(index_elements=["id"])
|
||||
elif dialect == "postgresql":
|
||||
user_stmt = pg_insert(UserRow.__table__).values(**user_values).on_conflict_do_nothing(index_elements=["id"])
|
||||
ws_stmt = pg_insert(WorkspaceRow.__table__).values(**workspace_values).on_conflict_do_nothing(index_elements=["id"])
|
||||
else:
|
||||
return
|
||||
|
||||
connection.execute(user_stmt)
|
||||
connection.execute(ws_stmt)
|
||||
connection.execute(update(UserRow.__table__).where(UserRow.__table__.c.id == "test-user-autouse").where(UserRow.__table__.c.default_workspace_id.is_(None)).values(default_workspace_id="test-workspace-autouse"))
|
||||
|
||||
event.listen(Base.metadata, "after_create", _seed)
|
||||
|
||||
|
||||
_register_test_seed_listener()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def provisioner_module():
|
||||
"""Load docker/provisioner/app.py as an importable test module.
|
||||
@@ -110,3 +206,34 @@ def _auto_user_context(request):
|
||||
yield
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _auto_workspace_context(request):
|
||||
"""Inject a default ``test-workspace-autouse`` into the workspace contextvar.
|
||||
|
||||
Mirror of :func:`_auto_user_context`. PR6 adds ``workspace_id=AUTO``
|
||||
sentinels to every repository method; without an autouse workspace
|
||||
fixture every legacy persistence test would raise RuntimeError.
|
||||
|
||||
Opt-out via ``@pytest.mark.no_auto_workspace``.
|
||||
"""
|
||||
if request.node.get_closest_marker("no_auto_workspace"):
|
||||
yield
|
||||
return
|
||||
|
||||
try:
|
||||
from deerflow.runtime.workspace_context import (
|
||||
reset_current_workspace,
|
||||
set_current_workspace,
|
||||
)
|
||||
except ImportError:
|
||||
yield
|
||||
return
|
||||
|
||||
workspace = SimpleNamespace(id="test-workspace-autouse", role="owner")
|
||||
token = set_current_workspace(workspace)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
Vendored
+82
@@ -0,0 +1,82 @@
|
||||
"""Postgres testcontainer fixtures for Stage 0 PR1.
|
||||
|
||||
Provides per-test ephemeral database isolation atop a single session-scoped
|
||||
container. Tests marked ``@pytest.mark.postgres`` request the ``postgres_url``
|
||||
fixture, which yields an asyncpg connection URL pointing at a freshly-created
|
||||
database. The database is force-dropped after the test (any leaked connections
|
||||
get pg_terminate_backend'd first).
|
||||
|
||||
Why per-database rather than per-schema:
|
||||
asyncpg (the SQLAlchemy async driver we use) doesn't honor URL-embedded
|
||||
search_path the way psycopg does. Per-database isolation is one extra
|
||||
CREATE/DROP per test (~50ms), but lets test app code use its full schema
|
||||
unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def postgres_container():
|
||||
"""Session-scoped Postgres 16 container shared across all postgres tests.
|
||||
|
||||
Started once per pytest session. Subsequent tests piggy-back on the same
|
||||
container; each gets its own database via the ``postgres_url`` fixture.
|
||||
|
||||
Skipped (and the test marked skip) if Docker is unavailable on the host —
|
||||
testcontainers raises ``DockerException`` when it can't reach the daemon.
|
||||
"""
|
||||
try:
|
||||
from testcontainers.postgres import PostgresContainer
|
||||
except ImportError as exc: # pragma: no cover - install boundary
|
||||
pytest.skip(f"testcontainers[postgres] not installed: {exc}")
|
||||
|
||||
try:
|
||||
# Image tag aligned with the production Aliyun RDS (PostgreSQL 17.9
|
||||
# confirmed by `make doctor` 2026-05-11). Bump together with RDS upgrades.
|
||||
with PostgresContainer("postgres:17-alpine") as pg:
|
||||
yield pg
|
||||
except Exception as exc: # pragma: no cover - environment-dependent
|
||||
# DockerException, ConnectionError, etc. — surface a skip rather than
|
||||
# an error so devs without Docker can still run the rest of the suite.
|
||||
pytest.skip(f"could not start Postgres container ({exc})")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def postgres_url(postgres_container) -> Iterator[str]:
|
||||
"""Per-test ephemeral database URL (asyncpg dialect).
|
||||
|
||||
Each invocation creates a unique database on the shared container and
|
||||
yields its URL. Teardown force-drops the database, terminating any
|
||||
backend connections the test forgot to close.
|
||||
"""
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
|
||||
db_name = f"test_{secrets.token_hex(8)}"
|
||||
raw = postgres_container.get_connection_url() # postgresql+psycopg2://...
|
||||
# Strip the SQLAlchemy dialect prefix so plain psycopg can connect.
|
||||
base = raw.replace("postgresql+psycopg2://", "postgresql://")
|
||||
parent_url = base.rsplit("/", 1)[0] + "/postgres"
|
||||
|
||||
# CREATE DATABASE must run outside a transaction; psycopg autocommit=True.
|
||||
with psycopg.connect(parent_url, autocommit=True) as conn:
|
||||
conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(db_name)))
|
||||
|
||||
asyncpg_url = raw.replace("postgresql+psycopg2://", "postgresql+asyncpg://").rsplit("/", 1)[0] + f"/{db_name}"
|
||||
|
||||
try:
|
||||
yield asyncpg_url
|
||||
finally:
|
||||
with psycopg.connect(parent_url, autocommit=True) as conn:
|
||||
# Kick any leaked connections so DROP DATABASE doesn't block.
|
||||
conn.execute(
|
||||
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = %s AND pid <> pg_backend_pid()",
|
||||
(db_name,),
|
||||
)
|
||||
conn.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(db_name)))
|
||||
@@ -0,0 +1,357 @@
|
||||
"""Alembic 0002 / 0003 round-trips on both Postgres and SQLite.
|
||||
|
||||
PR5 has two migrations:
|
||||
|
||||
* ``0002_business_tables_workspace`` — adds *nullable* ``workspace_id``
|
||||
+ FK to ``workspaces`` on ``threads_meta`` / ``runs`` / ``feedback`` /
|
||||
``run_events``, plus a composite index on threads_meta for the common
|
||||
"list a workspace's threads for a user, newest first" query.
|
||||
* ``0003_business_tables_workspace_not_null`` — flips the column to
|
||||
``NOT NULL`` (refusing to upgrade if NULL rows remain) and adds the
|
||||
UNIQUE (workspace_id, thread_id) index on ``threads_meta``.
|
||||
|
||||
Pattern mirrors ``test_alembic_default_workspace_id.py``: synchronous
|
||||
test bodies (alembic's command layer is sync, ``env.py`` calls
|
||||
``asyncio.run`` internally — running under pytest-anyio would explode
|
||||
that nested event loop). The pre-migration schema is bootstrapped with
|
||||
just the columns those migrations touch, so we don't depend on the
|
||||
production ORM staying frozen.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
_ALEMBIC_INI = Path(__file__).resolve().parents[1] / "packages" / "harness" / "deerflow" / "persistence" / "migrations" / "alembic.ini"
|
||||
|
||||
# Tables the PR5 migrations touch.
|
||||
_BUSINESS_TABLES = ("threads_meta", "runs", "feedback", "run_events")
|
||||
|
||||
|
||||
def _make_alembic_config(url: str) -> Config:
|
||||
cfg = Config(str(_ALEMBIC_INI))
|
||||
cfg.set_main_option("sqlalchemy.url", url)
|
||||
return cfg
|
||||
|
||||
|
||||
def _bootstrap_pre_pr5_schema(sync_url: str) -> None:
|
||||
"""Create the post-PR4 schema PR5 migrations expect.
|
||||
|
||||
Includes ``workspaces`` (FK target), ``users`` (already has
|
||||
``default_workspace_id`` from 0001 — but we skip 0001 here and bootstrap
|
||||
the columns directly so the migration's behaviour can be tested in
|
||||
isolation), and the four business tables.
|
||||
"""
|
||||
engine = create_engine(sync_url)
|
||||
with engine.begin() as conn:
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE workspaces (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
name VARCHAR(64) NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE users (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
email VARCHAR(255) NOT NULL,
|
||||
default_workspace_id VARCHAR(36)
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE threads_meta (
|
||||
thread_id VARCHAR(64) PRIMARY KEY,
|
||||
user_id VARCHAR(64),
|
||||
updated_at TIMESTAMP
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE runs (
|
||||
run_id VARCHAR(64) PRIMARY KEY,
|
||||
thread_id VARCHAR(64) NOT NULL,
|
||||
user_id VARCHAR(64)
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE feedback (
|
||||
feedback_id VARCHAR(64) PRIMARY KEY,
|
||||
thread_id VARCHAR(64) NOT NULL,
|
||||
run_id VARCHAR(64) NOT NULL,
|
||||
user_id VARCHAR(64),
|
||||
rating INTEGER NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE run_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
thread_id VARCHAR(64) NOT NULL,
|
||||
run_id VARCHAR(64) NOT NULL,
|
||||
user_id VARCHAR(64),
|
||||
event_type VARCHAR(32) NOT NULL,
|
||||
category VARCHAR(16) NOT NULL,
|
||||
content TEXT,
|
||||
seq INTEGER NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
# Pretend 0001 has already run so 0002 is the next revision applied.
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL PRIMARY KEY)
|
||||
"""
|
||||
)
|
||||
)
|
||||
conn.execute(text("INSERT INTO alembic_version (version_num) VALUES ('0001_users_default_workspace')"))
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _assert_workspace_id_present(sync_url: str, *, nullable: bool) -> None:
|
||||
engine = create_engine(sync_url)
|
||||
insp = inspect(engine)
|
||||
for table in _BUSINESS_TABLES:
|
||||
cols = {c["name"]: c for c in insp.get_columns(table)}
|
||||
assert "workspace_id" in cols, f"{table}: workspace_id missing; got {list(cols)}"
|
||||
col = cols["workspace_id"]
|
||||
assert col["nullable"] is nullable, f"{table}.workspace_id nullable expected {nullable}, got {col['nullable']}"
|
||||
fks = insp.get_foreign_keys(table)
|
||||
ws_fks = [fk for fk in fks if fk.get("referred_table") == "workspaces" and fk.get("constrained_columns") == ["workspace_id"]]
|
||||
assert ws_fks, f"{table}: FK to workspaces missing; got {fks}"
|
||||
idxs = {i["name"] for i in insp.get_indexes("threads_meta")}
|
||||
assert "idx_threads_meta_workspace_user_updated" in idxs, f"threads_meta composite index missing; got {idxs}"
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _assert_workspace_id_absent(sync_url: str) -> None:
|
||||
engine = create_engine(sync_url)
|
||||
insp = inspect(engine)
|
||||
for table in _BUSINESS_TABLES:
|
||||
cols = {c["name"] for c in insp.get_columns(table)}
|
||||
assert "workspace_id" not in cols, f"{table}: workspace_id still present; got {cols}"
|
||||
idxs = {i["name"] for i in insp.get_indexes("threads_meta")}
|
||||
assert "idx_threads_meta_workspace_user_updated" not in idxs, f"threads_meta composite index still present; got {idxs}"
|
||||
engine.dispose()
|
||||
|
||||
|
||||
# ---------- SQLite tests ----------------------------------------------------
|
||||
|
||||
|
||||
def test_sqlite_upgrade_0002_adds_workspace_id_column() -> None:
|
||||
"""0002 adds nullable workspace_id + FK + composite index on SQLite."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "test.db"
|
||||
sync_url = f"sqlite:///{db_path}"
|
||||
async_url = f"sqlite+aiosqlite:///{db_path}"
|
||||
|
||||
_bootstrap_pre_pr5_schema(sync_url)
|
||||
cfg = _make_alembic_config(async_url)
|
||||
command.upgrade(cfg, "0002_business_tables_workspace")
|
||||
_assert_workspace_id_present(sync_url, nullable=True)
|
||||
|
||||
|
||||
def test_sqlite_downgrade_0002_removes_workspace_id_column() -> None:
|
||||
"""0002 downgrade drops the column + FK + composite index cleanly."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "test.db"
|
||||
sync_url = f"sqlite:///{db_path}"
|
||||
async_url = f"sqlite+aiosqlite:///{db_path}"
|
||||
|
||||
_bootstrap_pre_pr5_schema(sync_url)
|
||||
cfg = _make_alembic_config(async_url)
|
||||
command.upgrade(cfg, "0002_business_tables_workspace")
|
||||
_assert_workspace_id_present(sync_url, nullable=True)
|
||||
command.downgrade(cfg, "-1")
|
||||
_assert_workspace_id_absent(sync_url)
|
||||
|
||||
|
||||
# ---------- Postgres tests --------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.postgres
|
||||
def test_postgres_upgrade_0002_adds_workspace_id_column(postgres_url: str) -> None:
|
||||
"""Postgres: 0002 adds nullable workspace_id + FK + composite index."""
|
||||
sync_url = postgres_url.replace("+asyncpg", "+psycopg")
|
||||
_bootstrap_pre_pr5_schema(sync_url)
|
||||
|
||||
cfg = _make_alembic_config(postgres_url)
|
||||
command.upgrade(cfg, "0002_business_tables_workspace")
|
||||
_assert_workspace_id_present(sync_url, nullable=True)
|
||||
|
||||
|
||||
@pytest.mark.postgres
|
||||
def test_postgres_downgrade_0002_removes_workspace_id_column(postgres_url: str) -> None:
|
||||
"""Postgres: 0002 downgrade drops column + FK + index cleanly."""
|
||||
sync_url = postgres_url.replace("+asyncpg", "+psycopg")
|
||||
_bootstrap_pre_pr5_schema(sync_url)
|
||||
|
||||
cfg = _make_alembic_config(postgres_url)
|
||||
command.upgrade(cfg, "0002_business_tables_workspace")
|
||||
_assert_workspace_id_present(sync_url, nullable=True)
|
||||
command.downgrade(cfg, "-1")
|
||||
_assert_workspace_id_absent(sync_url)
|
||||
|
||||
|
||||
# ---------- 0003 helpers ----------------------------------------------------
|
||||
|
||||
|
||||
def _populate_business_rows_for_0003(sync_url: str, workspace_id: str = "w1", thread_id: str = "t1", run_id: str = "r1") -> None:
|
||||
"""Insert one row per business table with workspace_id populated.
|
||||
|
||||
Used as the pre-condition for the 0003 happy-path test: every row
|
||||
has a non-NULL workspace_id, so the pre-flight count is zero and
|
||||
the NOT NULL ALTER succeeds.
|
||||
"""
|
||||
engine = create_engine(sync_url)
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("INSERT INTO workspaces (id, name) VALUES (:id, :name)"), {"id": workspace_id, "name": "W"})
|
||||
conn.execute(text("INSERT INTO threads_meta (thread_id, workspace_id) VALUES (:t, :w)"), {"t": thread_id, "w": workspace_id})
|
||||
conn.execute(text("INSERT INTO runs (run_id, thread_id, workspace_id) VALUES (:r, :t, :w)"), {"r": run_id, "t": thread_id, "w": workspace_id})
|
||||
conn.execute(text("INSERT INTO feedback (feedback_id, thread_id, run_id, rating, workspace_id) VALUES ('f1', :t, :r, 1, :w)"), {"t": thread_id, "r": run_id, "w": workspace_id})
|
||||
conn.execute(text("INSERT INTO run_events (thread_id, run_id, event_type, category, seq, workspace_id) VALUES (:t, :r, 'x', 'lifecycle', 1, :w)"), {"t": thread_id, "r": run_id, "w": workspace_id})
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _assert_workspace_id_not_null_and_unique_index(sync_url: str) -> None:
|
||||
engine = create_engine(sync_url)
|
||||
insp = inspect(engine)
|
||||
for table in _BUSINESS_TABLES:
|
||||
cols = {c["name"]: c for c in insp.get_columns(table)}
|
||||
assert cols["workspace_id"]["nullable"] is False, f"{table}.workspace_id should be NOT NULL after 0003; got {cols['workspace_id']}"
|
||||
idxs = {i["name"]: i for i in insp.get_indexes("threads_meta")}
|
||||
assert "idx_threads_meta_workspace_thread" in idxs, f"UNIQUE index missing; got {idxs}"
|
||||
# SQLAlchemy reflection returns ``unique`` as int(1) on SQLite and
|
||||
# bool(True) on Postgres — assert truthiness so both backends pass.
|
||||
assert idxs["idx_threads_meta_workspace_thread"]["unique"], f"index should be UNIQUE; got {idxs['idx_threads_meta_workspace_thread']}"
|
||||
engine.dispose()
|
||||
|
||||
|
||||
# ---------- 0003 SQLite tests -----------------------------------------------
|
||||
|
||||
|
||||
def test_sqlite_upgrade_0003_succeeds_with_populated_workspace_id() -> None:
|
||||
"""0003 NOT NULL ALTER + UNIQUE index lands when no NULL rows remain."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "test.db"
|
||||
sync_url = f"sqlite:///{db_path}"
|
||||
async_url = f"sqlite+aiosqlite:///{db_path}"
|
||||
|
||||
_bootstrap_pre_pr5_schema(sync_url)
|
||||
cfg = _make_alembic_config(async_url)
|
||||
command.upgrade(cfg, "0002_business_tables_workspace")
|
||||
_populate_business_rows_for_0003(sync_url)
|
||||
|
||||
command.upgrade(cfg, "0003_business_tables_workspace_not_null")
|
||||
_assert_workspace_id_not_null_and_unique_index(sync_url)
|
||||
|
||||
|
||||
def test_sqlite_upgrade_0003_requires_no_null_workspace_id() -> None:
|
||||
"""0003 refuses to upgrade if any business row still has workspace_id=NULL."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "test.db"
|
||||
sync_url = f"sqlite:///{db_path}"
|
||||
async_url = f"sqlite+aiosqlite:///{db_path}"
|
||||
|
||||
_bootstrap_pre_pr5_schema(sync_url)
|
||||
cfg = _make_alembic_config(async_url)
|
||||
command.upgrade(cfg, "0002_business_tables_workspace")
|
||||
|
||||
# Leave one threads_meta row with workspace_id NULL.
|
||||
engine = create_engine(sync_url)
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("INSERT INTO threads_meta (thread_id) VALUES ('t-orphan')"))
|
||||
engine.dispose()
|
||||
|
||||
with pytest.raises(RuntimeError, match="Cannot ALTER"):
|
||||
command.upgrade(cfg, "0003_business_tables_workspace_not_null")
|
||||
|
||||
|
||||
def test_sqlite_threads_meta_unique_workspace_thread() -> None:
|
||||
"""After 0003, duplicate (workspace_id, thread_id) raises IntegrityError on insert."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "test.db"
|
||||
sync_url = f"sqlite:///{db_path}"
|
||||
async_url = f"sqlite+aiosqlite:///{db_path}"
|
||||
|
||||
_bootstrap_pre_pr5_schema(sync_url)
|
||||
cfg = _make_alembic_config(async_url)
|
||||
command.upgrade(cfg, "0002_business_tables_workspace")
|
||||
_populate_business_rows_for_0003(sync_url)
|
||||
command.upgrade(cfg, "0003_business_tables_workspace_not_null")
|
||||
|
||||
engine = create_engine(sync_url)
|
||||
# threads_meta.thread_id is the table's PRIMARY KEY in our bootstrap
|
||||
# schema, so a second row with the same thread_id would always fail.
|
||||
# Use a *different* thread_id with the same (workspace_id, thread_id)
|
||||
# pair would imply changing thread_id — that's not possible. Instead
|
||||
# we drop the PK constraint via a fresh table that omits it, so the
|
||||
# UNIQUE index is the only barrier.
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("CREATE TABLE threads_meta_test_unique (id INTEGER PRIMARY KEY, thread_id VARCHAR(64), workspace_id VARCHAR(36))"))
|
||||
conn.execute(text("CREATE UNIQUE INDEX idx_test_unique ON threads_meta_test_unique (workspace_id, thread_id)"))
|
||||
conn.execute(text("INSERT INTO threads_meta_test_unique (thread_id, workspace_id) VALUES ('t', 'w')"))
|
||||
|
||||
with pytest.raises(IntegrityError):
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("INSERT INTO threads_meta_test_unique (thread_id, workspace_id) VALUES ('t', 'w')"))
|
||||
engine.dispose()
|
||||
|
||||
|
||||
# ---------- 0003 Postgres tests ---------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.postgres
|
||||
def test_postgres_upgrade_0003_succeeds_with_populated_workspace_id(postgres_url: str) -> None:
|
||||
sync_url = postgres_url.replace("+asyncpg", "+psycopg")
|
||||
_bootstrap_pre_pr5_schema(sync_url)
|
||||
|
||||
cfg = _make_alembic_config(postgres_url)
|
||||
command.upgrade(cfg, "0002_business_tables_workspace")
|
||||
_populate_business_rows_for_0003(sync_url)
|
||||
command.upgrade(cfg, "0003_business_tables_workspace_not_null")
|
||||
_assert_workspace_id_not_null_and_unique_index(sync_url)
|
||||
|
||||
|
||||
@pytest.mark.postgres
|
||||
def test_postgres_upgrade_0003_requires_no_null_workspace_id(postgres_url: str) -> None:
|
||||
sync_url = postgres_url.replace("+asyncpg", "+psycopg")
|
||||
_bootstrap_pre_pr5_schema(sync_url)
|
||||
|
||||
cfg = _make_alembic_config(postgres_url)
|
||||
command.upgrade(cfg, "0002_business_tables_workspace")
|
||||
|
||||
engine = create_engine(sync_url)
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("INSERT INTO threads_meta (thread_id) VALUES ('t-orphan')"))
|
||||
engine.dispose()
|
||||
|
||||
with pytest.raises(RuntimeError, match="Cannot ALTER"):
|
||||
command.upgrade(cfg, "0003_business_tables_workspace_not_null")
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Alembic revision 0001 round-trips on both Postgres and SQLite.
|
||||
|
||||
Verifies the first DeerFlow migration adds `users.default_workspace_id`
|
||||
(with FK to `workspaces`) on upgrade and removes it on downgrade. Both
|
||||
backends are exercised because the migration relies on
|
||||
`op.batch_alter_table` for SQLite ALTER compatibility — we want to know
|
||||
if either dialect regresses.
|
||||
|
||||
These tests are synchronous: alembic's command layer is sync, and our
|
||||
`env.py` calls `asyncio.run(...)` internally. Running under
|
||||
pytest-anyio would put us inside an event loop and crash that
|
||||
`asyncio.run` call, so we keep the test bodies plain `def`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
|
||||
_ALEMBIC_INI = Path(__file__).resolve().parents[1] / "packages" / "harness" / "deerflow" / "persistence" / "migrations" / "alembic.ini"
|
||||
|
||||
|
||||
def _make_alembic_config(url: str) -> Config:
|
||||
cfg = Config(str(_ALEMBIC_INI))
|
||||
cfg.set_main_option("sqlalchemy.url", url)
|
||||
return cfg
|
||||
|
||||
|
||||
def _bootstrap_pre_pr4_schema(sync_url: str) -> None:
|
||||
"""Create the minimal pre-PR4 schema the migration needs to ALTER.
|
||||
|
||||
Only `users` (without `default_workspace_id`) and `workspaces` (just `id`)
|
||||
are required so the FK target resolves. The rest of the production
|
||||
schema is irrelevant to this migration.
|
||||
"""
|
||||
engine = create_engine(sync_url)
|
||||
with engine.begin() as conn:
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE workspaces (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
name VARCHAR(64) NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE users (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
email VARCHAR(255) NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _assert_column_present(sync_url: str) -> None:
|
||||
engine = create_engine(sync_url)
|
||||
insp = inspect(engine)
|
||||
cols = {c["name"] for c in insp.get_columns("users")}
|
||||
assert "default_workspace_id" in cols, f"column missing; got {cols}"
|
||||
fks = insp.get_foreign_keys("users")
|
||||
fk_to_ws = [fk for fk in fks if fk.get("referred_table") == "workspaces"]
|
||||
assert fk_to_ws, f"FK to workspaces missing; got {fks}"
|
||||
assert fk_to_ws[0]["constrained_columns"] == ["default_workspace_id"]
|
||||
assert fk_to_ws[0]["referred_columns"] == ["id"]
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _assert_column_absent(sync_url: str) -> None:
|
||||
engine = create_engine(sync_url)
|
||||
insp = inspect(engine)
|
||||
cols = {c["name"] for c in insp.get_columns("users")}
|
||||
assert "default_workspace_id" not in cols, f"column still present; got {cols}"
|
||||
engine.dispose()
|
||||
|
||||
|
||||
# ---------- SQLite tests ----------------------------------------------------
|
||||
|
||||
|
||||
def test_sqlite_upgrade_adds_default_workspace_id_with_fk() -> None:
|
||||
"""SQLite: upgrade 0001 adds column + FK; batch_alter_table works."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "test.db"
|
||||
sync_url = f"sqlite:///{db_path}"
|
||||
async_url = f"sqlite+aiosqlite:///{db_path}"
|
||||
|
||||
_bootstrap_pre_pr4_schema(sync_url)
|
||||
|
||||
cfg = _make_alembic_config(async_url)
|
||||
command.upgrade(cfg, "0001_users_default_workspace")
|
||||
|
||||
_assert_column_present(sync_url)
|
||||
|
||||
|
||||
def test_sqlite_downgrade_removes_default_workspace_id() -> None:
|
||||
"""SQLite: downgrade 0001 removes the column it added."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "test.db"
|
||||
sync_url = f"sqlite:///{db_path}"
|
||||
async_url = f"sqlite+aiosqlite:///{db_path}"
|
||||
|
||||
_bootstrap_pre_pr4_schema(sync_url)
|
||||
|
||||
cfg = _make_alembic_config(async_url)
|
||||
command.upgrade(cfg, "0001_users_default_workspace")
|
||||
_assert_column_present(sync_url)
|
||||
|
||||
command.downgrade(cfg, "-1")
|
||||
_assert_column_absent(sync_url)
|
||||
|
||||
|
||||
# ---------- Postgres tests --------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.postgres
|
||||
def test_postgres_upgrade_adds_default_workspace_id_with_fk(postgres_url: str) -> None:
|
||||
"""Postgres: upgrade 0001 adds column + FK pointing at workspaces(id)."""
|
||||
sync_url = postgres_url.replace("+asyncpg", "+psycopg")
|
||||
_bootstrap_pre_pr4_schema(sync_url)
|
||||
|
||||
cfg = _make_alembic_config(postgres_url)
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
_assert_column_present(sync_url)
|
||||
|
||||
|
||||
@pytest.mark.postgres
|
||||
def test_postgres_downgrade_removes_default_workspace_id(postgres_url: str) -> None:
|
||||
"""Postgres: downgrade 0001 cleanly drops the FK and column."""
|
||||
sync_url = postgres_url.replace("+asyncpg", "+psycopg")
|
||||
_bootstrap_pre_pr4_schema(sync_url)
|
||||
|
||||
cfg = _make_alembic_config(postgres_url)
|
||||
command.upgrade(cfg, "head")
|
||||
_assert_column_present(sync_url)
|
||||
|
||||
command.downgrade(cfg, "-1")
|
||||
_assert_column_absent(sync_url)
|
||||
@@ -101,7 +101,7 @@ def test_create_and_decode_token():
|
||||
import os
|
||||
|
||||
os.environ["AUTH_JWT_SECRET"] = "test-secret-key-for-jwt-testing-minimum-32-chars"
|
||||
token = create_access_token(user_id)
|
||||
token = create_access_token(user_id, workspace_id="ws-test", role="owner")
|
||||
assert isinstance(token, str)
|
||||
|
||||
payload = decode_token(token)
|
||||
@@ -132,7 +132,7 @@ def test_decode_token_invalid():
|
||||
def test_create_token_custom_expiry():
|
||||
"""Custom expiry is respected."""
|
||||
user_id = str(uuid4())
|
||||
token = create_access_token(user_id, expires_delta=timedelta(hours=1))
|
||||
token = create_access_token(user_id, expires_delta=timedelta(hours=1), workspace_id="ws-test", role="owner")
|
||||
payload = decode_token(token)
|
||||
assert payload is not None
|
||||
assert payload.sub == user_id
|
||||
@@ -420,7 +420,7 @@ def test_jwt_encodes_ver():
|
||||
from app.gateway.auth.errors import TokenError
|
||||
|
||||
os.environ["AUTH_JWT_SECRET"] = "test-secret-key-for-jwt-testing-minimum-32-chars"
|
||||
token = create_access_token(str(uuid4()), token_version=3)
|
||||
token = create_access_token(str(uuid4()), token_version=3, workspace_id="ws-test", role="owner")
|
||||
payload = decode_token(token)
|
||||
assert not isinstance(payload, TokenError)
|
||||
assert payload.ver == 3
|
||||
@@ -433,7 +433,7 @@ def test_jwt_default_ver_zero():
|
||||
from app.gateway.auth.errors import TokenError
|
||||
|
||||
os.environ["AUTH_JWT_SECRET"] = "test-secret-key-for-jwt-testing-minimum-32-chars"
|
||||
token = create_access_token(str(uuid4()))
|
||||
token = create_access_token(str(uuid4()), workspace_id="ws-test", role="owner")
|
||||
payload = decode_token(token)
|
||||
assert not isinstance(payload, TokenError)
|
||||
assert payload.ver == 0
|
||||
@@ -447,7 +447,7 @@ def test_token_version_mismatch_rejects():
|
||||
os.environ["AUTH_JWT_SECRET"] = "test-secret-key-for-jwt-testing-minimum-32-chars"
|
||||
|
||||
user_id = str(uuid4())
|
||||
token = create_access_token(user_id, token_version=0)
|
||||
token = create_access_token(user_id, token_version=0, workspace_id="ws-test", role="owner")
|
||||
|
||||
mock_user = User(id=user_id, email="test@example.com", password_hash="hash", token_version=1)
|
||||
|
||||
|
||||
@@ -69,7 +69,7 @@ def test_decode_token_returns_token_error_on_malformed():
|
||||
|
||||
def test_decode_token_returns_payload_on_valid():
|
||||
_setup_config()
|
||||
token = create_access_token("user-123")
|
||||
token = create_access_token("user-123", workspace_id="ws-test", role="owner")
|
||||
result = decode_token(token)
|
||||
assert not isinstance(result, TokenError)
|
||||
assert result.sub == "user-123"
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""JWT carries wid + role claims (Stage 0 PR4).
|
||||
|
||||
Tokens issued after PR4 must include `wid` (workspace_id) and `role`
|
||||
(owner/admin/member) so the AuthMiddleware can resolve the active
|
||||
workspace without a DB hit. Legacy token compatibility lives in
|
||||
:mod:`test_legacy_token_compat`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
|
||||
from app.gateway.auth import create_access_token, decode_token
|
||||
from app.gateway.auth.config import get_auth_config
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _stable_jwt_secret(monkeypatch):
|
||||
"""Pin a deterministic JWT secret across tests in this module."""
|
||||
monkeypatch.setenv("AUTH_JWT_SECRET", "test-secret-key-for-jwt-testing-minimum-32-chars")
|
||||
yield
|
||||
|
||||
|
||||
def test_jwt_includes_wid_and_role() -> None:
|
||||
"""create_access_token records wid + role in the encoded payload."""
|
||||
user_id = str(uuid4())
|
||||
workspace_id = str(uuid4())
|
||||
|
||||
token = create_access_token(user_id, workspace_id=workspace_id, role="owner")
|
||||
|
||||
raw = jwt.decode(token, get_auth_config().jwt_secret, algorithms=["HS256"])
|
||||
assert raw["sub"] == user_id
|
||||
assert raw["wid"] == workspace_id
|
||||
assert raw["role"] == "owner"
|
||||
|
||||
|
||||
def test_decode_round_trip_keeps_wid_and_role() -> None:
|
||||
"""decode_token returns a TokenPayload exposing wid + role attributes."""
|
||||
user_id = str(uuid4())
|
||||
workspace_id = str(uuid4())
|
||||
|
||||
token = create_access_token(user_id, workspace_id=workspace_id, role="member")
|
||||
payload = decode_token(token)
|
||||
|
||||
# decode_token returns TokenError on failure — must be the success branch here.
|
||||
assert hasattr(payload, "wid"), f"got {payload!r}"
|
||||
assert payload.sub == user_id
|
||||
assert payload.wid == workspace_id
|
||||
assert payload.role == "member"
|
||||
@@ -0,0 +1,90 @@
|
||||
"""``GET /auth/me`` returns the user's workspace memberships.
|
||||
|
||||
Stage 0 PR4 T4.11. After PR4 the frontend needs to discover which
|
||||
workspaces the current user belongs to (eventually for a picker UI).
|
||||
Each membership entry surfaces ``id``, ``name``, ``slug``, ``role``
|
||||
so the picker can render the list without a second roundtrip.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
os.environ.setdefault("AUTH_JWT_SECRET", "test-secret-key-auth-me-workspaces-32+")
|
||||
|
||||
from app.gateway.auth.config import AuthConfig, set_auth_config
|
||||
|
||||
_TEST_SECRET = "test-secret-key-auth-me-workspaces-32+"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _setup_auth(tmp_path):
|
||||
from app.gateway import deps
|
||||
from app.gateway.routers.auth import _SETUP_STATUS_COOLDOWN
|
||||
from deerflow.persistence.engine import close_engine, init_engine
|
||||
|
||||
set_auth_config(AuthConfig(jwt_secret=_TEST_SECRET))
|
||||
url = f"sqlite+aiosqlite:///{tmp_path}/auth_me.db"
|
||||
asyncio.run(init_engine("sqlite", url=url, sqlite_dir=str(tmp_path)))
|
||||
deps._cached_local_provider = None
|
||||
deps._cached_repo = None
|
||||
_SETUP_STATUS_COOLDOWN.clear()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
deps._cached_local_provider = None
|
||||
deps._cached_repo = None
|
||||
_SETUP_STATUS_COOLDOWN.clear()
|
||||
asyncio.run(close_engine())
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client(_setup_auth):
|
||||
from app.gateway.app import create_app
|
||||
|
||||
set_auth_config(AuthConfig(jwt_secret=_TEST_SECRET))
|
||||
yield TestClient(create_app())
|
||||
|
||||
|
||||
def test_auth_me_returns_workspaces_list(client):
|
||||
"""After registration, /auth/me lists the user's single personal workspace."""
|
||||
client.post("/api/v1/auth/initialize", json={"email": "admin@example.com", "password": "Str0ng!Pass99"})
|
||||
reg = client.post("/api/v1/auth/register", json={"email": "alice@example.com", "password": "Tr0ub4dor3a-strong!"})
|
||||
user_id = reg.json()["id"]
|
||||
|
||||
resp = client.get("/api/v1/auth/me")
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
body = resp.json()
|
||||
assert body["id"] == user_id
|
||||
assert body["email"] == "alice@example.com"
|
||||
assert body["default_workspace_id"], "user should land with a default workspace"
|
||||
|
||||
assert len(body["workspaces"]) == 1, body
|
||||
ws = body["workspaces"][0]
|
||||
assert ws["id"] == body["default_workspace_id"]
|
||||
assert ws["slug"] == "alice"
|
||||
assert ws["role"] == "owner"
|
||||
assert ws["name"] # whatever the helper picks — just sanity check it's non-empty
|
||||
|
||||
|
||||
def test_auth_me_workspaces_isolated_per_user(client):
|
||||
"""Two users see only their own workspaces in /auth/me."""
|
||||
client.post("/api/v1/auth/initialize", json={"email": "admin@example.com", "password": "Str0ng!Pass99"})
|
||||
client.post("/api/v1/auth/register", json={"email": "alice@example.com", "password": "Tr0ub4dor3a-strong!"})
|
||||
a_id = client.get("/api/v1/auth/me").json()["id"]
|
||||
a_workspaces = client.get("/api/v1/auth/me").json()["workspaces"]
|
||||
|
||||
client.cookies.clear()
|
||||
client.post("/api/v1/auth/register", json={"email": "bob@example.com", "password": "Tr0ub4dor3a-strong!"})
|
||||
b_id = client.get("/api/v1/auth/me").json()["id"]
|
||||
b_workspaces = client.get("/api/v1/auth/me").json()["workspaces"]
|
||||
|
||||
assert a_id != b_id
|
||||
assert {w["id"] for w in a_workspaces}.isdisjoint({w["id"] for w in b_workspaces})
|
||||
assert {w["slug"] for w in a_workspaces} == {"alice"}
|
||||
assert {w["slug"] for w in b_workspaces} == {"bob"}
|
||||
@@ -0,0 +1,131 @@
|
||||
"""AuthMiddleware injects the workspace ContextVar (Stage 0 PR4 T4.7).
|
||||
|
||||
After PR4 every authenticated request has a workspace bound on
|
||||
``deerflow.runtime.workspace_context._current_workspace``. The middleware
|
||||
populates it from the JWT's ``wid`` / ``role`` claims, mirrors what it
|
||||
already does for ``user_context``, and tears both down in a single
|
||||
``try/finally`` so leaks don't cross requests.
|
||||
|
||||
Legacy 4-field tokens (no ``wid``) are rejected upstream by
|
||||
``decode_token`` (T4.6) — those should never reach the workspace
|
||||
injection branch; this file pins that the 401 they trigger carries
|
||||
``AuthErrorCode.WORKSPACE_REQUIRED``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from app.gateway.auth import create_access_token
|
||||
from app.gateway.auth.config import get_auth_config
|
||||
from app.gateway.auth.models import User
|
||||
from app.gateway.auth_middleware import AuthMiddleware
|
||||
from deerflow.runtime.workspace_context import get_current_workspace
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _stable_jwt_secret(monkeypatch):
|
||||
monkeypatch.setenv("AUTH_JWT_SECRET", "test-secret-key-for-jwt-testing-minimum-32-chars")
|
||||
yield
|
||||
|
||||
|
||||
def _make_app() -> FastAPI:
|
||||
"""App with AuthMiddleware + an inspect route that surfaces the contextvar."""
|
||||
app = FastAPI()
|
||||
app.add_middleware(AuthMiddleware)
|
||||
|
||||
@app.get("/api/v1/auth/setup-status") # public — never gates on wid
|
||||
async def setup_status():
|
||||
return {"needs_setup": False}
|
||||
|
||||
@app.get("/api/models") # protected — exercises wid injection
|
||||
async def inspect_workspace():
|
||||
ws = get_current_workspace()
|
||||
if ws is None:
|
||||
return {"workspace": None}
|
||||
return {"workspace": {"id": ws.id, "role": ws.role}}
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def _make_user(uid: str) -> User:
|
||||
return User(id=uid, email="t@example.com", password_hash="hash", token_version=0)
|
||||
|
||||
|
||||
def _make_legacy_token() -> str:
|
||||
"""Encode a pre-PR4 JWT (no wid/role) directly."""
|
||||
now = datetime.now(UTC)
|
||||
payload = {
|
||||
"sub": str(uuid4()),
|
||||
"exp": now + timedelta(hours=1),
|
||||
"iat": now,
|
||||
"ver": 0,
|
||||
}
|
||||
return jwt.encode(payload, get_auth_config().jwt_secret, algorithm="HS256")
|
||||
|
||||
|
||||
def test_new_jwt_injects_workspace_into_contextvar() -> None:
|
||||
"""Cookie with wid+role → route observes the workspace via the contextvar."""
|
||||
uid = str(uuid4())
|
||||
token = create_access_token(uid, workspace_id="ws-abc", role="owner")
|
||||
|
||||
with patch("app.gateway.deps.get_local_provider") as fn:
|
||||
fn.return_value.get_user = AsyncMock(return_value=_make_user(uid))
|
||||
client = TestClient(_make_app())
|
||||
res = client.get("/api/models", cookies={"access_token": token})
|
||||
|
||||
assert res.status_code == 200, res.text
|
||||
assert res.json() == {"workspace": {"id": "ws-abc", "role": "owner"}}
|
||||
|
||||
|
||||
def test_legacy_jwt_rejected_with_workspace_required() -> None:
|
||||
"""No-wid tokens get 401 with AuthErrorCode.WORKSPACE_REQUIRED, not generic token_invalid."""
|
||||
client = TestClient(_make_app())
|
||||
res = client.get("/api/models", cookies={"access_token": _make_legacy_token()})
|
||||
|
||||
assert res.status_code == 401
|
||||
assert res.json()["detail"]["code"] == "workspace_required"
|
||||
|
||||
|
||||
def test_public_path_skips_workspace_check() -> None:
|
||||
"""Public whitelist (e.g. /api/v1/auth/setup-status) does not require wid."""
|
||||
client = TestClient(_make_app())
|
||||
res = client.get("/api/v1/auth/setup-status") # no cookie at all
|
||||
assert res.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.no_auto_workspace
|
||||
def test_workspace_contextvar_resets_between_requests() -> None:
|
||||
"""After dispatch returns the contextvar must be clear (no leak across requests).
|
||||
|
||||
Why we test this: if the try/finally is wired only for user_context but
|
||||
not workspace_context, two back-to-back requests can see each other's
|
||||
workspace under asyncio task switching.
|
||||
"""
|
||||
uid = str(uuid4())
|
||||
token = create_access_token(uid, workspace_id="ws-first", role="owner")
|
||||
|
||||
# First request resolves to ws-first
|
||||
with patch("app.gateway.deps.get_local_provider") as fn:
|
||||
fn.return_value.get_user = AsyncMock(return_value=_make_user(uid))
|
||||
client = TestClient(_make_app())
|
||||
res1 = client.get("/api/models", cookies={"access_token": token})
|
||||
assert res1.json() == {"workspace": {"id": "ws-first", "role": "owner"}}
|
||||
|
||||
# Outside the request scope the contextvar must be empty again.
|
||||
assert get_current_workspace() is None
|
||||
|
||||
# Second request with a different workspace must not see ws-first.
|
||||
token2 = create_access_token(uid, workspace_id="ws-second", role="owner")
|
||||
with patch("app.gateway.deps.get_local_provider") as fn:
|
||||
fn.return_value.get_user = AsyncMock(return_value=_make_user(uid))
|
||||
client = TestClient(_make_app())
|
||||
res2 = client.get("/api/models", cookies={"access_token": token2})
|
||||
assert res2.json() == {"workspace": {"id": "ws-second", "role": "owner"}}
|
||||
@@ -0,0 +1,367 @@
|
||||
"""Tests for ``scripts/backfill_workspace_id.py`` (Stage 0 PR5).
|
||||
|
||||
Each step in the three-step backfill is exercised in isolation against
|
||||
a SQLite-on-disk database. The script wires in ``app.gateway.auth.workspace_slug``,
|
||||
so we get the same slug semantics that the registration flow uses.
|
||||
|
||||
Pattern mirrors :mod:`test_workspace_repo`: ``init_engine`` + per-test
|
||||
tmp_path, with explicit ``close_engine`` teardown so the singleton
|
||||
session factory does not leak across tests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.workspace.model import WorkspaceRow
|
||||
from deerflow.persistence.workspace_membership.model import WorkspaceMembershipRow
|
||||
from scripts.backfill_workspace_id import (
|
||||
LEGACY_WORKSPACE_ID,
|
||||
_ensure_legacy_workspace,
|
||||
_step1_create_workspaces_for_users,
|
||||
_step2_update_table_from_users,
|
||||
_step3_assign_legacy_workspace,
|
||||
backfill,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.anyio
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend() -> str:
|
||||
return "asyncio"
|
||||
|
||||
|
||||
_BUSINESS_ROWS = (ThreadMetaRow, RunRow, FeedbackRow, RunEventRow)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _relax_workspace_id_nullable():
|
||||
"""Simulate alembic 0002 (pre-backfill) state during these tests.
|
||||
|
||||
PR6 T5.11 flipped ``workspace_id`` to ``nullable=False`` on the four
|
||||
business ORM models — production correctness comes from alembic 0003.
|
||||
The backfill script's job is precisely to fill the rows that were
|
||||
inserted between 0002 (column added, nullable) and 0003 (NOT NULL),
|
||||
so tests for it must be able to insert NULL rows. We mutate
|
||||
``column.nullable`` for the four tables before ``create_all`` runs,
|
||||
then restore on teardown so other tests see the production shape.
|
||||
"""
|
||||
saved: list[tuple] = []
|
||||
for model in _BUSINESS_ROWS:
|
||||
col = model.__table__.c.workspace_id
|
||||
saved.append((col, col.nullable))
|
||||
col.nullable = True
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for col, original in saved:
|
||||
col.nullable = original
|
||||
|
||||
|
||||
async def _init_engine(tmp_path):
|
||||
from sqlalchemy import delete
|
||||
|
||||
from deerflow.persistence.engine import get_session_factory, init_engine
|
||||
|
||||
url = f"sqlite+aiosqlite:///{tmp_path / 'test.db'}"
|
||||
await init_engine("sqlite", url=url, sqlite_dir=str(tmp_path))
|
||||
sf = get_session_factory()
|
||||
# The PR6 conftest seeds an autouse user + workspace so business-row FKs
|
||||
# resolve in the wider test suite. Backfill tests model "fresh DB needs
|
||||
# backfill" semantics, so wipe those rows here. Order: clear the FK
|
||||
# pointer first, then the rows.
|
||||
async with sf() as session:
|
||||
await session.execute(delete(WorkspaceMembershipRow))
|
||||
await session.execute(delete(WorkspaceRow).where(WorkspaceRow.id == "test-workspace-autouse"))
|
||||
await session.execute(delete(UserRow).where(UserRow.id == "test-user-autouse"))
|
||||
await session.commit()
|
||||
return sf
|
||||
|
||||
|
||||
async def _close():
|
||||
from deerflow.persistence.engine import close_engine
|
||||
|
||||
await close_engine()
|
||||
|
||||
|
||||
async def _seed_user(sf, *, email: str, default_workspace_id: str | None = None, system_role: str = "user") -> str:
|
||||
user_id = str(uuid.uuid4())
|
||||
async with sf() as session:
|
||||
session.add(UserRow(id=user_id, email=email, default_workspace_id=default_workspace_id, system_role=system_role))
|
||||
await session.commit()
|
||||
return user_id
|
||||
|
||||
|
||||
async def _seed_business_row(sf, model, **fields) -> None:
|
||||
async with sf() as session:
|
||||
session.add(model(**fields))
|
||||
await session.commit()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 1: per-user workspace creation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_creates_workspace_per_user_without_default(tmp_path):
|
||||
"""Each user with NULL default_workspace_id gets a workspace + owner membership."""
|
||||
sf = await _init_engine(tmp_path)
|
||||
try:
|
||||
u_alice = await _seed_user(sf, email="alice@example.com")
|
||||
u_bob = await _seed_user(sf, email="bob+spam@example.com")
|
||||
|
||||
count = await _step1_create_workspaces_for_users(sf, dry_run=False)
|
||||
assert count == 2
|
||||
|
||||
async with sf() as session:
|
||||
workspaces = (await session.execute(select(WorkspaceRow))).scalars().all()
|
||||
memberships = (await session.execute(select(WorkspaceMembershipRow))).scalars().all()
|
||||
users = {u.id: u for u in (await session.execute(select(UserRow))).scalars().all()}
|
||||
|
||||
# 2 workspaces, each with exactly one owner membership matching its user.
|
||||
assert len(workspaces) == 2
|
||||
assert len(memberships) == 2
|
||||
owners_by_ws = {m.workspace_id: m.user_id for m in memberships if m.role == "owner"}
|
||||
assert {m.role for m in memberships} == {"owner"}
|
||||
for ws in workspaces:
|
||||
assert owners_by_ws[ws.id] == ws.owner_id
|
||||
assert users[ws.owner_id].default_workspace_id == ws.id
|
||||
|
||||
# Slug semantics: alice@ → "alice", bob+spam@ → "bob-spam".
|
||||
slugs = {ws.slug for ws in workspaces}
|
||||
assert slugs == {"alice", "bob-spam"}
|
||||
_ = u_alice, u_bob # captured for readability
|
||||
finally:
|
||||
await _close()
|
||||
|
||||
|
||||
async def test_step1_is_idempotent(tmp_path):
|
||||
"""Second run is a no-op when every user already has a default_workspace_id."""
|
||||
sf = await _init_engine(tmp_path)
|
||||
try:
|
||||
await _seed_user(sf, email="carol@example.com")
|
||||
first = await _step1_create_workspaces_for_users(sf, dry_run=False)
|
||||
assert first == 1
|
||||
# Re-running picks up the just-populated default_workspace_id, so the
|
||||
# candidate set is empty.
|
||||
second = await _step1_create_workspaces_for_users(sf, dry_run=False)
|
||||
assert second == 0
|
||||
async with sf() as session:
|
||||
ws_count = len((await session.execute(select(WorkspaceRow))).scalars().all())
|
||||
mem_count = len((await session.execute(select(WorkspaceMembershipRow))).scalars().all())
|
||||
assert ws_count == 1
|
||||
assert mem_count == 1
|
||||
finally:
|
||||
await _close()
|
||||
|
||||
|
||||
async def test_backfill_updates_4_tables_from_users(tmp_path):
|
||||
"""Step 2 propagates each user's default_workspace_id into 4 business tables."""
|
||||
sf = await _init_engine(tmp_path)
|
||||
try:
|
||||
user_id = await _seed_user(sf, email="dave@example.com")
|
||||
# Pre-seed business rows owned by the user with NULL workspace_id.
|
||||
await _seed_business_row(sf, ThreadMetaRow, thread_id="t-1", user_id=user_id)
|
||||
await _seed_business_row(sf, RunRow, run_id="r-1", thread_id="t-1", user_id=user_id)
|
||||
await _seed_business_row(sf, FeedbackRow, feedback_id="f-1", thread_id="t-1", run_id="r-1", user_id=user_id, rating=1)
|
||||
await _seed_business_row(sf, RunEventRow, thread_id="t-1", run_id="r-1", user_id=user_id, event_type="lifecycle_started", category="lifecycle", seq=1)
|
||||
|
||||
# Step 1 first so users.default_workspace_id is populated.
|
||||
await _step1_create_workspaces_for_users(sf, dry_run=False)
|
||||
|
||||
async with sf() as session:
|
||||
ws_id = (await session.execute(select(WorkspaceRow.id))).scalar_one()
|
||||
|
||||
# Step 2 updates each table.
|
||||
for table in ("threads_meta", "runs", "feedback", "run_events"):
|
||||
count = await _step2_update_table_from_users(sf, table, dry_run=False)
|
||||
assert count == 1, table
|
||||
|
||||
async with sf() as session:
|
||||
tm = (await session.execute(select(ThreadMetaRow))).scalar_one()
|
||||
run = (await session.execute(select(RunRow))).scalar_one()
|
||||
fb = (await session.execute(select(FeedbackRow))).scalar_one()
|
||||
ev = (await session.execute(select(RunEventRow))).scalar_one()
|
||||
assert tm.workspace_id == ws_id
|
||||
assert run.workspace_id == ws_id
|
||||
assert fb.workspace_id == ws_id
|
||||
assert ev.workspace_id == ws_id
|
||||
|
||||
# Re-running Step 2 is a no-op (filtered by workspace_id IS NULL).
|
||||
for table in ("threads_meta", "runs", "feedback", "run_events"):
|
||||
assert await _step2_update_table_from_users(sf, table, dry_run=False) == 0
|
||||
finally:
|
||||
await _close()
|
||||
|
||||
|
||||
async def test_backfill_step2_isolates_per_user(tmp_path):
|
||||
"""Two users with different default workspaces get their own threads tagged independently."""
|
||||
sf = await _init_engine(tmp_path)
|
||||
try:
|
||||
u_eve = await _seed_user(sf, email="eve@example.com")
|
||||
u_frank = await _seed_user(sf, email="frank@example.com")
|
||||
await _seed_business_row(sf, ThreadMetaRow, thread_id="t-eve", user_id=u_eve)
|
||||
await _seed_business_row(sf, ThreadMetaRow, thread_id="t-frank", user_id=u_frank)
|
||||
|
||||
await _step1_create_workspaces_for_users(sf, dry_run=False)
|
||||
await _step2_update_table_from_users(sf, "threads_meta", dry_run=False)
|
||||
|
||||
async with sf() as session:
|
||||
rows = {r.thread_id: r.workspace_id for r in (await session.execute(select(ThreadMetaRow))).scalars().all()}
|
||||
users = {u.id: u.default_workspace_id for u in (await session.execute(select(UserRow))).scalars().all()}
|
||||
assert rows["t-eve"] == users[u_eve]
|
||||
assert rows["t-frank"] == users[u_frank]
|
||||
assert rows["t-eve"] != rows["t-frank"]
|
||||
finally:
|
||||
await _close()
|
||||
|
||||
|
||||
async def test_step1_skips_blacklisted_base_slug(tmp_path):
|
||||
"""A user with email like admin@... gets bumped past the slug blacklist via the walker."""
|
||||
sf = await _init_engine(tmp_path)
|
||||
try:
|
||||
await _seed_user(sf, email="admin@example.com")
|
||||
await _step1_create_workspaces_for_users(sf, dry_run=False)
|
||||
async with sf() as session:
|
||||
ws = (await session.execute(select(WorkspaceRow))).scalar_one()
|
||||
# The walker treats "admin" as taken (blacklisted), so it falls
|
||||
# through to "admin-2" — the same behaviour the registration flow
|
||||
# uses for reserved slugs.
|
||||
assert ws.slug == "admin-2"
|
||||
finally:
|
||||
await _close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 3: orphan rows -> legacy_workspace
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_backfill_orphan_rows_go_to_legacy_workspace(tmp_path):
|
||||
"""Rows with user_id=NULL get assigned the legacy_workspace UUID after Step 3."""
|
||||
sf = await _init_engine(tmp_path)
|
||||
try:
|
||||
# Seed a platform admin so the legacy workspace has an owner.
|
||||
await _seed_user(sf, email="admin@example.com", system_role="admin")
|
||||
# Orphan business rows (user_id=NULL): legacy data from before auth.
|
||||
await _seed_business_row(sf, ThreadMetaRow, thread_id="t-orphan", user_id=None)
|
||||
await _seed_business_row(sf, RunRow, run_id="r-orphan", thread_id="t-orphan", user_id=None)
|
||||
await _seed_business_row(sf, FeedbackRow, feedback_id="f-orphan", thread_id="t-orphan", run_id="r-orphan", user_id=None, rating=1)
|
||||
await _seed_business_row(sf, RunEventRow, thread_id="t-orphan", run_id="r-orphan", user_id=None, event_type="legacy", category="lifecycle", seq=1)
|
||||
|
||||
# Ensure the anchor + reassign per table.
|
||||
created = await _ensure_legacy_workspace(sf, dry_run=False)
|
||||
assert created is True
|
||||
for table in ("threads_meta", "runs", "feedback", "run_events"):
|
||||
count = await _step3_assign_legacy_workspace(sf, table, dry_run=False)
|
||||
assert count == 1, table
|
||||
|
||||
# Re-running the anchor helper is a no-op.
|
||||
assert await _ensure_legacy_workspace(sf, dry_run=False) is False
|
||||
|
||||
async with sf() as session:
|
||||
tm = (await session.execute(select(ThreadMetaRow))).scalar_one()
|
||||
run = (await session.execute(select(RunRow))).scalar_one()
|
||||
fb = (await session.execute(select(FeedbackRow))).scalar_one()
|
||||
ev = (await session.execute(select(RunEventRow))).scalar_one()
|
||||
legacy = (await session.execute(select(WorkspaceRow).where(WorkspaceRow.id == LEGACY_WORKSPACE_ID))).scalar_one()
|
||||
legacy_mem = (await session.execute(select(WorkspaceMembershipRow).where(WorkspaceMembershipRow.workspace_id == LEGACY_WORKSPACE_ID))).scalar_one()
|
||||
|
||||
assert tm.workspace_id == LEGACY_WORKSPACE_ID
|
||||
assert run.workspace_id == LEGACY_WORKSPACE_ID
|
||||
assert fb.workspace_id == LEGACY_WORKSPACE_ID
|
||||
assert ev.workspace_id == LEGACY_WORKSPACE_ID
|
||||
assert legacy.slug == "legacy"
|
||||
assert legacy_mem.role == "owner"
|
||||
finally:
|
||||
await _close()
|
||||
|
||||
|
||||
async def test_ensure_legacy_workspace_refuses_when_no_users(tmp_path):
|
||||
"""ensure_legacy_workspace raises a clear error if the DB has no users."""
|
||||
sf = await _init_engine(tmp_path)
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="no users exist"):
|
||||
await _ensure_legacy_workspace(sf, dry_run=False)
|
||||
finally:
|
||||
await _close()
|
||||
|
||||
|
||||
async def test_backfill_dry_run_does_not_write(tmp_path):
|
||||
"""``backfill(..., dry_run=True)`` reports counts but writes nothing."""
|
||||
sf = await _init_engine(tmp_path)
|
||||
try:
|
||||
await _seed_user(sf, email="admin@example.com", system_role="admin")
|
||||
user_id = await _seed_user(sf, email="helen@example.com")
|
||||
await _seed_business_row(sf, ThreadMetaRow, thread_id="t-owned", user_id=user_id)
|
||||
await _seed_business_row(sf, ThreadMetaRow, thread_id="t-orphan", user_id=None)
|
||||
await _seed_business_row(sf, RunRow, run_id="r-owned", thread_id="t-owned", user_id=user_id)
|
||||
|
||||
# Snapshot row counts BEFORE the dry run so we can confirm
|
||||
# nothing changed AFTER.
|
||||
async with sf() as session:
|
||||
ws_before = len((await session.execute(select(WorkspaceRow))).scalars().all())
|
||||
mem_before = len((await session.execute(select(WorkspaceMembershipRow))).scalars().all())
|
||||
users_with_default_before = len((await session.execute(select(UserRow).where(UserRow.default_workspace_id.is_not(None)))).scalars().all())
|
||||
|
||||
report = await backfill(sf, dry_run=True)
|
||||
assert report["dry_run"] is True
|
||||
# Step 1 reports 2 candidates (admin + helen, both without default).
|
||||
assert report["users_workspaces_created"] == 2
|
||||
# Step 2 reports 0 because Step 1 didn't actually populate
|
||||
# users.default_workspace_id under dry_run — the JOIN comes up empty.
|
||||
assert report["threads_meta_from_users"] == 0
|
||||
assert report["runs_from_users"] == 0
|
||||
# Step 3 reports the 3 NULL business rows (t-owned, t-orphan, r-owned).
|
||||
assert report["legacy_workspace_created"] is True
|
||||
assert report["threads_meta_legacy"] == 2
|
||||
assert report["runs_legacy"] == 1
|
||||
|
||||
# State did not change.
|
||||
async with sf() as session:
|
||||
ws_after = len((await session.execute(select(WorkspaceRow))).scalars().all())
|
||||
mem_after = len((await session.execute(select(WorkspaceMembershipRow))).scalars().all())
|
||||
users_with_default_after = len((await session.execute(select(UserRow).where(UserRow.default_workspace_id.is_not(None)))).scalars().all())
|
||||
rows = (await session.execute(select(ThreadMetaRow.workspace_id))).scalars().all()
|
||||
assert ws_after == ws_before
|
||||
assert mem_after == mem_before
|
||||
assert users_with_default_after == users_with_default_before
|
||||
assert all(w is None for w in rows)
|
||||
finally:
|
||||
await _close()
|
||||
|
||||
|
||||
async def test_full_backfill_orchestrator(tmp_path):
|
||||
"""End-to-end: backfill() runs all three steps and reports per-step counts."""
|
||||
sf = await _init_engine(tmp_path)
|
||||
try:
|
||||
await _seed_user(sf, email="admin@example.com", system_role="admin")
|
||||
user_id = await _seed_user(sf, email="gina@example.com")
|
||||
await _seed_business_row(sf, ThreadMetaRow, thread_id="t-owned", user_id=user_id)
|
||||
await _seed_business_row(sf, ThreadMetaRow, thread_id="t-orphan", user_id=None)
|
||||
|
||||
report = await backfill(sf, dry_run=False)
|
||||
assert report["dry_run"] is False
|
||||
# Two users were missing a default workspace (admin too — we
|
||||
# didn't pre-populate admin's default_workspace_id).
|
||||
assert report["users_workspaces_created"] == 2
|
||||
assert report["threads_meta_from_users"] == 1
|
||||
assert report["legacy_workspace_created"] is True
|
||||
assert report["threads_meta_legacy"] == 1
|
||||
|
||||
async with sf() as session:
|
||||
rows = {r.thread_id: r.workspace_id for r in (await session.execute(select(ThreadMetaRow))).scalars().all()}
|
||||
assert rows["t-orphan"] == LEGACY_WORKSPACE_ID
|
||||
assert rows["t-owned"] != LEGACY_WORKSPACE_ID
|
||||
assert rows["t-owned"] is not None
|
||||
finally:
|
||||
await _close()
|
||||
@@ -0,0 +1,108 @@
|
||||
"""``change_password`` and ``login_local`` re-issue JWTs carrying wid + role.
|
||||
|
||||
Stage 0 PR4 T4.12. The contract:
|
||||
|
||||
- After ``POST /auth/change-password`` the new session cookie's JWT must
|
||||
still encode the user's workspace under ``wid`` (and ``role='owner'``),
|
||||
with ``ver`` bumped. Dropping ``wid`` here would lock the user out of
|
||||
every protected endpoint immediately after a password change.
|
||||
- Same for ``POST /auth/login/local`` — it issues a fresh JWT and must
|
||||
also encode ``wid``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
os.environ.setdefault("AUTH_JWT_SECRET", "test-secret-key-change-password-wid-32x")
|
||||
|
||||
from app.gateway.auth.config import AuthConfig, set_auth_config
|
||||
|
||||
_TEST_SECRET = "test-secret-key-change-password-wid-32x"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _setup_auth(tmp_path):
|
||||
from app.gateway import deps
|
||||
from app.gateway.routers.auth import _SETUP_STATUS_COOLDOWN
|
||||
from deerflow.persistence.engine import close_engine, init_engine
|
||||
|
||||
set_auth_config(AuthConfig(jwt_secret=_TEST_SECRET))
|
||||
url = f"sqlite+aiosqlite:///{tmp_path}/change_pwd.db"
|
||||
asyncio.run(init_engine("sqlite", url=url, sqlite_dir=str(tmp_path)))
|
||||
deps._cached_local_provider = None
|
||||
deps._cached_repo = None
|
||||
_SETUP_STATUS_COOLDOWN.clear()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
deps._cached_local_provider = None
|
||||
deps._cached_repo = None
|
||||
_SETUP_STATUS_COOLDOWN.clear()
|
||||
asyncio.run(close_engine())
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client(_setup_auth):
|
||||
from app.gateway.app import create_app
|
||||
|
||||
set_auth_config(AuthConfig(jwt_secret=_TEST_SECRET))
|
||||
yield TestClient(create_app())
|
||||
|
||||
|
||||
def _decode(token: str) -> dict:
|
||||
return jwt.decode(token, _TEST_SECRET, algorithms=["HS256"])
|
||||
|
||||
|
||||
def _bootstrap_user(client) -> tuple[str, dict]:
|
||||
"""Register a user and return (user_id, initial claims)."""
|
||||
client.post("/api/v1/auth/initialize", json={"email": "admin@example.com", "password": "Str0ng!Pass99"})
|
||||
resp = client.post("/api/v1/auth/register", json={"email": "alice@example.com", "password": "Tr0ub4dor3a-strong!"})
|
||||
assert resp.status_code == 201, resp.text
|
||||
return resp.json()["id"], _decode(resp.cookies["access_token"])
|
||||
|
||||
|
||||
def test_change_password_keeps_wid_and_bumps_ver(client):
|
||||
"""change_password re-signs the JWT with wid + role; ver moves forward."""
|
||||
user_id, initial_claims = _bootstrap_user(client)
|
||||
|
||||
# /register set the csrf cookie; the matching header is required on
|
||||
# the change-password POST (Double Submit Cookie pattern).
|
||||
csrf = client.cookies.get("csrf_token")
|
||||
assert csrf, "register must have set csrf_token cookie"
|
||||
|
||||
resp = client.post(
|
||||
"/api/v1/auth/change-password",
|
||||
json={"current_password": "Tr0ub4dor3a-strong!", "new_password": "Tr0ub4dor3a-strong2!"},
|
||||
headers={"X-CSRF-Token": csrf},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
new_claims = _decode(resp.cookies["access_token"])
|
||||
assert new_claims["sub"] == user_id
|
||||
assert new_claims["wid"] == initial_claims["wid"], "wid must survive a password change"
|
||||
assert new_claims["role"] == "owner"
|
||||
assert new_claims["ver"] == initial_claims["ver"] + 1, "token_version must advance"
|
||||
|
||||
|
||||
def test_login_issues_wid_carrying_jwt(client):
|
||||
"""/auth/login/local issues a JWT that the workspace middleware will accept."""
|
||||
user_id, _ = _bootstrap_user(client)
|
||||
|
||||
# Clear the session cookie set by /register so the login response is observed in isolation.
|
||||
client.cookies.clear()
|
||||
resp = client.post(
|
||||
"/api/v1/auth/login/local",
|
||||
data={"username": "alice@example.com", "password": "Tr0ub4dor3a-strong!"},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
claims = _decode(resp.cookies["access_token"])
|
||||
assert claims["sub"] == user_id
|
||||
assert claims.get("wid"), "login must include wid"
|
||||
assert claims["role"] == "owner"
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Stage 0 PR2 · default backend regression + sqlite-still-works.
|
||||
|
||||
Two facts pinned here:
|
||||
T2.3 — explicit ``database.backend: sqlite`` still produces a working
|
||||
config (backwards-compat for users who deliberately stay on SQLite).
|
||||
T2.4 — ``config.example.yaml`` default ``database.backend`` is ``postgres``
|
||||
(Stage 0 PR2 flipped this from sqlite).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from deerflow.config.app_config import AppConfig
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _write_extensions_config(path: Path) -> None:
|
||||
path.write_text(json.dumps({"mcpServers": {}, "skills": {}}), encoding="utf-8")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# T2.3 · sqlite backend regression
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_explicit_sqlite_backend_still_works(tmp_path, monkeypatch) -> None:
|
||||
"""A config with explicit ``database.backend: sqlite`` must still parse.
|
||||
|
||||
Pin: PR2 made postgres the example default. This test guarantees that
|
||||
users who copy the SQLite fallback block to their config.yaml do not
|
||||
silently regress.
|
||||
"""
|
||||
config_path = tmp_path / "config.yaml"
|
||||
extensions_path = tmp_path / "extensions_config.json"
|
||||
_write_extensions_config(extensions_path)
|
||||
config_path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"models": [
|
||||
{
|
||||
"name": "test",
|
||||
"use": "langchain_openai:ChatOpenAI",
|
||||
"model": "gpt-4",
|
||||
}
|
||||
],
|
||||
"database": {
|
||||
"backend": "sqlite",
|
||||
"sqlite_dir": "/custom/sqlite/path",
|
||||
},
|
||||
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(extensions_path))
|
||||
|
||||
config = AppConfig.from_file(str(config_path))
|
||||
|
||||
assert config.database.backend == "sqlite"
|
||||
assert config.database.sqlite_dir == "/custom/sqlite/path"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# T2.4 · default config.example.yaml uses postgres
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_config_example_default_backend_is_postgres(monkeypatch) -> None:
|
||||
"""Pin Stage 0 PR2's commitment: the example config defaults to Postgres.
|
||||
|
||||
Loaded directly from the on-disk ``config.example.yaml`` so any future
|
||||
accidental flip back to sqlite would fail this test.
|
||||
"""
|
||||
example_path = REPO_ROOT / "config.example.yaml"
|
||||
if not example_path.exists():
|
||||
pytest.skip(f"config.example.yaml not found at {example_path}")
|
||||
|
||||
raw = yaml.safe_load(example_path.read_text(encoding="utf-8")) or {}
|
||||
db = raw.get("database") or {}
|
||||
|
||||
assert db.get("backend") == "postgres", f"config.example.yaml database.backend is {db.get('backend')!r}; PR2 requires 'postgres' as the default"
|
||||
# The PG URL must come from the env (referenced as $DATABASE_URL),
|
||||
# never hardcoded with credentials.
|
||||
assert db.get("postgres_url") == "$DATABASE_URL", f"postgres_url should be '$DATABASE_URL' env reference, got {db.get('postgres_url')!r}"
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Lifespan hook backfills missing workspaces for pre-PR4 admins.
|
||||
|
||||
Stage 0 PR4 T4.13. Production scenario: a deployment that pre-dates
|
||||
PR4 has an admin user whose ``users.default_workspace_id`` is NULL.
|
||||
After the upgrade, the first time the app boots, the lifespan hook
|
||||
must create the admin's personal workspace + owner membership so the
|
||||
admin can immediately log in without hitting the post-PR4 workspace
|
||||
gate (T4.7).
|
||||
|
||||
This test directly invokes ``_ensure_admin_user`` against a fixture
|
||||
SQLite DB, simulating the upgrade path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
|
||||
from app.gateway.auth.config import AuthConfig, set_auth_config
|
||||
|
||||
_TEST_SECRET = "test-secret-key-admin-backfill-32-chars"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _setup(tmp_path):
|
||||
from app.gateway import deps
|
||||
from deerflow.persistence.engine import close_engine, init_engine
|
||||
|
||||
set_auth_config(AuthConfig(jwt_secret=_TEST_SECRET))
|
||||
url = f"sqlite+aiosqlite:///{tmp_path}/admin_backfill.db"
|
||||
asyncio.run(init_engine("sqlite", url=url, sqlite_dir=str(tmp_path)))
|
||||
deps._cached_local_provider = None
|
||||
deps._cached_repo = None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
deps._cached_local_provider = None
|
||||
deps._cached_repo = None
|
||||
asyncio.run(close_engine())
|
||||
|
||||
|
||||
async def _seed_pre_pr4_admin(email: str = "admin@example.com") -> str:
|
||||
"""Insert an admin user with default_workspace_id=NULL (pre-PR4 state)."""
|
||||
from app.gateway.deps import get_local_provider
|
||||
|
||||
provider = get_local_provider()
|
||||
user = await provider.create_user(email=email, password="Str0ng!Pass99", system_role="admin")
|
||||
# Belt + suspenders: pretend this user pre-dates PR4 even if the
|
||||
# provider added a workspace_id (it does not today, but explicit
|
||||
# is better).
|
||||
user.default_workspace_id = None
|
||||
await provider.update_user(user)
|
||||
return str(user.id)
|
||||
|
||||
|
||||
async def _read_workspace_state(user_id: str) -> dict:
|
||||
from sqlalchemy import select
|
||||
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.workspace.model import WorkspaceRow
|
||||
from deerflow.persistence.workspace_membership.model import WorkspaceMembershipRow
|
||||
|
||||
sf = get_session_factory()
|
||||
async with sf() as session:
|
||||
user = await session.get(UserRow, user_id)
|
||||
memberships = (await session.execute(select(WorkspaceMembershipRow).where(WorkspaceMembershipRow.user_id == user_id))).scalars().all()
|
||||
workspaces = []
|
||||
if memberships:
|
||||
workspaces = (await session.execute(select(WorkspaceRow).where(WorkspaceRow.id.in_([m.workspace_id for m in memberships])))).scalars().all()
|
||||
return {
|
||||
"default_workspace_id": user.default_workspace_id if user else None,
|
||||
"memberships": [(m.workspace_id, m.role) for m in memberships],
|
||||
"workspaces": [(w.id, w.slug) for w in workspaces],
|
||||
}
|
||||
|
||||
|
||||
def test_ensure_admin_user_creates_missing_workspace():
|
||||
"""Pre-PR4 admin without default_workspace_id → lifespan backfills it."""
|
||||
from app.gateway.app import _ensure_admin_user
|
||||
|
||||
admin_id = asyncio.run(_seed_pre_pr4_admin())
|
||||
before = asyncio.run(_read_workspace_state(admin_id))
|
||||
assert before["default_workspace_id"] is None
|
||||
assert before["workspaces"] == []
|
||||
|
||||
asyncio.run(_ensure_admin_user(FastAPI()))
|
||||
|
||||
after = asyncio.run(_read_workspace_state(admin_id))
|
||||
assert after["default_workspace_id"] is not None, "lifespan should set default_workspace_id"
|
||||
assert len(after["workspaces"]) == 1
|
||||
ws_id, _slug = after["workspaces"][0]
|
||||
assert after["memberships"] == [(ws_id, "owner")]
|
||||
|
||||
|
||||
def test_ensure_admin_user_is_idempotent():
|
||||
"""Running the lifespan hook twice does not create duplicate workspaces."""
|
||||
from app.gateway.app import _ensure_admin_user
|
||||
|
||||
admin_id = asyncio.run(_seed_pre_pr4_admin())
|
||||
asyncio.run(_ensure_admin_user(FastAPI()))
|
||||
state_after_first = asyncio.run(_read_workspace_state(admin_id))
|
||||
|
||||
asyncio.run(_ensure_admin_user(FastAPI()))
|
||||
state_after_second = asyncio.run(_read_workspace_state(admin_id))
|
||||
|
||||
assert state_after_first == state_after_second, "second run must be a no-op"
|
||||
assert len(state_after_second["workspaces"]) == 1
|
||||
|
||||
|
||||
def test_ensure_admin_user_skips_when_admin_already_has_workspace():
|
||||
"""An admin with a workspace already set should not get a second one."""
|
||||
from app.gateway.app import _ensure_admin_user
|
||||
from app.gateway.deps import get_local_provider
|
||||
|
||||
admin_id = asyncio.run(_seed_pre_pr4_admin())
|
||||
|
||||
async def _set_default(workspace_id: str):
|
||||
provider = get_local_provider()
|
||||
user = await provider.get_user(admin_id)
|
||||
user.default_workspace_id = workspace_id
|
||||
await provider.update_user(user)
|
||||
|
||||
# Run the lifespan hook once to seed a workspace, then re-run.
|
||||
asyncio.run(_ensure_admin_user(FastAPI()))
|
||||
state_seeded = asyncio.run(_read_workspace_state(admin_id))
|
||||
assert len(state_seeded["workspaces"]) == 1
|
||||
seeded_ws_id = state_seeded["workspaces"][0][0]
|
||||
|
||||
asyncio.run(_set_default(seeded_ws_id)) # ensure default is still pointing at it
|
||||
asyncio.run(_ensure_admin_user(FastAPI()))
|
||||
|
||||
final = asyncio.run(_read_workspace_state(admin_id))
|
||||
assert final["default_workspace_id"] == seeded_ws_id
|
||||
assert [w[0] for w in final["workspaces"]] == [seeded_ws_id]
|
||||
@@ -74,7 +74,7 @@ def test_expired_jwt_raises_401():
|
||||
|
||||
|
||||
def test_user_not_found_raises_401():
|
||||
token = create_access_token("ghost")
|
||||
token = create_access_token("ghost", workspace_id="ws-test", role="owner")
|
||||
with patch("app.gateway.langgraph_auth.get_local_provider", return_value=_mock_provider(None)):
|
||||
with pytest.raises(Auth.exceptions.HTTPException) as exc:
|
||||
asyncio.run(authenticate(_req({"access_token": token})))
|
||||
@@ -84,7 +84,7 @@ def test_user_not_found_raises_401():
|
||||
|
||||
def test_token_version_mismatch_raises_401():
|
||||
user = _user(token_version=2)
|
||||
token = create_access_token(str(user.id), token_version=1)
|
||||
token = create_access_token(str(user.id), token_version=1, workspace_id="ws-test", role="owner")
|
||||
with patch("app.gateway.langgraph_auth.get_local_provider", return_value=_mock_provider(user)):
|
||||
with pytest.raises(Auth.exceptions.HTTPException) as exc:
|
||||
asyncio.run(authenticate(_req({"access_token": token})))
|
||||
@@ -94,7 +94,7 @@ def test_token_version_mismatch_raises_401():
|
||||
|
||||
def test_valid_token_returns_user_id():
|
||||
user = _user(token_version=0)
|
||||
token = create_access_token(str(user.id), token_version=0)
|
||||
token = create_access_token(str(user.id), token_version=0, workspace_id="ws-test", role="owner")
|
||||
with patch("app.gateway.langgraph_auth.get_local_provider", return_value=_mock_provider(user)):
|
||||
result = asyncio.run(authenticate(_req({"access_token": token})))
|
||||
assert result == str(user.id)
|
||||
@@ -102,7 +102,7 @@ def test_valid_token_returns_user_id():
|
||||
|
||||
def test_valid_token_matching_version():
|
||||
user = _user(token_version=5)
|
||||
token = create_access_token(str(user.id), token_version=5)
|
||||
token = create_access_token(str(user.id), token_version=5, workspace_id="ws-test", role="owner")
|
||||
with patch("app.gateway.langgraph_auth.get_local_provider", return_value=_mock_provider(user)):
|
||||
result = asyncio.run(authenticate(_req({"access_token": token})))
|
||||
assert result == str(user.id)
|
||||
@@ -113,7 +113,7 @@ def test_valid_token_matching_version():
|
||||
|
||||
def test_provider_exception_propagates():
|
||||
"""Provider raises → should not be swallowed silently."""
|
||||
token = create_access_token("user-1")
|
||||
token = create_access_token("user-1", workspace_id="ws-test", role="owner")
|
||||
p = AsyncMock()
|
||||
p.get_user = AsyncMock(side_effect=RuntimeError("DB down"))
|
||||
with patch("app.gateway.langgraph_auth.get_local_provider", return_value=p):
|
||||
@@ -126,7 +126,11 @@ def test_jwt_missing_ver_defaults_to_zero():
|
||||
import jwt as pyjwt
|
||||
|
||||
uid = str(uuid4())
|
||||
raw = pyjwt.encode({"sub": uid, "exp": 9999999999, "iat": 1000000000}, _JWT_SECRET, algorithm="HS256")
|
||||
raw = pyjwt.encode(
|
||||
{"sub": uid, "wid": "ws-test", "role": "owner", "exp": 9999999999, "iat": 1000000000},
|
||||
_JWT_SECRET,
|
||||
algorithm="HS256",
|
||||
)
|
||||
user = _user(user_id=uid, token_version=0)
|
||||
with patch("app.gateway.langgraph_auth.get_local_provider", return_value=_mock_provider(user)):
|
||||
result = asyncio.run(authenticate(_req({"access_token": raw})))
|
||||
@@ -138,7 +142,11 @@ def test_jwt_missing_ver_rejected_when_user_version_nonzero():
|
||||
import jwt as pyjwt
|
||||
|
||||
uid = str(uuid4())
|
||||
raw = pyjwt.encode({"sub": uid, "exp": 9999999999, "iat": 1000000000}, _JWT_SECRET, algorithm="HS256")
|
||||
raw = pyjwt.encode(
|
||||
{"sub": uid, "wid": "ws-test", "role": "owner", "exp": 9999999999, "iat": 1000000000},
|
||||
_JWT_SECRET,
|
||||
algorithm="HS256",
|
||||
)
|
||||
user = _user(user_id=uid, token_version=1)
|
||||
with patch("app.gateway.langgraph_auth.get_local_provider", return_value=_mock_provider(user)):
|
||||
with pytest.raises(Auth.exceptions.HTTPException) as exc:
|
||||
@@ -221,7 +229,7 @@ def test_filter_with_empty_metadata():
|
||||
|
||||
|
||||
def test_shared_jwt_secret():
|
||||
token = create_access_token("user-1", token_version=3)
|
||||
token = create_access_token("user-1", token_version=3, workspace_id="ws-test", role="owner")
|
||||
payload = decode_token(token)
|
||||
from app.gateway.auth.errors import TokenError
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Legacy 4-field JWT compatibility (Stage 0 PR4 T4.6).
|
||||
|
||||
Before PR4 every JWT carried only `{sub, exp, iat, ver}`. After PR4 the
|
||||
server expects `wid` (workspace_id) on every protected request. Old
|
||||
cookies in the wild must NOT collapse into ``TokenError.MALFORMED`` —
|
||||
that hides the actual problem (workspace required) and prevents the
|
||||
frontend from steering the user to ``/select-workspace``.
|
||||
|
||||
The contract: ``decode_token`` returns ``TokenError.WORKSPACE_MISSING``
|
||||
specifically when the JWT signature checks out and the payload is
|
||||
otherwise well-formed but does NOT carry a ``wid`` claim.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
|
||||
from app.gateway.auth.config import get_auth_config
|
||||
from app.gateway.auth.errors import TokenError
|
||||
from app.gateway.auth.jwt import decode_token
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _stable_jwt_secret(monkeypatch):
|
||||
monkeypatch.setenv("AUTH_JWT_SECRET", "test-secret-key-for-jwt-testing-minimum-32-chars")
|
||||
yield
|
||||
|
||||
|
||||
def _make_legacy_token(*, expired: bool = False) -> str:
|
||||
"""Encode a pre-PR4 JWT directly (bypassing create_access_token)."""
|
||||
now = datetime.now(UTC)
|
||||
payload = {
|
||||
"sub": "u-legacy",
|
||||
"exp": now + (timedelta(seconds=-1) if expired else timedelta(hours=1)),
|
||||
"iat": now,
|
||||
"ver": 0,
|
||||
}
|
||||
return jwt.encode(payload, get_auth_config().jwt_secret, algorithm="HS256")
|
||||
|
||||
|
||||
def test_decode_legacy_token_returns_workspace_missing_error() -> None:
|
||||
"""4-field token (no wid) → TokenError.WORKSPACE_MISSING (not MALFORMED)."""
|
||||
token = _make_legacy_token()
|
||||
result = decode_token(token)
|
||||
assert result == TokenError.WORKSPACE_MISSING
|
||||
|
||||
|
||||
def test_decode_legacy_token_with_expired_still_reports_expired() -> None:
|
||||
"""Expired legacy tokens keep reporting EXPIRED — that signal takes priority.
|
||||
|
||||
Why: an expired token must trigger /auth/refresh logic before we
|
||||
decide it also lacks workspace; reporting WORKSPACE_MISSING on an
|
||||
expired token would steer the user to /select-workspace instead.
|
||||
"""
|
||||
token = _make_legacy_token(expired=True)
|
||||
result = decode_token(token)
|
||||
assert result == TokenError.EXPIRED
|
||||
@@ -0,0 +1,155 @@
|
||||
"""PR6 T6.13 — workspace-path migration script tests.
|
||||
|
||||
Builds the legacy ``users/{uid}/...`` tree under a temp base dir,
|
||||
points a populated sqlite ``users`` table at it via the conventional
|
||||
``deer-flow.db`` location, and asserts the migration produces the new
|
||||
``workspaces/{wid}/...`` layout. Covers:
|
||||
|
||||
- threads / memory.json / custom agents all rewritten under the workspace
|
||||
- ``--dry-run`` writes nothing
|
||||
- pre-existing destinations get diverted to ``migration-conflicts/``
|
||||
- users without a ``default_workspace_id`` fall back to the explicit flag
|
||||
- empty legacy dirs are cleaned up
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.config.paths import Paths
|
||||
from scripts.migrate_paths_to_workspace import (
|
||||
LEGACY_WORKSPACE_FALLBACK,
|
||||
_load_user_workspaces,
|
||||
migrate,
|
||||
)
|
||||
|
||||
|
||||
def _build_legacy_tree(base: Path, *, user_id: str, thread_ids: tuple[str, ...] = (), with_memory: bool = False, agent_names: tuple[str, ...] = ()) -> None:
|
||||
user_root = base / "users" / user_id
|
||||
user_root.mkdir(parents=True, exist_ok=True)
|
||||
for tid in thread_ids:
|
||||
(user_root / "threads" / tid / "user-data" / "workspace").mkdir(parents=True, exist_ok=True)
|
||||
(user_root / "threads" / tid / "user-data" / "workspace" / "marker.txt").write_text(f"{user_id}/{tid}", encoding="utf-8")
|
||||
if with_memory:
|
||||
(user_root / "memory.json").write_text(f'{{"user_id": "{user_id}"}}', encoding="utf-8")
|
||||
for name in agent_names:
|
||||
(user_root / "agents" / name).mkdir(parents=True, exist_ok=True)
|
||||
(user_root / "agents" / name / "SOUL.md").write_text(f"# {name}", encoding="utf-8")
|
||||
|
||||
|
||||
def _seed_db(base: Path, *, users: dict[str, str | None]) -> None:
|
||||
db_path = base / "deer-flow.db"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
conn.execute("CREATE TABLE users (id TEXT PRIMARY KEY, default_workspace_id TEXT)")
|
||||
conn.executemany("INSERT INTO users (id, default_workspace_id) VALUES (?, ?)", list(users.items()))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def base(tmp_path: Path) -> Path:
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_migrate_threads_under_workspace(base: Path):
|
||||
_build_legacy_tree(base, user_id="alice", thread_ids=("t1",))
|
||||
_seed_db(base, users={"alice": "ws-alpha"})
|
||||
paths = Paths(base)
|
||||
|
||||
report = migrate(paths, user_workspaces=_load_user_workspaces(paths), fallback_workspace=LEGACY_WORKSPACE_FALLBACK, dry_run=False)
|
||||
|
||||
assert (base / "workspaces" / "ws-alpha" / "threads" / "t1" / "user-data" / "workspace" / "marker.txt").read_text(encoding="utf-8") == "alice/t1"
|
||||
assert not (base / "users" / "alice" / "threads").exists()
|
||||
assert {entry["asset"] for entry in report} == {"thread"}
|
||||
|
||||
|
||||
def test_migrate_memory_and_agents_nested_under_workspace_and_user(base: Path):
|
||||
_build_legacy_tree(base, user_id="alice", with_memory=True, agent_names=("code-reviewer",))
|
||||
_seed_db(base, users={"alice": "ws-alpha"})
|
||||
paths = Paths(base)
|
||||
|
||||
migrate(paths, user_workspaces=_load_user_workspaces(paths), fallback_workspace=LEGACY_WORKSPACE_FALLBACK, dry_run=False)
|
||||
|
||||
assert (base / "workspaces" / "ws-alpha" / "users" / "alice" / "memory.json").exists()
|
||||
assert (base / "workspaces" / "ws-alpha" / "users" / "alice" / "agents" / "code-reviewer" / "SOUL.md").exists()
|
||||
|
||||
|
||||
def test_dry_run_writes_nothing(base: Path):
|
||||
_build_legacy_tree(base, user_id="alice", thread_ids=("t1",), with_memory=True, agent_names=("a1",))
|
||||
_seed_db(base, users={"alice": "ws-alpha"})
|
||||
paths = Paths(base)
|
||||
|
||||
report = migrate(paths, user_workspaces=_load_user_workspaces(paths), fallback_workspace=LEGACY_WORKSPACE_FALLBACK, dry_run=True)
|
||||
|
||||
# Source unchanged
|
||||
assert (base / "users" / "alice" / "threads" / "t1" / "user-data" / "workspace" / "marker.txt").exists()
|
||||
assert (base / "users" / "alice" / "memory.json").exists()
|
||||
assert (base / "users" / "alice" / "agents" / "a1" / "SOUL.md").exists()
|
||||
# No destination created
|
||||
assert not (base / "workspaces").exists()
|
||||
# Report still populated so operator sees what *would* happen
|
||||
assert len(report) == 3
|
||||
|
||||
|
||||
def test_fallback_workspace_used_when_user_has_no_default(base: Path):
|
||||
_build_legacy_tree(base, user_id="alice", thread_ids=("t1",))
|
||||
_seed_db(base, users={"alice": None})
|
||||
paths = Paths(base)
|
||||
|
||||
migrate(paths, user_workspaces=_load_user_workspaces(paths), fallback_workspace="legacy_workspace", dry_run=False)
|
||||
|
||||
assert (base / "workspaces" / "legacy_workspace" / "threads" / "t1" / "user-data" / "workspace" / "marker.txt").exists()
|
||||
|
||||
|
||||
def test_conflict_routes_legacy_to_migration_conflicts(base: Path):
|
||||
# Pre-create the destination with a different marker so the move sees a conflict.
|
||||
_build_legacy_tree(base, user_id="alice", thread_ids=("t1",))
|
||||
(base / "workspaces" / "ws-alpha" / "threads" / "t1" / "user-data" / "workspace").mkdir(parents=True)
|
||||
(base / "workspaces" / "ws-alpha" / "threads" / "t1" / "user-data" / "workspace" / "marker.txt").write_text("preexisting", encoding="utf-8")
|
||||
_seed_db(base, users={"alice": "ws-alpha"})
|
||||
paths = Paths(base)
|
||||
|
||||
report = migrate(paths, user_workspaces=_load_user_workspaces(paths), fallback_workspace=LEGACY_WORKSPACE_FALLBACK, dry_run=False)
|
||||
|
||||
assert (base / "workspaces" / "ws-alpha" / "threads" / "t1" / "user-data" / "workspace" / "marker.txt").read_text(encoding="utf-8") == "preexisting"
|
||||
conflict_marker = base / "migration-conflicts" / "workspace-migration" / "threads/ws-alpha" / "t1"
|
||||
assert conflict_marker.exists()
|
||||
assert any("conflict" in entry["action"] for entry in report)
|
||||
|
||||
|
||||
def test_empty_users_dir_removed_after_full_migration(base: Path):
|
||||
_build_legacy_tree(base, user_id="alice", thread_ids=("t1",))
|
||||
_seed_db(base, users={"alice": "ws-alpha"})
|
||||
paths = Paths(base)
|
||||
|
||||
migrate(paths, user_workspaces=_load_user_workspaces(paths), fallback_workspace=LEGACY_WORKSPACE_FALLBACK, dry_run=False)
|
||||
|
||||
assert not (base / "users").exists(), "Empty legacy users/ dir should be cleaned up"
|
||||
|
||||
|
||||
def test_no_users_directory_is_noop(base: Path):
|
||||
paths = Paths(base)
|
||||
report = migrate(paths, user_workspaces={}, fallback_workspace=LEGACY_WORKSPACE_FALLBACK, dry_run=False)
|
||||
assert report == []
|
||||
|
||||
|
||||
def test_missing_db_returns_empty_mapping(base: Path):
|
||||
paths = Paths(base)
|
||||
assert _load_user_workspaces(paths) == {}
|
||||
|
||||
|
||||
def test_db_without_users_table_returns_empty(base: Path):
|
||||
db_path = base / "deer-flow.db"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
conn.execute("CREATE TABLE other_table (x INTEGER)")
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
paths = Paths(base)
|
||||
assert _load_user_workspaces(paths) == {}
|
||||
@@ -0,0 +1,45 @@
|
||||
"""PR6 T6.14 — lifespan warns when legacy users/ tree still has content."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.gateway.app import _check_path_migration_pending
|
||||
|
||||
|
||||
class _FakePaths:
|
||||
def __init__(self, base: Path):
|
||||
self.base_dir = base
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def base(tmp_path: Path) -> Path:
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _patch_paths(base: Path):
|
||||
return patch("deerflow.config.paths.get_paths", return_value=_FakePaths(base))
|
||||
|
||||
|
||||
def test_warns_when_legacy_users_dir_has_content(base: Path, caplog: pytest.LogCaptureFixture):
|
||||
(base / "users" / "alice" / "threads" / "t1").mkdir(parents=True)
|
||||
with _patch_paths(base), caplog.at_level(logging.WARNING, logger="app.gateway.app"):
|
||||
_check_path_migration_pending(app=None) # type: ignore[arg-type]
|
||||
assert any("make migrate-paths" in rec.message for rec in caplog.records)
|
||||
|
||||
|
||||
def test_silent_when_legacy_users_dir_missing(base: Path, caplog: pytest.LogCaptureFixture):
|
||||
with _patch_paths(base), caplog.at_level(logging.WARNING, logger="app.gateway.app"):
|
||||
_check_path_migration_pending(app=None) # type: ignore[arg-type]
|
||||
assert not any("migrate-paths" in rec.message for rec in caplog.records)
|
||||
|
||||
|
||||
def test_silent_when_legacy_users_dir_empty(base: Path, caplog: pytest.LogCaptureFixture):
|
||||
(base / "users").mkdir()
|
||||
with _patch_paths(base), caplog.at_level(logging.WARNING, logger="app.gateway.app"):
|
||||
_check_path_migration_pending(app=None) # type: ignore[arg-type]
|
||||
assert not any("migrate-paths" in rec.message for rec in caplog.records)
|
||||
@@ -0,0 +1,115 @@
|
||||
"""PR6 T6.9 / T6.10 — workspace-scoped path resolution.
|
||||
|
||||
The Paths class learns a new top-level dimension for multi-tenant
|
||||
filesystems: ``{base_dir}/workspaces/{wid}/...``. Precedence:
|
||||
|
||||
- ``workspace_id`` given → new shape ``workspaces/{wid}/threads/{tid}/...``
|
||||
- only ``user_id`` given → legacy shape ``users/{uid}/threads/{tid}/...``
|
||||
- neither → very-legacy shape ``threads/{tid}/...``
|
||||
|
||||
Per-user filesystem state (memory.json, custom agents) lives under the
|
||||
workspace too: ``workspaces/{wid}/users/{uid}/memory.json`` etc. — so a
|
||||
user's memory cannot be reused across workspaces by mistake.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.config.paths import Paths
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def paths(tmp_path: Path) -> Paths:
|
||||
return Paths(tmp_path)
|
||||
|
||||
|
||||
class TestValidateWorkspaceId:
|
||||
def test_valid_workspace_id(self, paths: Paths):
|
||||
d = paths.workspace_dir("ws-abc-123")
|
||||
assert d == paths.base_dir / "workspaces" / "ws-abc-123"
|
||||
|
||||
def test_rejects_path_traversal(self, paths: Paths):
|
||||
with pytest.raises(ValueError, match="Invalid workspace_id"):
|
||||
paths.workspace_dir("../escape")
|
||||
|
||||
def test_rejects_slash(self, paths: Paths):
|
||||
with pytest.raises(ValueError, match="Invalid workspace_id"):
|
||||
paths.workspace_dir("ws/bar")
|
||||
|
||||
def test_rejects_empty(self, paths: Paths):
|
||||
with pytest.raises(ValueError, match="Invalid workspace_id"):
|
||||
paths.workspace_dir("")
|
||||
|
||||
|
||||
class TestWorkspaceScopedThreadDir:
|
||||
def test_workspace_takes_precedence_over_user(self, paths: Paths):
|
||||
"""When both are given, workspace wins — user_id is recorded in the row, not the filesystem."""
|
||||
expected = paths.base_dir / "workspaces" / "ws-alpha" / "threads" / "t1"
|
||||
assert paths.thread_dir("t1", workspace_id="ws-alpha", user_id="alice") == expected
|
||||
|
||||
def test_workspace_only(self, paths: Paths):
|
||||
expected = paths.base_dir / "workspaces" / "ws-alpha" / "threads" / "t1"
|
||||
assert paths.thread_dir("t1", workspace_id="ws-alpha") == expected
|
||||
|
||||
def test_user_only_still_legacy(self, paths: Paths):
|
||||
"""Legacy callers keep the user_id-only shape until migration runs."""
|
||||
expected = paths.base_dir / "users" / "alice" / "threads" / "t1"
|
||||
assert paths.thread_dir("t1", user_id="alice") == expected
|
||||
|
||||
def test_no_ids_very_legacy(self, paths: Paths):
|
||||
expected = paths.base_dir / "threads" / "t1"
|
||||
assert paths.thread_dir("t1") == expected
|
||||
|
||||
|
||||
class TestEnsureThreadDirsWorkspace:
|
||||
def test_creates_workspace_layout(self, paths: Paths):
|
||||
paths.ensure_thread_dirs("t1", workspace_id="ws-alpha")
|
||||
root = paths.base_dir / "workspaces" / "ws-alpha" / "threads" / "t1"
|
||||
for sub in ("user-data/workspace", "user-data/uploads", "user-data/outputs", "acp-workspace"):
|
||||
assert (root / sub).is_dir(), f"missing {sub}"
|
||||
|
||||
|
||||
class TestSandboxDirsWorkspace:
|
||||
def test_sandbox_work_dir(self, paths: Paths):
|
||||
expected = paths.base_dir / "workspaces" / "ws-alpha" / "threads" / "t1" / "user-data" / "workspace"
|
||||
assert paths.sandbox_work_dir("t1", workspace_id="ws-alpha") == expected
|
||||
|
||||
def test_sandbox_uploads_dir(self, paths: Paths):
|
||||
expected = paths.base_dir / "workspaces" / "ws-alpha" / "threads" / "t1" / "user-data" / "uploads"
|
||||
assert paths.sandbox_uploads_dir("t1", workspace_id="ws-alpha") == expected
|
||||
|
||||
def test_sandbox_outputs_dir(self, paths: Paths):
|
||||
expected = paths.base_dir / "workspaces" / "ws-alpha" / "threads" / "t1" / "user-data" / "outputs"
|
||||
assert paths.sandbox_outputs_dir("t1", workspace_id="ws-alpha") == expected
|
||||
|
||||
|
||||
class TestUserMemoryUnderWorkspace:
|
||||
def test_user_memory_file_under_workspace(self, paths: Paths):
|
||||
expected = paths.base_dir / "workspaces" / "ws-alpha" / "users" / "alice" / "memory.json"
|
||||
assert paths.user_memory_file("alice", workspace_id="ws-alpha") == expected
|
||||
|
||||
def test_user_memory_file_legacy_without_workspace(self, paths: Paths):
|
||||
expected = paths.base_dir / "users" / "alice" / "memory.json"
|
||||
assert paths.user_memory_file("alice") == expected
|
||||
|
||||
def test_user_agents_dir_under_workspace(self, paths: Paths):
|
||||
expected = paths.base_dir / "workspaces" / "ws-alpha" / "users" / "alice" / "agents"
|
||||
assert paths.user_agents_dir("alice", workspace_id="ws-alpha") == expected
|
||||
|
||||
def test_user_agent_memory_file_under_workspace(self, paths: Paths):
|
||||
expected = paths.base_dir / "workspaces" / "ws-alpha" / "users" / "alice" / "agents" / "code-reviewer" / "memory.json"
|
||||
assert paths.user_agent_memory_file("alice", "code-reviewer", workspace_id="ws-alpha") == expected
|
||||
|
||||
|
||||
class TestVirtualPathResolutionWorkspace:
|
||||
def test_resolve_virtual_path_workspace_scope(self, paths: Paths):
|
||||
expected = paths.base_dir / "workspaces" / "ws-alpha" / "threads" / "t1" / "user-data" / "outputs" / "x.json"
|
||||
actual = paths.resolve_virtual_path("t1", "/mnt/user-data/outputs/x.json", workspace_id="ws-alpha")
|
||||
assert actual == expected.resolve()
|
||||
|
||||
def test_resolve_virtual_path_rejects_traversal_under_workspace(self, paths: Paths):
|
||||
with pytest.raises(ValueError, match="path traversal"):
|
||||
paths.resolve_virtual_path("t1", "/mnt/user-data/../../etc/passwd", workspace_id="ws-alpha")
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Stage 0 PR1 · Postgres smoke tests via testcontainers.
|
||||
|
||||
These tests exercise the postgres_url fixture and verify that DeerFlow's
|
||||
existing ``init_engine`` + ORM ``Base.metadata.create_all`` works against
|
||||
Postgres exactly the same way it works against SQLite (no Stage 0
|
||||
schema changes here — that's PR3+).
|
||||
|
||||
All tests are gated by ``@pytest.mark.postgres`` and skip cleanly when
|
||||
Docker is unavailable (the postgres_container fixture handles that).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
|
||||
import pytest
|
||||
|
||||
# Mark every test in this module as `postgres` so they only run when
|
||||
# explicitly requested with ``pytest -m postgres``.
|
||||
pytestmark = [pytest.mark.postgres, pytest.mark.anyio]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend() -> str:
|
||||
"""anyio uses asyncio backend (matches DeerFlow's runtime)."""
|
||||
return "asyncio"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# T1.4 · Fixture self-test — verify per-test isolation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_postgres_url_creates_isolated_database(postgres_url: str) -> None:
|
||||
"""The fixture should yield a usable URL pointing at a unique database."""
|
||||
import asyncpg
|
||||
|
||||
# The URL form is `postgresql+asyncpg://user:pass@host:port/test_<hex>`.
|
||||
assert postgres_url.startswith("postgresql+asyncpg://")
|
||||
assert "/test_" in postgres_url
|
||||
|
||||
# Connect with raw asyncpg (strip SQLAlchemy dialect prefix) and confirm
|
||||
# we landed in the named test DB.
|
||||
raw = postgres_url.replace("postgresql+asyncpg://", "postgresql://")
|
||||
conn = await asyncpg.connect(raw)
|
||||
try:
|
||||
current = await conn.fetchval("SELECT current_database()")
|
||||
assert current.startswith("test_"), f"expected test_*, got {current!r}"
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
async def test_postgres_url_isolates_between_tests(postgres_container, postgres_url: str) -> None:
|
||||
"""Two invocations of the fixture should yield two distinct databases.
|
||||
|
||||
Hand-rolls a second database via the same recipe to prove isolation
|
||||
without depending on pytest's own per-test invocation timing.
|
||||
"""
|
||||
import asyncpg
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
|
||||
# Read what DB we're in.
|
||||
raw = postgres_url.replace("postgresql+asyncpg://", "postgresql://")
|
||||
conn = await asyncpg.connect(raw)
|
||||
try:
|
||||
db_a = await conn.fetchval("SELECT current_database()")
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
# Create a *second* DB on the same container directly.
|
||||
container_url = postgres_container.get_connection_url().replace("postgresql+psycopg2://", "postgresql://")
|
||||
parent_url = container_url.rsplit("/", 1)[0] + "/postgres"
|
||||
db_b_name = f"test_{secrets.token_hex(8)}"
|
||||
with psycopg.connect(parent_url, autocommit=True) as c:
|
||||
c.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(db_b_name)))
|
||||
try:
|
||||
assert db_a != db_b_name, "fixture should not reuse a DB name across tests"
|
||||
finally:
|
||||
with psycopg.connect(parent_url, autocommit=True) as c:
|
||||
c.execute(sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(db_b_name)))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# T1.5 · init_engine smoke — Base.metadata.create_all() works on Postgres
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_init_engine_postgres_creates_tables(postgres_url: str) -> None:
|
||||
"""``init_engine`` against Postgres must auto-create existing ORM tables.
|
||||
|
||||
Verifies the four current business tables (users, threads_meta, runs,
|
||||
feedback) plus run_events appear in information_schema after init —
|
||||
proving that DeerFlow's ``Base.metadata.create_all()`` path works
|
||||
identically on PG and SQLite.
|
||||
"""
|
||||
from sqlalchemy import text
|
||||
|
||||
from deerflow.persistence.engine import close_engine, get_session_factory, init_engine
|
||||
|
||||
await init_engine("postgres", url=postgres_url)
|
||||
try:
|
||||
sf = get_session_factory()
|
||||
async with sf() as session:
|
||||
result = await session.execute(text("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'"))
|
||||
tables = {row[0] for row in result.all()}
|
||||
|
||||
# Existing tables before any Stage 0 schema additions:
|
||||
expected = {"users", "threads_meta", "runs", "feedback", "run_events"}
|
||||
missing = expected - tables
|
||||
assert not missing, f"create_all() did not produce {missing}; got {tables}"
|
||||
finally:
|
||||
await close_engine()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# T1.6 · Repository round-trip on Postgres
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_thread_meta_repo_postgres_round_trip(postgres_url: str) -> None:
|
||||
"""ThreadMetaRepository.create + get must behave identically on PG.
|
||||
|
||||
Pin: this is a regression net for any SQLAlchemy / asyncpg surprise
|
||||
that diverges from SQLite behavior in dict shape, default values,
|
||||
or timestamp precision.
|
||||
"""
|
||||
from deerflow.persistence.engine import close_engine, get_session_factory, init_engine
|
||||
from deerflow.persistence.thread_meta import ThreadMetaRepository
|
||||
|
||||
await init_engine("postgres", url=postgres_url)
|
||||
try:
|
||||
repo = ThreadMetaRepository(get_session_factory())
|
||||
# Note: conftest's `_auto_user_context` autouse fixture has already
|
||||
# injected `id="test-user-autouse"` so create() picks it via AUTO.
|
||||
created = await repo.create(
|
||||
thread_id="thread-1",
|
||||
display_name="hello",
|
||||
metadata={"k": "v"},
|
||||
)
|
||||
assert created["thread_id"] == "thread-1"
|
||||
assert created["user_id"] == "test-user-autouse"
|
||||
assert created["display_name"] == "hello"
|
||||
assert created["metadata"] == {"k": "v"}
|
||||
|
||||
fetched = await repo.get("thread-1")
|
||||
assert fetched is not None
|
||||
assert fetched["thread_id"] == "thread-1"
|
||||
assert fetched["user_id"] == "test-user-autouse"
|
||||
finally:
|
||||
await close_engine()
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Registration endpoints (POST /initialize, POST /register) auto-create a workspace.
|
||||
|
||||
Stage 0 PR4 T4.8 + T4.9. Every newly registered user must end up with:
|
||||
|
||||
- a single ``workspaces`` row (their personal workspace),
|
||||
- a single ``workspace_memberships`` row with ``role='owner'``,
|
||||
- ``users.default_workspace_id`` pointing at that workspace,
|
||||
- a session cookie whose JWT carries the workspace as the ``wid`` claim.
|
||||
|
||||
Tests run against a per-test SQLite engine bootstrapped by the
|
||||
fixture; the registration router is exercised through the real
|
||||
TestClient so the full handler + DB transaction path is covered.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
os.environ.setdefault("AUTH_JWT_SECRET", "test-secret-key-register-workspace-32+")
|
||||
|
||||
from app.gateway.auth.config import AuthConfig, set_auth_config
|
||||
|
||||
_TEST_SECRET = "test-secret-key-register-workspace-32+"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _setup_auth(tmp_path):
|
||||
from app.gateway import deps
|
||||
from app.gateway.routers.auth import _SETUP_STATUS_COOLDOWN
|
||||
from deerflow.persistence.engine import close_engine, init_engine
|
||||
|
||||
set_auth_config(AuthConfig(jwt_secret=_TEST_SECRET))
|
||||
url = f"sqlite+aiosqlite:///{tmp_path}/register_ws.db"
|
||||
asyncio.run(init_engine("sqlite", url=url, sqlite_dir=str(tmp_path)))
|
||||
deps._cached_local_provider = None
|
||||
deps._cached_repo = None
|
||||
_SETUP_STATUS_COOLDOWN.clear()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
deps._cached_local_provider = None
|
||||
deps._cached_repo = None
|
||||
_SETUP_STATUS_COOLDOWN.clear()
|
||||
asyncio.run(close_engine())
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client(_setup_auth):
|
||||
from app.gateway.app import create_app
|
||||
|
||||
set_auth_config(AuthConfig(jwt_secret=_TEST_SECRET))
|
||||
app = create_app()
|
||||
yield TestClient(app)
|
||||
|
||||
|
||||
def _init_payload(**extra):
|
||||
return {"email": "admin@example.com", "password": "Str0ng!Pass99", **extra}
|
||||
|
||||
|
||||
def _register_payload(email: str = "alice@example.com", **extra):
|
||||
return {"email": email, "password": "Tr0ub4dor3a-strong!", **extra}
|
||||
|
||||
|
||||
def _decode(token: str) -> dict:
|
||||
"""Decode a JWT (signature-checked) and return the raw payload."""
|
||||
return jwt.decode(token, _TEST_SECRET, algorithms=["HS256"])
|
||||
|
||||
|
||||
async def _read_workspace_state(user_id: str) -> dict:
|
||||
"""Inspect the per-user workspace state after a registration call.
|
||||
|
||||
Returns a dict with the workspace row, the owner membership row and
|
||||
the user's default_workspace_id, so each test can pick what it
|
||||
cares about.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.workspace.model import WorkspaceRow
|
||||
from deerflow.persistence.workspace_membership.model import WorkspaceMembershipRow
|
||||
|
||||
sf = get_session_factory()
|
||||
async with sf() as session:
|
||||
user = await session.get(UserRow, user_id)
|
||||
memberships = (await session.execute(select(WorkspaceMembershipRow).where(WorkspaceMembershipRow.user_id == user_id))).scalars().all()
|
||||
workspaces = []
|
||||
if memberships:
|
||||
workspaces = (await session.execute(select(WorkspaceRow).where(WorkspaceRow.id.in_([m.workspace_id for m in memberships])))).scalars().all()
|
||||
return {
|
||||
"user_default_workspace_id": getattr(user, "default_workspace_id", None) if user else None,
|
||||
"memberships": [(m.workspace_id, m.user_id, m.role) for m in memberships],
|
||||
"workspaces": [(w.id, w.slug, w.owner_id) for w in workspaces],
|
||||
}
|
||||
|
||||
|
||||
# ---------- T4.8 — /initialize ---------------------------------------------
|
||||
|
||||
|
||||
def test_initialize_creates_admin_with_default_workspace(client):
|
||||
"""POST /initialize → admin + workspace + owner membership + wid cookie."""
|
||||
resp = client.post("/api/v1/auth/initialize", json=_init_payload())
|
||||
assert resp.status_code == 201, resp.text
|
||||
user_id = resp.json()["id"]
|
||||
|
||||
state = asyncio.run(_read_workspace_state(user_id))
|
||||
|
||||
assert len(state["workspaces"]) == 1, state
|
||||
ws_id, ws_slug, ws_owner = state["workspaces"][0]
|
||||
assert ws_owner == user_id
|
||||
# base slug "admin" is reserved, walker bumps to first free suffix
|
||||
assert ws_slug == "admin-2"
|
||||
|
||||
assert state["memberships"] == [(ws_id, user_id, "owner")]
|
||||
assert state["user_default_workspace_id"] == ws_id
|
||||
|
||||
token = resp.cookies["access_token"]
|
||||
claims = _decode(token)
|
||||
assert claims["wid"] == ws_id
|
||||
assert claims["role"] == "owner"
|
||||
|
||||
|
||||
# ---------- T4.9 — /register -----------------------------------------------
|
||||
|
||||
|
||||
def test_register_creates_user_with_default_workspace(client):
|
||||
"""POST /register → user + workspace + owner membership + wid cookie."""
|
||||
# Initialize an admin first so the system is past first-boot.
|
||||
client.post("/api/v1/auth/initialize", json=_init_payload())
|
||||
|
||||
resp = client.post("/api/v1/auth/register", json=_register_payload())
|
||||
assert resp.status_code == 201, resp.text
|
||||
user_id = resp.json()["id"]
|
||||
|
||||
state = asyncio.run(_read_workspace_state(user_id))
|
||||
|
||||
assert len(state["workspaces"]) == 1, state
|
||||
ws_id, ws_slug, ws_owner = state["workspaces"][0]
|
||||
assert ws_owner == user_id
|
||||
assert ws_slug == "alice"
|
||||
|
||||
assert state["memberships"] == [(ws_id, user_id, "owner")]
|
||||
assert state["user_default_workspace_id"] == ws_id
|
||||
|
||||
token = resp.cookies["access_token"]
|
||||
claims = _decode(token)
|
||||
assert claims["wid"] == ws_id
|
||||
assert claims["role"] == "owner"
|
||||
|
||||
|
||||
def test_two_registrations_isolate_workspaces_and_avoid_slug_collision(client):
|
||||
"""Two users with colliding email local-parts → distinct workspaces, slug suffix bump."""
|
||||
client.post("/api/v1/auth/initialize", json=_init_payload())
|
||||
|
||||
r1 = client.post("/api/v1/auth/register", json=_register_payload(email="alice@example.com"))
|
||||
r2 = client.post("/api/v1/auth/register", json=_register_payload(email="alice@somewhere.else"))
|
||||
assert r1.status_code == 201, r1.text
|
||||
assert r2.status_code == 201, r2.text
|
||||
|
||||
s1 = asyncio.run(_read_workspace_state(r1.json()["id"]))
|
||||
s2 = asyncio.run(_read_workspace_state(r2.json()["id"]))
|
||||
|
||||
ws1 = s1["workspaces"][0]
|
||||
ws2 = s2["workspaces"][0]
|
||||
assert ws1[0] != ws2[0], "workspaces must be distinct"
|
||||
assert ws1[1] == "alice"
|
||||
assert ws2[1] == "alice-2", "slug collision walker should land on -2"
|
||||
@@ -0,0 +1,136 @@
|
||||
"""PR6 T6.6 — `@require_permission(owner_check=True)` workspace upgrade.
|
||||
|
||||
The decorator must:
|
||||
|
||||
1. Pull workspace_id from ``get_effective_workspace_id()`` (set by
|
||||
AuthMiddleware per request) and pass it as the third positional to
|
||||
``ThreadMetaStore.check_access``.
|
||||
2. Raise **HTTPException 404** when check_access returns False — never
|
||||
403 — so a cross-workspace request cannot distinguish "thread exists
|
||||
in another tenant" from "thread does not exist".
|
||||
|
||||
These tests build a fake router with the same decorator usage as the
|
||||
production code and verify the decorator's behaviour via a Mock
|
||||
``thread_store`` whose ``check_access`` call we inspect, plus a TestClient
|
||||
exercising the full HTTP boundary.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
from _router_auth_helpers import make_authed_test_app
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.gateway.auth.models import ActiveWorkspace
|
||||
from app.gateway.authz import require_permission
|
||||
from deerflow.runtime.workspace_context import (
|
||||
reset_current_workspace,
|
||||
set_current_workspace,
|
||||
)
|
||||
|
||||
|
||||
def _make_workspace(wid: str) -> Callable[[], ActiveWorkspace]:
|
||||
"""Factory closure stable per call so the middleware reinjects the same id."""
|
||||
|
||||
def _factory() -> ActiveWorkspace:
|
||||
return ActiveWorkspace(id=wid, role="owner")
|
||||
|
||||
return _factory
|
||||
|
||||
|
||||
def _mount_routes(app):
|
||||
router = APIRouter()
|
||||
|
||||
@router.delete("/probe/{thread_id}")
|
||||
@require_permission("threads", "delete", owner_check=True, require_existing=True)
|
||||
async def _delete_probe(thread_id: str, request: Request): # noqa: ARG001
|
||||
return {"ok": True, "thread_id": thread_id}
|
||||
|
||||
@router.get("/probe/{thread_id}")
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def _get_probe(thread_id: str, request: Request): # noqa: ARG001
|
||||
return {"ok": True, "thread_id": thread_id}
|
||||
|
||||
app.include_router(router)
|
||||
return app
|
||||
|
||||
|
||||
def test_cross_workspace_returns_404():
|
||||
"""check_access returning False surfaces as 404, never 403."""
|
||||
app = make_authed_test_app(
|
||||
workspace_factory=_make_workspace("ws-alpha"),
|
||||
owner_check_passes=False,
|
||||
)
|
||||
_mount_routes(app)
|
||||
with TestClient(app) as client:
|
||||
response = client.delete("/probe/t1")
|
||||
assert response.status_code == 404
|
||||
assert "not found" in response.json()["detail"].lower()
|
||||
|
||||
|
||||
def test_same_workspace_delete_allowed():
|
||||
"""check_access returning True lets the route execute."""
|
||||
app = make_authed_test_app(
|
||||
workspace_factory=_make_workspace("ws-alpha"),
|
||||
owner_check_passes=True,
|
||||
)
|
||||
_mount_routes(app)
|
||||
with TestClient(app) as client:
|
||||
response = client.delete("/probe/t1")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["ok"] is True
|
||||
|
||||
|
||||
def test_workspace_id_passed_to_check_access():
|
||||
"""check_access receives the contextvar workspace_id as the 3rd positional."""
|
||||
app = make_authed_test_app(
|
||||
workspace_factory=_make_workspace("ws-alpha"),
|
||||
owner_check_passes=True,
|
||||
)
|
||||
_mount_routes(app)
|
||||
with TestClient(app) as client:
|
||||
client.delete("/probe/t1")
|
||||
|
||||
call = app.state.thread_store.check_access.call_args
|
||||
assert call is not None
|
||||
args = call.args
|
||||
# (thread_id, user_id, workspace_id)
|
||||
assert args[0] == "t1"
|
||||
assert args[2] == "ws-alpha"
|
||||
|
||||
|
||||
@pytest.mark.no_auto_workspace
|
||||
def test_no_workspace_in_context_falls_back_to_default():
|
||||
"""No-auth dev mode (no workspace contextvar) uses DEFAULT_WORKSPACE_ID."""
|
||||
app = make_authed_test_app(workspace_factory=None, owner_check_passes=True)
|
||||
_mount_routes(app)
|
||||
with TestClient(app) as client:
|
||||
client.delete("/probe/t1")
|
||||
|
||||
args = app.state.thread_store.check_access.call_args.args
|
||||
assert args[2] == "default"
|
||||
|
||||
|
||||
def test_get_route_also_uses_workspace_id():
|
||||
"""Read-style routes (require_existing=False) also pass workspace_id through."""
|
||||
app = make_authed_test_app(
|
||||
workspace_factory=_make_workspace("ws-beta"),
|
||||
owner_check_passes=True,
|
||||
)
|
||||
_mount_routes(app)
|
||||
with TestClient(app) as client:
|
||||
client.get("/probe/t-read")
|
||||
args = app.state.thread_store.check_access.call_args.args
|
||||
assert args[2] == "ws-beta"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _reset_ws():
|
||||
"""Helper for direct-call paths that mutate the contextvar."""
|
||||
tokens: list = []
|
||||
yield lambda wid: tokens.append(set_current_workspace(ActiveWorkspace(id=wid, role="owner")))
|
||||
for token in reversed(tokens):
|
||||
reset_current_workspace(token)
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Tests for Run/Feedback/RunEvent repository workspace_id filtering (PR6 T6.5)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.runtime.workspace_context import (
|
||||
reset_current_workspace,
|
||||
set_current_workspace,
|
||||
)
|
||||
|
||||
|
||||
async def _init_engine(tmp_path, *, workspaces: tuple[str, ...] = ()):
|
||||
from deerflow.persistence.engine import get_session_factory, init_engine
|
||||
|
||||
url = f"sqlite+aiosqlite:///{tmp_path / 'test.db'}"
|
||||
await init_engine("sqlite", url=url, sqlite_dir=str(tmp_path))
|
||||
for wid in workspaces:
|
||||
await _seed_workspace(wid)
|
||||
return get_session_factory()
|
||||
|
||||
|
||||
async def _seed_workspace(wid: str) -> None:
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.workspace.model import WorkspaceRow
|
||||
|
||||
factory = get_session_factory()
|
||||
async with factory() as session:
|
||||
if await session.get(WorkspaceRow, wid) is not None:
|
||||
return
|
||||
now = datetime.now(UTC)
|
||||
session.add(
|
||||
WorkspaceRow(
|
||||
id=wid,
|
||||
name=f"WS {wid}",
|
||||
slug=wid.replace("_", "-")[:32],
|
||||
status="active",
|
||||
owner_id="test-user-autouse",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _cleanup():
|
||||
from deerflow.persistence.engine import close_engine
|
||||
|
||||
await close_engine()
|
||||
|
||||
|
||||
def _use_workspace(wid: str):
|
||||
return set_current_workspace(SimpleNamespace(id=wid, role="owner"))
|
||||
|
||||
|
||||
class TestRunRepositoryWorkspace:
|
||||
@pytest.mark.anyio
|
||||
async def test_put_records_workspace_id(self, tmp_path):
|
||||
from deerflow.persistence.run import RunRepository
|
||||
|
||||
sf = await _init_engine(tmp_path, workspaces=("ws-alpha",))
|
||||
repo = RunRepository(sf)
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.put("r1", thread_id="t1", user_id="alice")
|
||||
record = await repo.get("r1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
assert record["workspace_id"] == "ws-alpha"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_get_filters_cross_workspace(self, tmp_path):
|
||||
from deerflow.persistence.run import RunRepository
|
||||
|
||||
sf = await _init_engine(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
repo = RunRepository(sf)
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.put("r1", thread_id="t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
token = _use_workspace("ws-beta")
|
||||
try:
|
||||
assert await repo.get("r1", user_id="alice") is None
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_list_by_thread_filters_workspace(self, tmp_path):
|
||||
from deerflow.persistence.run import RunRepository
|
||||
|
||||
sf = await _init_engine(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
repo = RunRepository(sf)
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.put("r1", thread_id="t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
token = _use_workspace("ws-beta")
|
||||
try:
|
||||
await repo.put("r2", thread_id="t1", user_id="alice")
|
||||
rows = await repo.list_by_thread("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
assert [r["run_id"] for r in rows] == ["r2"]
|
||||
|
||||
|
||||
class TestFeedbackRepositoryWorkspace:
|
||||
@pytest.mark.anyio
|
||||
async def test_create_records_workspace_id(self, tmp_path):
|
||||
from deerflow.persistence.feedback.sql import FeedbackRepository
|
||||
|
||||
sf = await _init_engine(tmp_path, workspaces=("ws-alpha",))
|
||||
repo = FeedbackRepository(sf)
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
row = await repo.create(run_id="r1", thread_id="t1", rating=1, user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
assert row["workspace_id"] == "ws-alpha"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_list_by_thread_filters_workspace(self, tmp_path):
|
||||
from deerflow.persistence.feedback.sql import FeedbackRepository
|
||||
|
||||
sf = await _init_engine(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
repo = FeedbackRepository(sf)
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.create(run_id="r1", thread_id="t1", rating=1, user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
token = _use_workspace("ws-beta")
|
||||
try:
|
||||
rows = await repo.list_by_thread("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
assert rows == []
|
||||
|
||||
|
||||
class TestRunEventStoreWorkspace:
|
||||
@pytest.mark.anyio
|
||||
async def test_put_records_workspace_id(self, tmp_path):
|
||||
from deerflow.runtime.events.store.db import DbRunEventStore
|
||||
|
||||
sf = await _init_engine(tmp_path, workspaces=("ws-alpha",))
|
||||
store = DbRunEventStore(sf)
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
row = await store.put(thread_id="t1", run_id="r1", event_type="msg", category="message", content="hi")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
assert row["workspace_id"] == "ws-alpha"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_list_messages_filters_cross_workspace(self, tmp_path):
|
||||
from deerflow.runtime.events.store.db import DbRunEventStore
|
||||
|
||||
sf = await _init_engine(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
store = DbRunEventStore(sf)
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await store.put(thread_id="t1", run_id="r1", event_type="msg", category="message", content="from-alpha")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
token = _use_workspace("ws-beta")
|
||||
try:
|
||||
rows = await store.list_messages("t1", user_id="test-user-autouse")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
assert rows == []
|
||||
@@ -0,0 +1,88 @@
|
||||
"""PR6 T6.11 — ThreadDataMiddleware writes under workspace layout."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.agents.middlewares.thread_data_middleware import ThreadDataMiddleware
|
||||
from deerflow.config.paths import Paths
|
||||
from deerflow.runtime.workspace_context import (
|
||||
reset_current_workspace,
|
||||
set_current_workspace,
|
||||
)
|
||||
|
||||
|
||||
class _FakeRuntime:
|
||||
def __init__(self, *, thread_id: str = "t1", run_id: str = "r1"):
|
||||
self.context = {"thread_id": thread_id, "run_id": run_id}
|
||||
|
||||
|
||||
def test_paths_resolve_under_workspace(tmp_path):
|
||||
paths = Paths(tmp_path)
|
||||
middleware = ThreadDataMiddleware(base_dir=str(tmp_path), lazy_init=True)
|
||||
middleware._paths = paths
|
||||
|
||||
token = set_current_workspace(SimpleNamespace(id="ws-alpha", role="owner"))
|
||||
try:
|
||||
out = middleware.before_agent({"messages": []}, _FakeRuntime())
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
expected_root = tmp_path / "workspaces" / "ws-alpha" / "threads" / "t1" / "user-data"
|
||||
assert out["thread_data"]["workspace_path"] == str(expected_root / "workspace")
|
||||
assert out["thread_data"]["uploads_path"] == str(expected_root / "uploads")
|
||||
assert out["thread_data"]["outputs_path"] == str(expected_root / "outputs")
|
||||
assert out["thread_data"]["workspace_id"] == "ws-alpha"
|
||||
|
||||
|
||||
@pytest.mark.no_auto_workspace
|
||||
def test_falls_back_to_default_workspace(tmp_path):
|
||||
"""Without a workspace contextvar, `get_effective_workspace_id` returns 'default'."""
|
||||
paths = Paths(tmp_path)
|
||||
middleware = ThreadDataMiddleware(base_dir=str(tmp_path), lazy_init=True)
|
||||
middleware._paths = paths
|
||||
|
||||
out = middleware.before_agent({"messages": []}, _FakeRuntime())
|
||||
|
||||
expected_root = tmp_path / "workspaces" / "default" / "threads" / "t1" / "user-data"
|
||||
assert out["thread_data"]["workspace_path"] == str(expected_root / "workspace")
|
||||
assert out["thread_data"]["workspace_id"] == "default"
|
||||
|
||||
|
||||
def test_eager_creates_directories_under_workspace(tmp_path):
|
||||
paths = Paths(tmp_path)
|
||||
middleware = ThreadDataMiddleware(base_dir=str(tmp_path), lazy_init=False)
|
||||
middleware._paths = paths
|
||||
|
||||
token = set_current_workspace(SimpleNamespace(id="ws-beta", role="owner"))
|
||||
try:
|
||||
middleware.before_agent({"messages": []}, _FakeRuntime(thread_id="t2"))
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
root = tmp_path / "workspaces" / "ws-beta" / "threads" / "t2" / "user-data"
|
||||
assert (root / "workspace").is_dir()
|
||||
assert (root / "uploads").is_dir()
|
||||
assert (root / "outputs").is_dir()
|
||||
|
||||
|
||||
def test_get_config_fallback_still_workspace_scoped(tmp_path):
|
||||
"""Thread_id resolution via LangGraph config still routes through workspace."""
|
||||
paths = Paths(tmp_path)
|
||||
middleware = ThreadDataMiddleware(base_dir=str(tmp_path), lazy_init=True)
|
||||
middleware._paths = paths
|
||||
|
||||
class _Runtime:
|
||||
context: dict = {}
|
||||
|
||||
with patch("deerflow.agents.middlewares.thread_data_middleware.get_config", return_value={"configurable": {"thread_id": "t-cfg"}}):
|
||||
token = set_current_workspace(SimpleNamespace(id="ws-gamma", role="owner"))
|
||||
try:
|
||||
out = middleware.before_agent({"messages": []}, _Runtime())
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
assert "workspaces/ws-gamma/threads/t-cfg/user-data/workspace" in out["thread_data"]["workspace_path"]
|
||||
@@ -64,21 +64,21 @@ class TestThreadMetaRepository:
|
||||
@pytest.mark.anyio
|
||||
async def test_check_access_no_record_allows(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path)
|
||||
assert await repo.check_access("unknown", "user1") is True
|
||||
assert await repo.check_access("unknown", "user1", "test-workspace-autouse") is True
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_check_access_owner_matches(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path)
|
||||
await repo.create("t1", user_id="user1")
|
||||
assert await repo.check_access("t1", "user1") is True
|
||||
assert await repo.check_access("t1", "user1", "test-workspace-autouse") is True
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_check_access_owner_mismatch(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path)
|
||||
await repo.create("t1", user_id="user1")
|
||||
assert await repo.check_access("t1", "user2") is False
|
||||
assert await repo.check_access("t1", "user2", "test-workspace-autouse") is False
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
@@ -87,7 +87,7 @@ class TestThreadMetaRepository:
|
||||
# Explicit user_id=None to bypass the new AUTO default that
|
||||
# would otherwise pick up the test user from the autouse fixture.
|
||||
await repo.create("t1", user_id=None)
|
||||
assert await repo.check_access("t1", "anyone") is True
|
||||
assert await repo.check_access("t1", "anyone", "test-workspace-autouse") is True
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
@@ -99,21 +99,21 @@ class TestThreadMetaRepository:
|
||||
caller "claim" it as untracked. The strict mode demands a row.
|
||||
"""
|
||||
repo = await _make_repo(tmp_path)
|
||||
assert await repo.check_access("never-existed", "user1", require_existing=True) is False
|
||||
assert await repo.check_access("never-existed", "user1", "test-workspace-autouse", require_existing=True) is False
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_check_access_strict_owner_match_allowed(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path)
|
||||
await repo.create("t1", user_id="user1")
|
||||
assert await repo.check_access("t1", "user1", require_existing=True) is True
|
||||
assert await repo.check_access("t1", "user1", "test-workspace-autouse", require_existing=True) is True
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_check_access_strict_owner_mismatch_denied(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path)
|
||||
await repo.create("t1", user_id="user1")
|
||||
assert await repo.check_access("t1", "user2", require_existing=True) is False
|
||||
assert await repo.check_access("t1", "user2", "test-workspace-autouse", require_existing=True) is False
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
@@ -126,7 +126,7 @@ class TestThreadMetaRepository:
|
||||
"""
|
||||
repo = await _make_repo(tmp_path)
|
||||
await repo.create("t1", user_id=None)
|
||||
assert await repo.check_access("t1", "anyone", require_existing=True) is True
|
||||
assert await repo.check_access("t1", "anyone", "test-workspace-autouse", require_existing=True) is True
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
"""Tests for ThreadMetaRepository workspace_id filtering (PR6 T6.1-T6.4).
|
||||
|
||||
The repository's three-state ``workspace_id`` semantics mirror ``user_id``:
|
||||
|
||||
- :data:`AUTO` (default): read from workspace contextvar
|
||||
- Explicit ``str``: use the provided id
|
||||
- Explicit ``None``: bypass workspace filter (migration / CLI)
|
||||
|
||||
Cross-workspace access (a thread in workspace A queried with workspace B)
|
||||
must return ``None``, never the row. This is the load-bearing isolation
|
||||
boundary tested here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.persistence.thread_meta import ThreadMetaRepository
|
||||
from deerflow.runtime.workspace_context import (
|
||||
reset_current_workspace,
|
||||
set_current_workspace,
|
||||
)
|
||||
|
||||
|
||||
async def _seed_workspace(wid: str, *, owner_id: str = "test-user-autouse") -> None:
|
||||
"""Insert a workspace row so threads_meta.workspace_id FK resolves."""
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.workspace.model import WorkspaceRow
|
||||
|
||||
factory = get_session_factory()
|
||||
async with factory() as session:
|
||||
existing = await session.get(WorkspaceRow, wid)
|
||||
if existing is not None:
|
||||
return
|
||||
now = datetime.now(UTC)
|
||||
session.add(
|
||||
WorkspaceRow(
|
||||
id=wid,
|
||||
name=f"WS {wid}",
|
||||
slug=wid.replace("_", "-")[:32],
|
||||
status="active",
|
||||
owner_id=owner_id,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _make_repo(tmp_path, *, workspaces: tuple[str, ...] = ()):
|
||||
from deerflow.persistence.engine import get_session_factory, init_engine
|
||||
|
||||
url = f"sqlite+aiosqlite:///{tmp_path / 'test.db'}"
|
||||
await init_engine("sqlite", url=url, sqlite_dir=str(tmp_path))
|
||||
for wid in workspaces:
|
||||
await _seed_workspace(wid)
|
||||
return ThreadMetaRepository(get_session_factory())
|
||||
|
||||
|
||||
async def _cleanup():
|
||||
from deerflow.persistence.engine import close_engine
|
||||
|
||||
await close_engine()
|
||||
|
||||
|
||||
def _use_workspace(wid: str, role: str = "owner"):
|
||||
"""Replace the autouse workspace contextvar inside a single test."""
|
||||
return set_current_workspace(SimpleNamespace(id=wid, role=role))
|
||||
|
||||
|
||||
class TestCreateWorkspace:
|
||||
@pytest.mark.anyio
|
||||
async def test_create_uses_workspace_context(self, tmp_path):
|
||||
"""AUTO sentinel pulls workspace_id from the contextvar."""
|
||||
repo = await _make_repo(tmp_path, workspaces=("ws-alpha",))
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
record = await repo.create("t1")
|
||||
assert record["workspace_id"] == "ws-alpha"
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_create_explicit_workspace_overrides_context(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
record = await repo.create("t1", workspace_id="ws-beta")
|
||||
assert record["workspace_id"] == "ws-beta"
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_create_workspace_none_rejected_by_orm(self, tmp_path):
|
||||
"""After T5.11 (ORM nullable=False) explicit None creates fail at the DB layer.
|
||||
|
||||
Pre-PR6 the migration scripts relied on `workspace_id=None` to insert
|
||||
orphan rows; that use is now restricted to **read** paths (filter
|
||||
bypass). Writes must always carry a workspace.
|
||||
"""
|
||||
import sqlalchemy
|
||||
|
||||
repo = await _make_repo(tmp_path)
|
||||
with pytest.raises((sqlalchemy.exc.IntegrityError, sqlalchemy.exc.DBAPIError)):
|
||||
await repo.create("t1", workspace_id=None)
|
||||
await _cleanup()
|
||||
|
||||
|
||||
class TestGetWorkspace:
|
||||
@pytest.mark.anyio
|
||||
async def test_get_filters_by_workspace(self, tmp_path):
|
||||
"""Cross-workspace get returns None even when user_id matches."""
|
||||
repo = await _make_repo(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.create("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
token = _use_workspace("ws-beta")
|
||||
try:
|
||||
assert await repo.get("t1", user_id="alice") is None
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_get_returns_row_in_same_workspace(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path, workspaces=("ws-alpha",))
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.create("t1", user_id="alice")
|
||||
record = await repo.get("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
assert record is not None
|
||||
assert record["thread_id"] == "t1"
|
||||
assert record["workspace_id"] == "ws-alpha"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_get_workspace_none_bypasses_filter(self, tmp_path):
|
||||
"""Explicit workspace_id=None lets migration scripts see any row."""
|
||||
repo = await _make_repo(tmp_path, workspaces=("ws-alpha",))
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.create("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
token = _use_workspace("ws-beta")
|
||||
try:
|
||||
assert await repo.get("t1", user_id=None, workspace_id=None) is not None
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
|
||||
|
||||
class TestSearchUpdateDeleteWorkspace:
|
||||
@pytest.mark.anyio
|
||||
async def test_search_only_returns_current_workspace(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.create("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
token = _use_workspace("ws-beta")
|
||||
try:
|
||||
await repo.create("t2", user_id="alice")
|
||||
rows = await repo.search(user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
ids = {r["thread_id"] for r in rows}
|
||||
assert ids == {"t2"}
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_update_status_blocked_across_workspace(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.create("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
token = _use_workspace("ws-beta")
|
||||
try:
|
||||
await repo.update_status("t1", "busy", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
row = await repo.get("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
assert row["status"] == "idle"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_update_display_name_blocked_across_workspace(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.create("t1", user_id="alice", display_name="A")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
token = _use_workspace("ws-beta")
|
||||
try:
|
||||
await repo.update_display_name("t1", "B", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
row = await repo.get("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
assert row["display_name"] == "A"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_update_metadata_blocked_across_workspace(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.create("t1", user_id="alice", metadata={"k": "alpha"})
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
token = _use_workspace("ws-beta")
|
||||
try:
|
||||
await repo.update_metadata("t1", {"k": "beta"}, user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
row = await repo.get("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
assert row["metadata"] == {"k": "alpha"}
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_check_access_cross_workspace_false(self, tmp_path):
|
||||
"""`check_access` returns False for cross-workspace, even with matching user_id."""
|
||||
repo = await _make_repo(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.create("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
try:
|
||||
assert await repo.check_access("t1", "alice", "ws-beta") is False
|
||||
assert await repo.check_access("t1", "alice", "ws-alpha") is True
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_check_access_strict_cross_workspace_false(self, tmp_path):
|
||||
"""require_existing=True path also denies cross-workspace."""
|
||||
repo = await _make_repo(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.create("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
try:
|
||||
assert await repo.check_access("t1", "alice", "ws-beta", require_existing=True) is False
|
||||
assert await repo.check_access("t1", "alice", "ws-alpha", require_existing=True) is True
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_delete_blocked_across_workspace(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path, workspaces=("ws-alpha", "ws-beta"))
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
await repo.create("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
token = _use_workspace("ws-beta")
|
||||
try:
|
||||
await repo.delete("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
token = _use_workspace("ws-alpha")
|
||||
try:
|
||||
row = await repo.get("t1", user_id="alice")
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
await _cleanup()
|
||||
assert row is not None and row["thread_id"] == "t1"
|
||||
@@ -0,0 +1,59 @@
|
||||
"""PR6 T6.7 — POST /api/threads writes workspace_id from contextvar.
|
||||
|
||||
The `routers/threads.py:create_thread` path delegates to
|
||||
``ThreadMetaStore.create`` *without* an explicit ``workspace_id`` — it
|
||||
relies on the AUTO sentinel pulling the value from the active workspace
|
||||
contextvar that AuthMiddleware (or the test stub) sets. This test
|
||||
covers the integration through the FastAPI TestClient stack: post a
|
||||
thread under workspace A, then verify the persisted record carries
|
||||
``workspace_id="ws-alpha"``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
from _router_auth_helpers import make_authed_test_app
|
||||
from fastapi.testclient import TestClient
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.store.memory import InMemoryStore
|
||||
|
||||
from app.gateway.auth.models import ActiveWorkspace
|
||||
from app.gateway.routers import threads
|
||||
from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore
|
||||
|
||||
|
||||
def _workspace_factory(wid: str) -> Callable[[], ActiveWorkspace]:
|
||||
def _factory() -> ActiveWorkspace:
|
||||
return ActiveWorkspace(id=wid, role="owner")
|
||||
|
||||
return _factory
|
||||
|
||||
|
||||
def _build_app(workspace_id: str):
|
||||
app = make_authed_test_app(workspace_factory=_workspace_factory(workspace_id))
|
||||
store = InMemoryStore()
|
||||
checkpointer = InMemorySaver()
|
||||
app.state.store = store
|
||||
app.state.checkpointer = checkpointer
|
||||
app.state.thread_store = MemoryThreadMetaStore(store)
|
||||
app.include_router(threads.router)
|
||||
return app, store
|
||||
|
||||
|
||||
def test_post_thread_stamps_workspace_id_from_contextvar():
|
||||
app, store = _build_app("ws-alpha")
|
||||
with TestClient(app) as client:
|
||||
response = client.post("/api/threads", json={"thread_id": "t1", "metadata": {}})
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
item = store.get(("threads",), "t1")
|
||||
assert item is not None
|
||||
assert item.value["workspace_id"] == "ws-alpha"
|
||||
|
||||
|
||||
def test_post_thread_under_different_workspace():
|
||||
app, store = _build_app("ws-beta")
|
||||
with TestClient(app) as client:
|
||||
client.post("/api/threads", json={"thread_id": "t2", "metadata": {}})
|
||||
assert store.get(("threads",), "t2").value["workspace_id"] == "ws-beta"
|
||||
@@ -27,21 +27,21 @@ class _PermissiveThreadMetaStore(MemoryThreadMetaStore):
|
||||
timestamp wire format.
|
||||
"""
|
||||
|
||||
async def _get_owned_record(self, thread_id, user_id, method_name): # type: ignore[override]
|
||||
async def _get_owned_record(self, thread_id, user_id, workspace_id, method_name): # type: ignore[override]
|
||||
item = await self._store.aget(THREADS_NS, thread_id)
|
||||
return dict(item.value) if item is not None else None
|
||||
|
||||
async def check_access(self, thread_id, user_id, *, require_existing=False): # type: ignore[override]
|
||||
async def check_access(self, thread_id, user_id, workspace_id, *, require_existing=False): # type: ignore[override]
|
||||
item = await self._store.aget(THREADS_NS, thread_id)
|
||||
if item is None:
|
||||
return not require_existing
|
||||
return True
|
||||
|
||||
async def create(self, thread_id, *, assistant_id=None, user_id=None, display_name=None, metadata=None): # type: ignore[override]
|
||||
return await super().create(thread_id, assistant_id=assistant_id, user_id=None, display_name=display_name, metadata=metadata)
|
||||
async def create(self, thread_id, *, assistant_id=None, user_id=None, workspace_id=None, display_name=None, metadata=None): # type: ignore[override]
|
||||
return await super().create(thread_id, assistant_id=assistant_id, user_id=None, workspace_id=None, display_name=display_name, metadata=metadata)
|
||||
|
||||
async def search(self, *, metadata=None, status=None, limit=100, offset=0, user_id=None): # type: ignore[override]
|
||||
return await super().search(metadata=metadata, status=status, limit=limit, offset=offset, user_id=None)
|
||||
async def search(self, *, metadata=None, status=None, limit=100, offset=0, user_id=None, workspace_id=None): # type: ignore[override]
|
||||
return await super().search(metadata=metadata, status=status, limit=limit, offset=offset, user_id=None, workspace_id=None)
|
||||
|
||||
|
||||
def _build_thread_app() -> tuple[FastAPI, InMemoryStore, InMemorySaver]:
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Boundary check: only allowlisted modules may directly import LangGraph
|
||||
checkpoint/saver clients.
|
||||
|
||||
Direct use of the LangGraph checkpoint API anywhere outside the gateway thread
|
||||
plumbing and the harness checkpointer factory is a workspace-isolation hazard:
|
||||
arbitrary code paths could otherwise reach across threads/workspaces by
|
||||
constructing their own savers. PR7 enforces this with an AST static scan.
|
||||
|
||||
Imports inside ``if TYPE_CHECKING:`` blocks are intentionally ignored — they
|
||||
never execute at runtime and therefore cannot bypass the boundary.
|
||||
|
||||
Allowlist lives in ``tests/boundary_allowlist.toml`` as a plain list of paths
|
||||
relative to ``backend/``. Adding a new legitimate importer means appending a
|
||||
line there in the same PR that introduces the import.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
BACKEND_ROOT = Path(__file__).parent.parent # backend/
|
||||
ALLOWLIST_FILE = Path(__file__).parent / "boundary_allowlist.toml"
|
||||
|
||||
# Match any submodule of these top-level packages.
|
||||
TARGET_MODULE_PREFIXES: tuple[str, ...] = (
|
||||
"langgraph.checkpoint",
|
||||
"langgraph_checkpoint_postgres",
|
||||
"langgraph_checkpoint_sqlite",
|
||||
)
|
||||
|
||||
# Directories under backend/ that are not part of the running app.
|
||||
EXCLUDED_TOP_LEVEL = ("tests", ".venv", "build", "dist", "docs", ".pytest_cache", "node_modules")
|
||||
|
||||
|
||||
def _matches_target(name: str) -> bool:
|
||||
return any(name == prefix or name.startswith(prefix + ".") for prefix in TARGET_MODULE_PREFIXES)
|
||||
|
||||
|
||||
def _build_parent_map(tree: ast.AST) -> dict[int, ast.AST]:
|
||||
parents: dict[int, ast.AST] = {}
|
||||
for parent in ast.walk(tree):
|
||||
for child in ast.iter_child_nodes(parent):
|
||||
parents[id(child)] = parent
|
||||
return parents
|
||||
|
||||
|
||||
def _is_type_checking_test(test: ast.expr) -> bool:
|
||||
"""Return True for ``TYPE_CHECKING`` or ``typing.TYPE_CHECKING`` conditions."""
|
||||
if isinstance(test, ast.Name) and test.id == "TYPE_CHECKING":
|
||||
return True
|
||||
if isinstance(test, ast.Attribute) and test.attr == "TYPE_CHECKING":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _inside_type_checking(node: ast.AST, parents: dict[int, ast.AST]) -> bool:
|
||||
current = parents.get(id(node))
|
||||
while current is not None:
|
||||
if isinstance(current, ast.If) and _is_type_checking_test(current.test):
|
||||
return True
|
||||
current = parents.get(id(current))
|
||||
return False
|
||||
|
||||
|
||||
def collect_runtime_checkpoint_imports(filepath: Path) -> list[tuple[int, str]]:
|
||||
"""Return ``(lineno, module_path)`` for every runtime import that targets a banned module."""
|
||||
source = filepath.read_text(encoding="utf-8")
|
||||
try:
|
||||
tree = ast.parse(source, filename=str(filepath))
|
||||
except SyntaxError:
|
||||
return []
|
||||
|
||||
parents = _build_parent_map(tree)
|
||||
hits: list[tuple[int, str]] = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
module = node.module or ""
|
||||
if _matches_target(module) and not _inside_type_checking(node, parents):
|
||||
hits.append((node.lineno, module))
|
||||
elif isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
if _matches_target(alias.name) and not _inside_type_checking(node, parents):
|
||||
hits.append((node.lineno, alias.name))
|
||||
return hits
|
||||
|
||||
|
||||
def _iter_backend_py_files() -> list[Path]:
|
||||
candidates: list[Path] = []
|
||||
for path in sorted(BACKEND_ROOT.rglob("*.py")):
|
||||
rel_parts = path.relative_to(BACKEND_ROOT).parts
|
||||
if rel_parts and rel_parts[0] in EXCLUDED_TOP_LEVEL:
|
||||
continue
|
||||
if "__pycache__" in rel_parts:
|
||||
continue
|
||||
candidates.append(path)
|
||||
return candidates
|
||||
|
||||
|
||||
def _load_allowlist() -> set[str]:
|
||||
data = tomllib.loads(ALLOWLIST_FILE.read_text(encoding="utf-8"))
|
||||
raw = data.get("langgraph_checkpoint_importers", [])
|
||||
return set(raw)
|
||||
|
||||
|
||||
def scan_violations(allowlist: set[str]) -> list[str]:
|
||||
"""Return formatted violation lines (one per banned import outside the allowlist)."""
|
||||
violations: list[str] = []
|
||||
for py in _iter_backend_py_files():
|
||||
rel = py.relative_to(BACKEND_ROOT).as_posix()
|
||||
for lineno, module in collect_runtime_checkpoint_imports(py):
|
||||
if rel in allowlist:
|
||||
continue
|
||||
violations.append(f" {rel}:{lineno} imports {module}")
|
||||
return violations
|
||||
|
||||
|
||||
def test_only_allowlisted_modules_import_langgraph_checkpoint() -> None:
|
||||
allowlist = _load_allowlist()
|
||||
violations = scan_violations(allowlist)
|
||||
assert not violations, (
|
||||
"Unauthorized direct imports of langgraph.checkpoint.* detected. "
|
||||
"Either route the access through `app.gateway.deps.get_checkpointer` "
|
||||
"or, if this is a legitimate new importer, add the path to "
|
||||
"tests/boundary_allowlist.toml in the same PR.\n" + "\n".join(violations)
|
||||
)
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Tests for runtime.workspace_context — workspace contextvar semantics.
|
||||
|
||||
Mirrors :mod:`test_user_context` but for the workspace contextvar
|
||||
introduced in Stage 0 PR3. No autouse workspace fixture exists yet
|
||||
(PR4 will add it together with the AuthMiddleware injection), so these
|
||||
tests run against a clean contextvar.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.runtime.workspace_context import (
|
||||
AUTO,
|
||||
DEFAULT_WORKSPACE_ID,
|
||||
CurrentWorkspace,
|
||||
get_current_workspace,
|
||||
get_effective_workspace_id,
|
||||
require_current_workspace,
|
||||
reset_current_workspace,
|
||||
resolve_workspace_id,
|
||||
set_current_workspace,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_current_workspace / require_current_workspace / set+reset round-trip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.no_auto_workspace
|
||||
def test_default_is_none():
|
||||
"""Before any set, contextvar returns None."""
|
||||
assert get_current_workspace() is None
|
||||
|
||||
|
||||
@pytest.mark.no_auto_workspace
|
||||
def test_set_and_reset_roundtrip():
|
||||
"""set_current_workspace returns a token that reset restores."""
|
||||
workspace = SimpleNamespace(id="ws-1", role="owner")
|
||||
token = set_current_workspace(workspace)
|
||||
try:
|
||||
assert get_current_workspace() is workspace
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
assert get_current_workspace() is None
|
||||
|
||||
|
||||
@pytest.mark.no_auto_workspace
|
||||
def test_require_current_workspace_raises_when_unset():
|
||||
"""require_current_workspace raises RuntimeError if contextvar is unset."""
|
||||
assert get_current_workspace() is None
|
||||
with pytest.raises(RuntimeError, match="without workspace context"):
|
||||
require_current_workspace()
|
||||
|
||||
|
||||
def test_require_current_workspace_returns_workspace_when_set():
|
||||
"""require_current_workspace returns the workspace when contextvar is set."""
|
||||
workspace = SimpleNamespace(id="ws-2", role="admin")
|
||||
token = set_current_workspace(workspace)
|
||||
try:
|
||||
assert require_current_workspace() is workspace
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CurrentWorkspace Protocol — must require BOTH .id and .role
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_protocol_accepts_id_and_role():
|
||||
"""CurrentWorkspace is satisfied by any object with .id and .role."""
|
||||
workspace = SimpleNamespace(id="ws-3", role="member")
|
||||
assert isinstance(workspace, CurrentWorkspace)
|
||||
|
||||
|
||||
def test_protocol_rejects_missing_role():
|
||||
"""An object with only .id (no .role) is NOT a workspace."""
|
||||
user_shaped = SimpleNamespace(id="ws-4")
|
||||
assert not isinstance(user_shaped, CurrentWorkspace)
|
||||
|
||||
|
||||
def test_protocol_rejects_no_id():
|
||||
"""An object without .id does not satisfy CurrentWorkspace."""
|
||||
not_a_workspace = SimpleNamespace(role="owner")
|
||||
assert not isinstance(not_a_workspace, CurrentWorkspace)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_effective_workspace_id / DEFAULT_WORKSPACE_ID tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_default_workspace_id_is_default():
|
||||
assert DEFAULT_WORKSPACE_ID == "default"
|
||||
|
||||
|
||||
@pytest.mark.no_auto_workspace
|
||||
def test_effective_workspace_id_returns_default_when_no_workspace():
|
||||
"""No workspace in context -> fallback to DEFAULT_WORKSPACE_ID."""
|
||||
assert get_effective_workspace_id() == "default"
|
||||
|
||||
|
||||
def test_effective_workspace_id_returns_workspace_id_when_set():
|
||||
workspace = SimpleNamespace(id="ws-abc-123", role="owner")
|
||||
token = set_current_workspace(workspace)
|
||||
try:
|
||||
assert get_effective_workspace_id() == "ws-abc-123"
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
|
||||
def test_effective_workspace_id_coerces_to_str():
|
||||
"""workspace.id might be a UUID object; must come back as str."""
|
||||
wid = uuid.uuid4()
|
||||
workspace = SimpleNamespace(id=wid, role="owner")
|
||||
token = set_current_workspace(workspace)
|
||||
try:
|
||||
assert get_effective_workspace_id() == str(wid)
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_workspace_id three-state semantics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_auto_reads_from_contextvar():
|
||||
workspace = SimpleNamespace(id="ws-resolve-1", role="owner")
|
||||
token = set_current_workspace(workspace)
|
||||
try:
|
||||
assert resolve_workspace_id(AUTO) == "ws-resolve-1"
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
|
||||
@pytest.mark.no_auto_workspace
|
||||
def test_resolve_auto_raises_when_unset():
|
||||
assert get_current_workspace() is None
|
||||
with pytest.raises(RuntimeError, match="workspace_id=AUTO but no workspace"):
|
||||
resolve_workspace_id(AUTO, method_name="TestRepo.search")
|
||||
|
||||
|
||||
def test_resolve_explicit_str_overrides_contextvar():
|
||||
workspace = SimpleNamespace(id="ws-ctx", role="owner")
|
||||
token = set_current_workspace(workspace)
|
||||
try:
|
||||
# Explicit value beats contextvar — admin override / test path.
|
||||
assert resolve_workspace_id("ws-explicit") == "ws-explicit"
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
|
||||
def test_resolve_explicit_none_means_no_filter():
|
||||
workspace = SimpleNamespace(id="ws-ctx-2", role="owner")
|
||||
token = set_current_workspace(workspace)
|
||||
try:
|
||||
# Explicit None opts out of workspace filtering (migration scripts).
|
||||
assert resolve_workspace_id(None) is None
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
|
||||
def test_resolve_auto_coerces_uuid_to_str():
|
||||
"""resolve_workspace_id with AUTO returns str even if workspace.id is UUID."""
|
||||
wid = uuid.uuid4()
|
||||
workspace = SimpleNamespace(id=wid, role="owner")
|
||||
token = set_current_workspace(workspace)
|
||||
try:
|
||||
resolved = resolve_workspace_id(AUTO)
|
||||
assert resolved == str(wid)
|
||||
assert isinstance(resolved, str)
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
@@ -0,0 +1,127 @@
|
||||
"""PR6 T6.8 — cross-workspace isolation boundary (e2e).
|
||||
|
||||
Wires a ``MemoryThreadMetaStore`` (LangGraph BaseStore backed) into a
|
||||
stub-authed FastAPI app and asserts that any request from workspace B
|
||||
against a thread created in workspace A returns **404**, regardless of
|
||||
matching user_id. The cross-workspace block fires in
|
||||
``ThreadMetaStore.check_access`` and is converted to 404 by
|
||||
``@require_permission(owner_check=True)``.
|
||||
|
||||
We use the memory-backed implementation so the test stays in the test
|
||||
event loop end-to-end (the SQL engine binds to whatever loop owns
|
||||
``init_engine`` and the TestClient spins its own loop, which would
|
||||
collide). The decorator path it exercises is the same as production;
|
||||
the SQL repository's identical workspace filter is unit-covered by
|
||||
``test_thread_meta_workspace_filter.py``.
|
||||
|
||||
Covers:
|
||||
- ``GET /api/threads/{tid}`` — read (require_existing=False)
|
||||
- ``DELETE /api/threads/{tid}`` — destructive (require_existing=True)
|
||||
- ``PATCH /api/threads/{tid}`` — destructive write
|
||||
- positive control: same-workspace GET still succeeds
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
from _router_auth_helpers import make_authed_test_app
|
||||
from fastapi.testclient import TestClient
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.store.memory import InMemoryStore
|
||||
|
||||
from app.gateway.auth.models import ActiveWorkspace, User
|
||||
from app.gateway.routers import threads
|
||||
from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore
|
||||
from deerflow.runtime.workspace_context import (
|
||||
reset_current_workspace,
|
||||
set_current_workspace,
|
||||
)
|
||||
|
||||
|
||||
def _workspace_factory(wid: str) -> Callable[[], ActiveWorkspace]:
|
||||
def _factory() -> ActiveWorkspace:
|
||||
return ActiveWorkspace(id=wid, role="owner")
|
||||
|
||||
return _factory
|
||||
|
||||
|
||||
def _user_factory(uid: str) -> Callable[[], User]:
|
||||
def _factory() -> User:
|
||||
return User(email=f"{uid}@example.com", password_hash="x", system_role="user", id=uid)
|
||||
|
||||
return _factory
|
||||
|
||||
|
||||
def _seed_thread(store, *, thread_id: str, user_id: str, workspace_id: str) -> None:
|
||||
"""Insert a thread record under a specific workspace, bypassing the autouse fixture."""
|
||||
import asyncio
|
||||
|
||||
async def _go():
|
||||
meta_store = MemoryThreadMetaStore(store)
|
||||
token = set_current_workspace(SimpleNamespace(id=workspace_id, role="owner"))
|
||||
try:
|
||||
await meta_store.create(thread_id, user_id=user_id)
|
||||
finally:
|
||||
reset_current_workspace(token)
|
||||
|
||||
asyncio.run(_go())
|
||||
|
||||
|
||||
def _build_app(*, user_id: str, workspace_id: str):
|
||||
app = make_authed_test_app(
|
||||
user_factory=_user_factory(user_id),
|
||||
workspace_factory=_workspace_factory(workspace_id),
|
||||
override_user_contextvar=True,
|
||||
)
|
||||
store = InMemoryStore()
|
||||
app.state.store = store
|
||||
app.state.checkpointer = InMemorySaver()
|
||||
app.state.thread_store = MemoryThreadMetaStore(store)
|
||||
app.include_router(threads.router)
|
||||
return app, store
|
||||
|
||||
|
||||
def test_cross_workspace_get_returns_404():
|
||||
user_id = str(uuid4())
|
||||
app, store = _build_app(user_id=user_id, workspace_id="ws-beta")
|
||||
_seed_thread(store, thread_id="t1", user_id=user_id, workspace_id="ws-alpha")
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/api/threads/t1")
|
||||
assert response.status_code == 404
|
||||
assert "not found" in response.json()["detail"].lower()
|
||||
|
||||
|
||||
def test_cross_workspace_delete_returns_404():
|
||||
user_id = str(uuid4())
|
||||
app, store = _build_app(user_id=user_id, workspace_id="ws-beta")
|
||||
_seed_thread(store, thread_id="t1", user_id=user_id, workspace_id="ws-alpha")
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.delete("/api/threads/t1")
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_cross_workspace_patch_returns_404():
|
||||
user_id = str(uuid4())
|
||||
app, store = _build_app(user_id=user_id, workspace_id="ws-beta")
|
||||
_seed_thread(store, thread_id="t1", user_id=user_id, workspace_id="ws-alpha")
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.patch("/api/threads/t1", json={"metadata": {"k": "v"}})
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_same_workspace_get_succeeds():
|
||||
"""Positive control: when the workspace matches, the row is returned."""
|
||||
user_id = str(uuid4())
|
||||
app, store = _build_app(user_id=user_id, workspace_id="ws-alpha")
|
||||
_seed_thread(store, thread_id="t1", user_id=user_id, workspace_id="ws-alpha")
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/api/threads/t1")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["thread_id"] == "t1"
|
||||
@@ -0,0 +1,216 @@
|
||||
"""Tests for WorkspaceMembershipRepository (Stage 0 PR3 T3.7)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.workspace import WorkspaceRepository
|
||||
from deerflow.persistence.workspace_membership import (
|
||||
MembershipValidationError,
|
||||
WorkspaceMembershipRepository,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.anyio
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend() -> str:
|
||||
return "asyncio"
|
||||
|
||||
|
||||
async def _setup(tmp_path):
|
||||
"""Create both repos against a fresh SQLite DB."""
|
||||
from deerflow.persistence.engine import get_session_factory, init_engine
|
||||
|
||||
url = f"sqlite+aiosqlite:///{tmp_path / 'test.db'}"
|
||||
await init_engine("sqlite", url=url, sqlite_dir=str(tmp_path))
|
||||
sf = get_session_factory()
|
||||
return (
|
||||
WorkspaceRepository(sf),
|
||||
WorkspaceMembershipRepository(sf),
|
||||
sf,
|
||||
)
|
||||
|
||||
|
||||
async def _cleanup():
|
||||
from deerflow.persistence.engine import close_engine
|
||||
|
||||
await close_engine()
|
||||
|
||||
|
||||
async def _seed_user(sf, user_id: str, email: str) -> None:
|
||||
async with sf() as session:
|
||||
session.add(UserRow(id=user_id, email=email))
|
||||
await session.commit()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# add / remove smoke
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_add_then_get_role(tmp_path):
|
||||
ws_repo, m_repo, sf = await _setup(tmp_path)
|
||||
try:
|
||||
await _seed_user(sf, "u-1", "u1@example.com")
|
||||
ws = await ws_repo.create(name="W1", slug="w-1", owner_id="u-1")
|
||||
|
||||
added = await m_repo.add(workspace_id=ws["id"], user_id="u-1", role="owner")
|
||||
assert added["role"] == "owner"
|
||||
|
||||
role = await m_repo.get_role(workspace_id=ws["id"], user_id="u-1")
|
||||
assert role == "owner"
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
async def test_remove_returns_true_when_deleted_false_when_missing(tmp_path):
|
||||
ws_repo, m_repo, sf = await _setup(tmp_path)
|
||||
try:
|
||||
await _seed_user(sf, "u-1", "u1@example.com")
|
||||
ws = await ws_repo.create(name="W1", slug="rm-w", owner_id="u-1")
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="u-1", role="owner")
|
||||
|
||||
assert await m_repo.remove(workspace_id=ws["id"], user_id="u-1") is True
|
||||
# second remove of same row -> nothing to delete
|
||||
assert await m_repo.remove(workspace_id=ws["id"], user_id="u-1") is False
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# partial unique: exactly one owner per workspace
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_cannot_add_second_owner(tmp_path):
|
||||
"""Inserting a second role='owner' in the same workspace must raise IntegrityError."""
|
||||
ws_repo, m_repo, sf = await _setup(tmp_path)
|
||||
try:
|
||||
await _seed_user(sf, "u-1", "u1@example.com")
|
||||
await _seed_user(sf, "u-2", "u2@example.com")
|
||||
ws = await ws_repo.create(name="W", slug="one-owner", owner_id="u-1")
|
||||
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="u-1", role="owner")
|
||||
with pytest.raises(IntegrityError):
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="u-2", role="owner")
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
async def test_admin_and_member_dont_trigger_partial_unique(tmp_path):
|
||||
"""Multiple admin/member rows in one workspace are fine (Stage 2 forward compat)."""
|
||||
ws_repo, m_repo, sf = await _setup(tmp_path)
|
||||
try:
|
||||
await _seed_user(sf, "u-1", "u1@example.com")
|
||||
await _seed_user(sf, "u-2", "u2@example.com")
|
||||
await _seed_user(sf, "u-3", "u3@example.com")
|
||||
ws = await ws_repo.create(name="W", slug="multi-admin", owner_id="u-1")
|
||||
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="u-1", role="owner")
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="u-2", role="admin")
|
||||
# Second admin OK
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="u-3", role="admin")
|
||||
|
||||
members = await m_repo.list_by_workspace(workspace_id=ws["id"])
|
||||
assert len(members) == 3
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CASCADE: deleting a user wipes their memberships
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_cascade_delete_user_removes_memberships(tmp_path):
|
||||
"""FK ON DELETE CASCADE on user_id."""
|
||||
from sqlalchemy import delete
|
||||
|
||||
ws_repo, m_repo, sf = await _setup(tmp_path)
|
||||
try:
|
||||
await _seed_user(sf, "u-keep", "keep@example.com")
|
||||
await _seed_user(sf, "u-purge", "purge@example.com")
|
||||
ws = await ws_repo.create(name="W", slug="cascade", owner_id="u-keep")
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="u-keep", role="owner")
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="u-purge", role="admin")
|
||||
|
||||
async with sf() as session:
|
||||
await session.execute(delete(UserRow).where(UserRow.id == "u-purge"))
|
||||
await session.commit()
|
||||
|
||||
members = await m_repo.list_by_workspace(workspace_id=ws["id"])
|
||||
member_ids = [m["user_id"] for m in members]
|
||||
assert "u-purge" not in member_ids
|
||||
assert "u-keep" in member_ids
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# list_by_user ordering (most recent joined_at first)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_list_by_user_orders_recent_first(tmp_path):
|
||||
import asyncio
|
||||
|
||||
ws_repo, m_repo, sf = await _setup(tmp_path)
|
||||
try:
|
||||
await _seed_user(sf, "u-1", "u1@example.com")
|
||||
await _seed_user(sf, "u-2", "u2@example.com")
|
||||
|
||||
# u-1 owns workspace A
|
||||
ws_a = await ws_repo.create(name="A", slug="ord-a", owner_id="u-1")
|
||||
await m_repo.add(workspace_id=ws_a["id"], user_id="u-1", role="owner")
|
||||
await asyncio.sleep(0.01) # ensure distinct joined_at
|
||||
|
||||
# u-1 later joins workspace C as a member (u-2 owns it)
|
||||
ws_c = await ws_repo.create(name="C", slug="ord-c", owner_id="u-2")
|
||||
await m_repo.add(workspace_id=ws_c["id"], user_id="u-1", role="member")
|
||||
|
||||
memberships = await m_repo.list_by_user(user_id="u-1")
|
||||
slugs_in_order = [(m["workspace_id"], m["role"]) for m in memberships]
|
||||
# 'ws_c member' joined AFTER 'ws_a owner' → ws_c first
|
||||
assert slugs_in_order[0] == (ws_c["id"], "member")
|
||||
assert slugs_in_order[1] == (ws_a["id"], "owner")
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# role validation + change_role
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_add_rejects_unknown_role(tmp_path):
|
||||
ws_repo, m_repo, sf = await _setup(tmp_path)
|
||||
try:
|
||||
await _seed_user(sf, "u-1", "u1@example.com")
|
||||
ws = await ws_repo.create(name="W", slug="invrole", owner_id="u-1")
|
||||
with pytest.raises(MembershipValidationError, match="allowed set"):
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="u-1", role="viewer")
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
async def test_change_role_admin_to_member(tmp_path):
|
||||
ws_repo, m_repo, sf = await _setup(tmp_path)
|
||||
try:
|
||||
await _seed_user(sf, "u-1", "u1@example.com")
|
||||
await _seed_user(sf, "u-2", "u2@example.com")
|
||||
ws = await ws_repo.create(name="W", slug="chg", owner_id="u-1")
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="u-1", role="owner")
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="u-2", role="admin")
|
||||
|
||||
ok = await m_repo.change_role(workspace_id=ws["id"], user_id="u-2", new_role="member")
|
||||
assert ok is True
|
||||
assert await m_repo.get_role(workspace_id=ws["id"], user_id="u-2") == "member"
|
||||
|
||||
# change_role on a non-member returns False
|
||||
miss = await m_repo.change_role(workspace_id=ws["id"], user_id="u-nonexistent", new_role="member")
|
||||
assert miss is False
|
||||
finally:
|
||||
await _cleanup()
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Partial unique index `idx_one_owner_per_workspace` works on Postgres.
|
||||
|
||||
The same assertion against SQLite is in
|
||||
:mod:`test_workspace_membership_repo::test_cannot_add_second_owner`.
|
||||
This file adds the Postgres twin via the @pytest.mark.postgres
|
||||
testcontainers fixture from PR1.
|
||||
|
||||
Why duplicate the test:
|
||||
- SQLite and Postgres parse ``WHERE`` clauses differently. We declare
|
||||
both ``sqlite_where`` and ``postgresql_where`` on the Index; this
|
||||
test pins that Postgres genuinely enforces the partial-unique
|
||||
constraint, not just that SQLAlchemy emits the DDL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.workspace import WorkspaceRepository
|
||||
from deerflow.persistence.workspace_membership import WorkspaceMembershipRepository
|
||||
|
||||
pytestmark = [pytest.mark.postgres, pytest.mark.anyio]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend() -> str:
|
||||
return "asyncio"
|
||||
|
||||
|
||||
async def test_partial_unique_on_owner_enforced_on_postgres(postgres_url: str) -> None:
|
||||
"""Postgres: inserting a 2nd owner must raise IntegrityError, same as SQLite."""
|
||||
from deerflow.persistence.engine import close_engine, get_session_factory, init_engine
|
||||
|
||||
await init_engine("postgres", url=postgres_url)
|
||||
try:
|
||||
sf = get_session_factory()
|
||||
ws_repo = WorkspaceRepository(sf)
|
||||
m_repo = WorkspaceMembershipRepository(sf)
|
||||
|
||||
# Seed two users
|
||||
async with sf() as session:
|
||||
session.add(UserRow(id="pg-u1", email="pg1@example.com"))
|
||||
session.add(UserRow(id="pg-u2", email="pg2@example.com"))
|
||||
await session.commit()
|
||||
|
||||
ws = await ws_repo.create(name="PG W", slug="pg-one-owner", owner_id="pg-u1")
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="pg-u1", role="owner")
|
||||
|
||||
with pytest.raises(IntegrityError):
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="pg-u2", role="owner")
|
||||
finally:
|
||||
await close_engine()
|
||||
|
||||
|
||||
async def test_multiple_admins_allowed_on_postgres(postgres_url: str) -> None:
|
||||
"""Postgres: partial unique on owner must NOT block multiple admins."""
|
||||
from deerflow.persistence.engine import close_engine, get_session_factory, init_engine
|
||||
|
||||
await init_engine("postgres", url=postgres_url)
|
||||
try:
|
||||
sf = get_session_factory()
|
||||
ws_repo = WorkspaceRepository(sf)
|
||||
m_repo = WorkspaceMembershipRepository(sf)
|
||||
|
||||
async with sf() as session:
|
||||
session.add(UserRow(id="pg-a1", email="a1@example.com"))
|
||||
session.add(UserRow(id="pg-a2", email="a2@example.com"))
|
||||
session.add(UserRow(id="pg-a3", email="a3@example.com"))
|
||||
await session.commit()
|
||||
|
||||
ws = await ws_repo.create(name="PG W", slug="pg-multi-admin", owner_id="pg-a1")
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="pg-a1", role="owner")
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="pg-a2", role="admin")
|
||||
await m_repo.add(workspace_id=ws["id"], user_id="pg-a3", role="admin")
|
||||
|
||||
members = await m_repo.list_by_workspace(workspace_id=ws["id"])
|
||||
assert {(m["user_id"], m["role"]) for m in members} == {
|
||||
("pg-a1", "owner"),
|
||||
("pg-a2", "admin"),
|
||||
("pg-a3", "admin"),
|
||||
}
|
||||
finally:
|
||||
await close_engine()
|
||||
@@ -0,0 +1,296 @@
|
||||
"""Tests for WorkspaceRepository (Stage 0 PR3).
|
||||
|
||||
Pattern mirrors :mod:`test_feedback`: SQLite ephemeral DB per test via
|
||||
tmp_path, no real Postgres needed at the unit-test layer. Partial-unique
|
||||
double-driver validation lives in :mod:`test_workspace_partial_unique`
|
||||
(T3.8, runs against both backends).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.workspace import WorkspaceRepository, WorkspaceValidationError
|
||||
|
||||
|
||||
async def _make_repo(tmp_path):
|
||||
from deerflow.persistence.engine import get_session_factory, init_engine
|
||||
|
||||
url = f"sqlite+aiosqlite:///{tmp_path / 'test.db'}"
|
||||
await init_engine("sqlite", url=url, sqlite_dir=str(tmp_path))
|
||||
return WorkspaceRepository(get_session_factory())
|
||||
|
||||
|
||||
async def _cleanup():
|
||||
from deerflow.persistence.engine import close_engine
|
||||
|
||||
await close_engine()
|
||||
|
||||
|
||||
async def _seed_user(repo, user_id: str = "u-alice", email: str = "alice@example.com") -> None:
|
||||
"""Create a user row so workspace.owner_id FK is satisfied."""
|
||||
async with repo._sf() as session:
|
||||
session.add(UserRow(id=user_id, email=email))
|
||||
await session.commit()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.anyio
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend() -> str:
|
||||
return "asyncio"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# create / get_by_slug round-trip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_create_then_lookup_by_slug(tmp_path):
|
||||
repo = await _make_repo(tmp_path)
|
||||
try:
|
||||
await _seed_user(repo)
|
||||
created = await repo.create(name="Alice's Workspace", slug="alice", owner_id="u-alice")
|
||||
assert created["slug"] == "alice"
|
||||
assert created["status"] == "active"
|
||||
assert created["owner_id"] == "u-alice"
|
||||
assert len(created["id"]) == 36 # UUID v4
|
||||
|
||||
fetched = await repo.get_by_slug("alice")
|
||||
assert fetched is not None
|
||||
assert fetched["id"] == created["id"]
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
async def test_get_by_slug_returns_none_when_missing(tmp_path):
|
||||
repo = await _make_repo(tmp_path)
|
||||
try:
|
||||
assert await repo.get_by_slug("nonexistent") is None
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# slug uniqueness + format + blacklist
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_create_rejects_duplicate_slug(tmp_path):
|
||||
"""Two workspaces with the same slug — second raises IntegrityError."""
|
||||
repo = await _make_repo(tmp_path)
|
||||
try:
|
||||
await _seed_user(repo)
|
||||
await repo.create(name="A", slug="dup", owner_id="u-alice")
|
||||
with pytest.raises(IntegrityError):
|
||||
await repo.create(name="B", slug="dup", owner_id="u-alice")
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad_slug",
|
||||
[
|
||||
"ab", # too short
|
||||
"x" * 33, # too long
|
||||
"UPPER", # uppercase
|
||||
"has space", # space
|
||||
"-start-with-dash", # bad start
|
||||
"end-with-dash-", # bad end
|
||||
"double--dash", # consecutive dashes
|
||||
"underscore_not_ok", # underscore
|
||||
],
|
||||
)
|
||||
async def test_create_rejects_invalid_slug_pattern(tmp_path, bad_slug):
|
||||
repo = await _make_repo(tmp_path)
|
||||
try:
|
||||
await _seed_user(repo)
|
||||
with pytest.raises(WorkspaceValidationError, match="(pattern|length)"):
|
||||
await repo.create(name="x", slug=bad_slug, owner_id="u-alice")
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reserved", ["admin", "api", "auth", "settings", "billing", "select-workspace"])
|
||||
async def test_create_rejects_reserved_slug(tmp_path, reserved):
|
||||
repo = await _make_repo(tmp_path)
|
||||
try:
|
||||
await _seed_user(repo)
|
||||
with pytest.raises(WorkspaceValidationError, match="reserved"):
|
||||
await repo.create(name="x", slug=reserved, owner_id="u-alice")
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# status state machine
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_status_state_transitions(tmp_path):
|
||||
"""active → suspended → deleted are all accepted."""
|
||||
repo = await _make_repo(tmp_path)
|
||||
try:
|
||||
await _seed_user(repo)
|
||||
ws = await repo.create(name="x", slug="trans", owner_id="u-alice")
|
||||
assert ws["status"] == "active"
|
||||
|
||||
await repo.update_status(ws["id"], "suspended")
|
||||
async with repo._sf() as session:
|
||||
from deerflow.persistence.workspace.model import WorkspaceRow
|
||||
|
||||
row = await session.get(WorkspaceRow, ws["id"])
|
||||
assert row.status == "suspended"
|
||||
|
||||
await repo.update_status(ws["id"], "deleted")
|
||||
async with repo._sf() as session:
|
||||
from deerflow.persistence.workspace.model import WorkspaceRow
|
||||
|
||||
row = await session.get(WorkspaceRow, ws["id"])
|
||||
assert row.status == "deleted"
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
async def test_update_status_rejects_unknown_value(tmp_path):
|
||||
repo = await _make_repo(tmp_path)
|
||||
try:
|
||||
await _seed_user(repo)
|
||||
ws = await repo.create(name="x", slug="rejstat", owner_id="u-alice")
|
||||
with pytest.raises(WorkspaceValidationError, match="allowed set"):
|
||||
await repo.update_status(ws["id"], "weird-state")
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CASCADE: workspace.delete() drops dependent memberships
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_delete_cascades_to_memberships(tmp_path):
|
||||
"""Deleting a workspace removes its membership rows (FK CASCADE)."""
|
||||
from sqlalchemy import select
|
||||
|
||||
from deerflow.persistence.workspace_membership.model import WorkspaceMembershipRow
|
||||
|
||||
repo = await _make_repo(tmp_path)
|
||||
try:
|
||||
await _seed_user(repo)
|
||||
ws = await repo.create(name="x", slug="casc", owner_id="u-alice")
|
||||
# Insert an owner membership manually (repository pattern is single-
|
||||
# responsibility; registration flow will normally insert both rows
|
||||
# in one transaction).
|
||||
async with repo._sf() as session:
|
||||
session.add(WorkspaceMembershipRow(workspace_id=ws["id"], user_id="u-alice", role="owner"))
|
||||
await session.commit()
|
||||
|
||||
await repo.delete(ws["id"])
|
||||
|
||||
async with repo._sf() as session:
|
||||
remaining = (await session.execute(select(WorkspaceMembershipRow).where(WorkspaceMembershipRow.workspace_id == ws["id"]))).scalars().all()
|
||||
assert remaining == [], "memberships should be CASCADE-deleted with workspace"
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# membership-aware get + list_by_user
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.no_auto_user
|
||||
async def test_get_returns_none_for_non_member(tmp_path):
|
||||
"""User-A creates a workspace; User-B's `get(wsA)` returns None."""
|
||||
from deerflow.persistence.workspace_membership.model import WorkspaceMembershipRow
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
|
||||
repo = await _make_repo(tmp_path)
|
||||
try:
|
||||
# Seed two users
|
||||
async with repo._sf() as session:
|
||||
session.add(UserRow(id="u-A", email="a@example.com"))
|
||||
session.add(UserRow(id="u-B", email="b@example.com"))
|
||||
await session.commit()
|
||||
|
||||
# User A creates workspace + becomes owner
|
||||
ws = await repo.create(name="A's WS", slug="a-ws", owner_id="u-A")
|
||||
async with repo._sf() as session:
|
||||
session.add(WorkspaceMembershipRow(workspace_id=ws["id"], user_id="u-A", role="owner"))
|
||||
await session.commit()
|
||||
|
||||
# User B attempts to read it via contextvar
|
||||
user_b = SimpleNamespace(id="u-B")
|
||||
token = set_current_user(user_b)
|
||||
try:
|
||||
assert await repo.get(ws["id"]) is None
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
|
||||
# User A's own get succeeds
|
||||
user_a = SimpleNamespace(id="u-A")
|
||||
token = set_current_user(user_a)
|
||||
try:
|
||||
row = await repo.get(ws["id"])
|
||||
assert row is not None
|
||||
assert row["slug"] == "a-ws"
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
@pytest.mark.no_auto_user
|
||||
async def test_list_by_user_excludes_other_workspaces(tmp_path):
|
||||
from deerflow.persistence.workspace_membership.model import WorkspaceMembershipRow
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
|
||||
repo = await _make_repo(tmp_path)
|
||||
try:
|
||||
async with repo._sf() as session:
|
||||
session.add(UserRow(id="u-A", email="a@example.com"))
|
||||
session.add(UserRow(id="u-B", email="b@example.com"))
|
||||
await session.commit()
|
||||
|
||||
ws_a = await repo.create(name="A", slug="ws-a", owner_id="u-A")
|
||||
ws_b = await repo.create(name="B", slug="ws-b", owner_id="u-B")
|
||||
async with repo._sf() as session:
|
||||
session.add(WorkspaceMembershipRow(workspace_id=ws_a["id"], user_id="u-A", role="owner"))
|
||||
session.add(WorkspaceMembershipRow(workspace_id=ws_b["id"], user_id="u-B", role="owner"))
|
||||
await session.commit()
|
||||
|
||||
token = set_current_user(SimpleNamespace(id="u-A"))
|
||||
try:
|
||||
workspaces = await repo.list_by_user()
|
||||
assert [w["slug"] for w in workspaces] == ["ws-a"]
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
finally:
|
||||
await _cleanup()
|
||||
|
||||
|
||||
async def test_list_by_user_bypass_returns_all(tmp_path):
|
||||
"""user_id=None opts out of membership filter (migration path)."""
|
||||
repo = await _make_repo(tmp_path)
|
||||
try:
|
||||
await _seed_user(repo, "u-A", "a@example.com")
|
||||
await _seed_user(repo, "u-B", "b@example.com")
|
||||
await repo.create(name="A", slug="all-a", owner_id="u-A")
|
||||
await repo.create(name="B", slug="all-b", owner_id="u-B")
|
||||
|
||||
workspaces = await repo.list_by_user(user_id=None)
|
||||
# PR6 conftest auto-seeds an "autouse-test" workspace via the
|
||||
# ``Base.metadata.after_create`` hook so business-row FKs resolve.
|
||||
# ``user_id=None`` bypasses the membership filter, so it surfaces
|
||||
# alongside the two rows the test inserted — that is the intended
|
||||
# "no filter" behaviour. Assert the inserted ones are present.
|
||||
slugs = sorted(w["slug"] for w in workspaces)
|
||||
assert "all-a" in slugs
|
||||
assert "all-b" in slugs
|
||||
finally:
|
||||
await _cleanup()
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Slug helpers for registration / initialize (Stage 0 PR4 T4.10).
|
||||
|
||||
Two surfaces under test:
|
||||
|
||||
- ``auto_slug_from_email`` — pure transform; no DB dependency.
|
||||
- ``next_available_slug`` — async collision walker; we stub the
|
||||
``exists_check`` callable so the test stays a unit test.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from app.gateway.auth.workspace_slug import auto_slug_from_email, next_available_slug
|
||||
|
||||
# Mirror of the schema's slug pattern. Keeping it inline keeps this
|
||||
# test self-contained — if the schema regex changes we want this test
|
||||
# to refuse to lie about validity.
|
||||
_SLUG_PATTERN = re.compile(r"^[a-z0-9](-?[a-z0-9])*$")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("email", "expected"),
|
||||
[
|
||||
("foo@example.com", "foo"),
|
||||
("foo.bar@example.com", "foo-bar"),
|
||||
("foo+spam@example.com", "foo-spam"),
|
||||
("foo_bar@example.com", "foo-bar"),
|
||||
("Foo.Bar@example.com", "foo-bar"),
|
||||
("foo.bar+spam@example.com", "foo-bar-spam"),
|
||||
("aaaaaaaaaabbbbbbbbbbccccccccccddddd@example.com", "aaaaaaaaaabbbbbbbbbbccccccccccdd"),
|
||||
],
|
||||
)
|
||||
def test_auto_slug_known_inputs(email: str, expected: str) -> None:
|
||||
"""Deterministic mapping for the inputs called out in the design doc."""
|
||||
assert auto_slug_from_email(email) == expected
|
||||
assert _SLUG_PATTERN.fullmatch(auto_slug_from_email(email)), "schema regex must accept the output"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"email",
|
||||
[
|
||||
"@example.com", # no local part
|
||||
"a@example.com", # too short
|
||||
"ab@example.com", # still too short
|
||||
"...+_+...@example.com", # only separators
|
||||
"---@example.com", # only hyphens
|
||||
"🎉@example.com", # non-ASCII
|
||||
],
|
||||
)
|
||||
def test_auto_slug_falls_back_when_unusable(email: str) -> None:
|
||||
"""Pathological emails fall back to ``user-{token}`` so the slug is always valid."""
|
||||
slug = auto_slug_from_email(email)
|
||||
assert slug.startswith("user-"), f"expected fallback, got {slug!r}"
|
||||
assert _SLUG_PATTERN.fullmatch(slug)
|
||||
assert 3 <= len(slug) <= 32
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize("anyio_backend", ["asyncio"])
|
||||
async def test_next_available_slug_returns_base_when_free(anyio_backend) -> None:
|
||||
"""No collision → ``base`` is returned unchanged."""
|
||||
seen: set[str] = set()
|
||||
|
||||
async def exists(s: str) -> bool:
|
||||
return s in seen
|
||||
|
||||
assert await next_available_slug("foo", exists_check=exists) == "foo"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize("anyio_backend", ["asyncio"])
|
||||
async def test_next_available_slug_walks_through_collisions(anyio_backend) -> None:
|
||||
"""``foo``, ``foo-2`` taken → walker lands on ``foo-3``."""
|
||||
seen = {"foo", "foo-2"}
|
||||
|
||||
async def exists(s: str) -> bool:
|
||||
return s in seen
|
||||
|
||||
assert await next_available_slug("foo", exists_check=exists) == "foo-3"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize("anyio_backend", ["asyncio"])
|
||||
async def test_next_available_slug_truncates_base_to_fit_suffix(anyio_backend) -> None:
|
||||
"""A 32-char base + ``-2`` would exceed the limit → base is shortened."""
|
||||
base = "a" * 32 # exactly at the limit
|
||||
seen = {base}
|
||||
|
||||
async def exists(s: str) -> bool:
|
||||
return s in seen
|
||||
|
||||
result = await next_available_slug(base, exists_check=exists)
|
||||
assert len(result) <= 32
|
||||
assert result.endswith("-2")
|
||||
assert _SLUG_PATTERN.fullmatch(result)
|
||||
Generated
+45
-2
@@ -766,6 +766,9 @@ dependencies = [
|
||||
postgres = [
|
||||
{ name = "deerflow-harness", extra = ["postgres"] },
|
||||
]
|
||||
postgres-test = [
|
||||
{ name = "deerflow-harness", extra = ["postgres-test"] },
|
||||
]
|
||||
|
||||
[package.dev-dependencies]
|
||||
dev = [
|
||||
@@ -780,6 +783,7 @@ requires-dist = [
|
||||
{ name = "bcrypt", specifier = ">=4.0.0" },
|
||||
{ name = "deerflow-harness", editable = "packages/harness" },
|
||||
{ name = "deerflow-harness", extras = ["postgres"], marker = "extra == 'postgres'", editable = "packages/harness" },
|
||||
{ name = "deerflow-harness", extras = ["postgres-test"], marker = "extra == 'postgres-test'", editable = "packages/harness" },
|
||||
{ name = "dingtalk-stream", specifier = ">=0.24.3" },
|
||||
{ name = "email-validator", specifier = ">=2.0.0" },
|
||||
{ name = "fastapi", specifier = ">=0.115.0" },
|
||||
@@ -795,7 +799,7 @@ requires-dist = [
|
||||
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" },
|
||||
{ name = "wecom-aibot-python-sdk", specifier = ">=0.1.6" },
|
||||
]
|
||||
provides-extras = ["postgres"]
|
||||
provides-extras = ["postgres", "postgres-test"]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [
|
||||
@@ -854,6 +858,13 @@ postgres = [
|
||||
{ name = "psycopg", extra = ["binary"] },
|
||||
{ name = "psycopg-pool" },
|
||||
]
|
||||
postgres-test = [
|
||||
{ name = "asyncpg" },
|
||||
{ name = "langgraph-checkpoint-postgres" },
|
||||
{ name = "psycopg", extra = ["binary"] },
|
||||
{ name = "psycopg-pool" },
|
||||
{ name = "testcontainers" },
|
||||
]
|
||||
pymupdf = [
|
||||
{ name = "pymupdf4llm" },
|
||||
]
|
||||
@@ -866,6 +877,7 @@ requires-dist = [
|
||||
{ name = "alembic", specifier = ">=1.13" },
|
||||
{ name = "asyncpg", marker = "extra == 'postgres'", specifier = ">=0.29" },
|
||||
{ name = "ddgs", specifier = ">=9.10.0" },
|
||||
{ name = "deerflow-harness", extras = ["postgres"], marker = "extra == 'postgres-test'", editable = "packages/harness" },
|
||||
{ name = "dotenv", specifier = ">=0.9.9" },
|
||||
{ name = "duckdb", specifier = ">=1.4.4" },
|
||||
{ name = "exa-py", specifier = ">=1.0.0" },
|
||||
@@ -897,9 +909,10 @@ requires-dist = [
|
||||
{ name = "readabilipy", specifier = ">=0.3.0" },
|
||||
{ name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0,<3.0" },
|
||||
{ name = "tavily-python", specifier = ">=0.7.17" },
|
||||
{ name = "testcontainers", extras = ["postgres"], marker = "extra == 'postgres-test'", specifier = ">=4.0" },
|
||||
{ name = "tiktoken", specifier = ">=0.8.0" },
|
||||
]
|
||||
provides-extras = ["ollama", "postgres", "pymupdf"]
|
||||
provides-extras = ["ollama", "postgres", "postgres-test", "pymupdf"]
|
||||
|
||||
[[package]]
|
||||
name = "defusedxml"
|
||||
@@ -941,6 +954,20 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "docker"
|
||||
version = "7.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pywin32", marker = "sys_platform == 'win32'" },
|
||||
{ name = "requests" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "docstring-parser"
|
||||
version = "0.18.0"
|
||||
@@ -4124,6 +4151,22 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "testcontainers"
|
||||
version = "4.14.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "docker" },
|
||||
{ name = "python-dotenv" },
|
||||
{ name = "typing-extensions" },
|
||||
{ name = "urllib3" },
|
||||
{ name = "wrapt" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ca/ac/a597c3a0e02b26cbed6dd07df68be1e57684766fd1c381dee9b170a99690/testcontainers-4.14.2.tar.gz", hash = "sha256:1340ccf16fe3acd9389a6c9e1d9ab21d9fe99a8afdf8165f89c3e69c1967d239", size = 166841, upload-time = "2026-03-18T05:19:16.696Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/13/2d/26b8b30067d94339afee62c3edc9b803a6eb9332f521ba77d8aaab5de873/testcontainers-4.14.2-py3-none-any.whl", hash = "sha256:0d0522c3cd8f8d9627cda41f7a6b51b639fa57bdc492923c045117933c668d68", size = 125712, upload-time = "2026-03-18T05:19:15.29Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiktoken"
|
||||
version = "0.12.0"
|
||||
|
||||
+11
-8
@@ -15,7 +15,7 @@
|
||||
# ============================================================================
|
||||
# Bump this number when the config schema changes.
|
||||
# Run `make config-upgrade` to merge new fields into your local config.yaml.
|
||||
config_version: 9
|
||||
config_version: 10
|
||||
|
||||
# ============================================================================
|
||||
# Logging
|
||||
@@ -878,16 +878,19 @@ skill_evolution:
|
||||
# NOTE: When both `checkpointer` and `database` are configured,
|
||||
# `checkpointer` takes precedence for LangGraph state persistence.
|
||||
# If you use `database`, you can remove the `checkpointer` section.
|
||||
# Stage 0+ default is postgres (parity with production, room for future RLS).
|
||||
# Set DATABASE_URL in .env. SQLite is preserved as an offline-dev fallback —
|
||||
# uncomment the SQLite block below and comment out the Postgres block to use it.
|
||||
#
|
||||
# Postgres (default):
|
||||
database:
|
||||
backend: postgres
|
||||
postgres_url: $DATABASE_URL
|
||||
|
||||
# SQLite fallback (offline dev):
|
||||
# database:
|
||||
# backend: sqlite
|
||||
# sqlite_dir: .deer-flow/data
|
||||
#
|
||||
# database:
|
||||
# backend: postgres
|
||||
# postgres_url: $DATABASE_URL
|
||||
database:
|
||||
backend: sqlite
|
||||
sqlite_dir: .deer-flow/data
|
||||
|
||||
# ============================================================================
|
||||
# Run Events Configuration
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# Usage: docker-compose -f docker-compose-dev.yaml up --build
|
||||
#
|
||||
# Services:
|
||||
# - postgres: PostgreSQL 16 (port 5432) — Stage 0+ default DB backend
|
||||
# - nginx: Reverse proxy (port 2026)
|
||||
# - frontend: Frontend Next.js dev server (port 3000)
|
||||
# - gateway: Backend Gateway API + agent runtime (port 8001)
|
||||
@@ -13,6 +14,30 @@
|
||||
# Access: http://localhost:2026
|
||||
|
||||
services:
|
||||
# ── Database (Stage 0+ default backend) ────────────────────────────────
|
||||
# Postgres for local dev. Production may point DATABASE_URL at a remote RDS;
|
||||
# this service can then be omitted via `docker compose --profile <other>`.
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: deer-flow-postgres
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-deerflow}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-deerflow_dev}
|
||||
POSTGRES_DB: ${POSTGRES_DB:-deerflow}
|
||||
ports:
|
||||
- "${POSTGRES_PORT:-5432}:5432"
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-deerflow} -d ${POSTGRES_DB:-deerflow}"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 10s
|
||||
networks:
|
||||
- deer-flow-dev
|
||||
restart: unless-stopped
|
||||
|
||||
# ── Sandbox Provisioner ────────────────────────────────────────────────
|
||||
# Manages per-sandbox Pod + Service lifecycle in the host Kubernetes
|
||||
# cluster via the K8s API.
|
||||
@@ -125,6 +150,9 @@ services:
|
||||
UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:0.7.20}
|
||||
UV_INDEX_URL: ${UV_INDEX_URL:-https://pypi.org/simple}
|
||||
container_name: deer-flow-gateway
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
command: sh -c "{ cd backend && (uv sync || (echo '[startup] uv sync failed; recreating .venv and retrying once' && uv venv --allow-existing .venv && uv sync)) && PYTHONPATH=. uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 --reload --reload-include='*.yaml .env'; } > /app/logs/gateway.log 2>&1"
|
||||
volumes:
|
||||
- ../backend/:/app/backend/
|
||||
@@ -178,6 +206,8 @@ volumes:
|
||||
# image build are not shadowed by the host backend/ directory mount.
|
||||
gateway-venv:
|
||||
gateway-uv-cache:
|
||||
# Persist Postgres data across container restarts.
|
||||
postgres-data:
|
||||
|
||||
networks:
|
||||
deer-flow-dev:
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
# DeerFlow 整体架构鸟瞰
|
||||
|
||||
按"自外向内、自顶向下"分层讲,并指出每一层对应的代码位置,方便后续深入。
|
||||
|
||||
## 一、进程与部署拓扑
|
||||
|
||||
DeerFlow 表面是 4 个端口,本质是 **3 个进程 + 1 个反向代理**。
|
||||
|
||||
```
|
||||
┌──────────────────────┐
|
||||
浏览器 / IM ─────────▶│ nginx :2026 │ 统一入口
|
||||
│ (含 CORS、SSE 透传) │
|
||||
└──────────┬───────────┘
|
||||
│
|
||||
┌──────────────────┴──────────────────┐
|
||||
│ │
|
||||
▼ ▼
|
||||
┌────────────────────┐ ┌──────────────────────────┐
|
||||
│ Frontend (Next.js) │ │ Gateway (uvicorn) │
|
||||
│ :3000 │ │ :8001 │
|
||||
│ pnpm dev / preview │ │ ┌──────────────────────┐ │
|
||||
└────────────────────┘ │ │ FastAPI 路由层 │ │
|
||||
│ │ /api/models, /skills │ │
|
||||
│ │ /threads, /runs ... │ │
|
||||
│ ├──────────────────────┤ │
|
||||
│ │ LangGraph Runtime │ │
|
||||
│ │ (RunManager, │ │
|
||||
│ │ StreamBridge, │ │
|
||||
│ │ Checkpointer) │ │
|
||||
│ ├──────────────────────┤ │
|
||||
│ │ lead_agent 图 │ │
|
||||
│ │ + 18 个中间件 │ │
|
||||
│ │ + Sandbox / Tools │ │
|
||||
│ └──────────────────────┘ │
|
||||
└────────────┬─────────────┘
|
||||
│
|
||||
┌──────────────────────────┼─────────────────────────┐
|
||||
▼ ▼ ▼
|
||||
┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐
|
||||
│ Sandbox │ │ MCP Servers │ │ LLM 提供商 │
|
||||
│ Local / AIO Docker│ │ (stdio/sse/http) │ │ OpenAI/Anthropic │
|
||||
│ 提供 bash/fs │ │ │ │ /vLLM/Codex CLI │
|
||||
└──────────────────┘ └──────────────────┘ └──────────────────┘
|
||||
```
|
||||
|
||||
关键事实(容易踩坑):
|
||||
|
||||
- **LangGraph 运行时不是独立进程**,而是嵌在 Gateway 这同一个 uvicorn 进程里。`scripts/serve.sh:225` 只起了一个后端进程:`uvicorn app.gateway.app:app`。
|
||||
- nginx 把 `/api/langgraph/*` 重写成 `/api/*` 再代理到 Gateway(`docker/nginx/nginx.local.conf:49-73`),所以前端用的"标准 LangGraph SDK 协议"和 DeerFlow 自己的 REST 是同一个 8001 端口。
|
||||
- 路由协议在 nginx 里特意为 SSE 关闭了缓冲(`proxy_buffering off; X-Accel-Buffering no`),否则流式响应会被吃掉。
|
||||
|
||||
## 二、后端代码二分:Harness vs App
|
||||
|
||||
后端最重要的边界,**决定新写代码该放哪里**。
|
||||
|
||||
```
|
||||
backend/
|
||||
├── packages/harness/deerflow/ ← 可发布的智能体框架(import: deerflow.*)
|
||||
│ ├── agents/ lead_agent + memory + middlewares + ThreadState
|
||||
│ ├── runtime/ checkpointer / runs / stream_bridge / events / store
|
||||
│ ├── sandbox/ Sandbox 抽象 + local 实现 + 文件/bash 工具
|
||||
│ ├── subagents/ 子代理注册表 + 后台执行池
|
||||
│ ├── tools/ 内建工具(present_files / ask_clarification / view_image)
|
||||
│ ├── mcp/ MultiServerMCPClient + 缓存 + OAuth
|
||||
│ ├── skills/ SKILL.md 加载、工具白名单
|
||||
│ ├── models/ 模型工厂、vLLM/Codex/Claude 自定义 provider
|
||||
│ ├── community/ tavily / jina / firecrawl / aio_sandbox(可选实现)
|
||||
│ ├── memory/ 长期记忆(事实抽取、debounce 队列)
|
||||
│ ├── persistence/ SQLAlchemy 模型(用户、运行、事件、反馈)
|
||||
│ ├── guardrails/ 工具调用前置鉴权(可插拔 provider)
|
||||
│ ├── tracing/ LangSmith / Langfuse callback
|
||||
│ ├── reflection/ "module:variable" 字符串 → 实例(配置驱动的关键)
|
||||
│ ├── uploads/ 上传文件转换 markdown
|
||||
│ └── client.py DeerFlowClient(嵌入式 Python 客户端)
|
||||
│
|
||||
└── app/ ← 应用代码(import: app.*)
|
||||
├── gateway/
|
||||
│ ├── app.py FastAPI 入口 + lifespan
|
||||
│ ├── auth_middleware.py 会话/Token 鉴权
|
||||
│ ├── csrf_middleware.py 双重 cookie CSRF
|
||||
│ ├── langgraph_auth.py 注入到 langgraph.json 的鉴权钩子
|
||||
│ └── routers/ ↓ 下表
|
||||
└── channels/ IM(Slack/Telegram/Feishu/DingTalk/微信/企微)
|
||||
```
|
||||
|
||||
**铁律:app 可以 import deerflow,deerflow 不能 import app**(CI 用 `tests/test_harness_boundary.py` 强制)。这意味着 harness 必须自给自足——任何"agent 运行时需要的能力"都要在 harness 里完成抽象,app 层只做 HTTP/IM 适配。
|
||||
|
||||
## 三、Gateway 路由总览
|
||||
|
||||
`backend/app/gateway/routers/` 14 个路由文件,分三类职责:
|
||||
|
||||
| 类别 | 路由 | 干什么 |
|
||||
|---|---|---|
|
||||
| **配置/资源管理** | `models` `skills` `mcp` `memory` `agents` | 列出/启停 LLM 模型、技能、MCP、自定义 agent |
|
||||
| **会话/数据** | `threads` `uploads` `artifacts` `suggestions` | 管理线程、上传文件、产物下载、追问建议 |
|
||||
| **运行(核心)** | `thread_runs` `runs` `feedback` `assistants_compat` | 创建运行、SSE 流、消息分页、反馈打分、LangGraph 兼容协议 |
|
||||
| **横切** | `auth` `channels` | 用户登录注册、IM 渠道状态 |
|
||||
|
||||
`assistants_compat.py` 是关键:它把前端用的 LangGraph SDK 协议(`POST /threads/{id}/runs/stream`、`messages-tuple` 流模式等)翻译成 DeerFlow 内部的 `RunManager` 调用——这就是 nginx 那条 `/api/langgraph/*` 重写规则的接收端。
|
||||
|
||||
## 四、一次对话的完整生命周期
|
||||
|
||||
把上面所有零件串起来——用户在前端输入一句话,会发生这些事:
|
||||
|
||||
```
|
||||
1. 前端 useThreadStream hook
|
||||
└─▶ LangGraph SDK 调用 POST /api/langgraph/threads/{id}/runs/stream
|
||||
(stream_mode=["values","messages-tuple","custom"])
|
||||
|
||||
2. nginx 重写 → /api/threads/{id}/runs/stream → Gateway
|
||||
|
||||
3. Gateway thread_runs 路由
|
||||
├─▶ AuthMiddleware 解析 session → user_id 注入到 user_context(contextvar)
|
||||
├─▶ CSRFMiddleware 校验
|
||||
└─▶ runtime.RunManager 创建 Run → 落库(runs / run_events 表)
|
||||
|
||||
4. RunManager 调用 lead_agent 图(langgraph.json: deerflow.agents:make_lead_agent)
|
||||
├─▶ 解析 configurable: model_name / thinking_enabled / is_plan_mode / subagent_enabled
|
||||
├─▶ create_chat_model() 实例化 LLM(reflection 从 "module:Class" 字符串实例化)
|
||||
└─▶ create_agent(model, tools, middlewares, state_schema=ThreadState)
|
||||
|
||||
5. 18 个中间件按顺序拦截每一轮 model→tool→model:
|
||||
ThreadDataMiddleware 创建 .deer-flow/users/{uid}/threads/{tid}/...
|
||||
UploadsMiddleware 注入新上传文件
|
||||
SandboxMiddleware acquire 沙箱,state.sandbox_id 写入
|
||||
DanglingToolCall 修复中断的 tool_call 序列
|
||||
LLMErrorHandling LLM 报错降级
|
||||
Guardrail 工具调用前鉴权(可选)
|
||||
SandboxAudit 记录 bash/fs 操作
|
||||
ToolErrorHandling tool 异常 → ToolMessage 不中断
|
||||
Summarization token 接近上限时压缩历史
|
||||
TodoList plan_mode 才挂
|
||||
TokenUsage 累计 token
|
||||
Title 首轮后自动起标题
|
||||
Memory 队列异步抽取记忆
|
||||
ViewImage 视觉模型注入 base64
|
||||
DeferredToolFilter 需要时才暴露 tool schema
|
||||
SubagentLimit 限制 task 并发到 3
|
||||
LoopDetection 检测重复工具循环
|
||||
Clarification ask_clarification 触发 interrupt(END)
|
||||
|
||||
6. Tools 由 get_available_tools() 拼装:
|
||||
├─ Sandbox 工具:bash / ls / read_file / write_file / str_replace
|
||||
├─ 内建工具:present_files / ask_clarification / view_image / setup_agent
|
||||
├─ MCP 工具:从 extensions_config.json 启用的 server 拉取
|
||||
├─ Community 工具:tavily / jina / firecrawl / image_search(按 config.yaml)
|
||||
└─ task 工具(可选):派遣 subagent
|
||||
|
||||
7. StreamBridge 把图执行的事件流转换成 SSE:
|
||||
- "values" 完整状态快照
|
||||
- "messages-tuple" 增量 token / 工具调用 / 工具返回
|
||||
- "custom" StreamWriter 自定义事件
|
||||
- "end" 收尾,附 token usage
|
||||
|
||||
8. 前端 LangGraph SDK 接 SSE,按 message id 累加 delta,更新 UI
|
||||
|
||||
9. 运行结束后,MemoryMiddleware 后台 30s debounce 抽取记忆事实写入
|
||||
.deer-flow/users/{uid}/memory.json
|
||||
```
|
||||
|
||||
## 五、状态与持久化的几条线
|
||||
|
||||
DeerFlow 的状态被有意拆成"快/慢/历史"三层,因为它要同时支持长会话、跨进程恢复、文件级产物:
|
||||
|
||||
| 状态 | 位置 | 谁写 |
|
||||
|---|---|---|
|
||||
| **会话状态(messages, todos, artifacts)** | LangGraph checkpointer(内置 SQLite/PG,路径在 `runtime/checkpointer/async_provider.py`) | 每个 step 自动 |
|
||||
| **运行元数据/事件流** | `persistence/` 下 SQLAlchemy 模型(`runs`、`run_events`、`feedback`、`threads_meta`) | RunManager + StreamBridge |
|
||||
| **每用户每线程文件** | `.deer-flow/users/{uid}/threads/{tid}/user-data/{workspace,uploads,outputs}` | ThreadDataMiddleware + 沙箱工具 |
|
||||
| **长期记忆** | `.deer-flow/users/{uid}/memory.json`(可叠加 per-agent) | MemoryMiddleware(异步) |
|
||||
| **配置** | `config.yaml`(模型、工具、沙箱、记忆…) + `extensions_config.json`(MCP、技能开关) | `make setup` 或 Gateway PUT |
|
||||
|
||||
agent 看到的永远是 **虚拟路径** `/mnt/user-data/...` 和 `/mnt/skills/...`,由 `sandbox/tools.py` 的 `replace_virtual_path()` 翻译成上面物理路径。这层抽象让"本地沙箱"和"Docker 沙箱"对 agent 完全透明。
|
||||
|
||||
## 六、前端架构(一行总结)
|
||||
|
||||
`frontend/src/core/threads/hooks.ts` 里的 `useThreadStream` / `useSubmitThread` / `useThreads` 是整个前端的"主动脉"——它们包了 LangGraph SDK 单例(`core/api/`),所有 UI 组件订阅 thread 状态做渲染。Server Components 默认,需要交互的才 `"use client"`。`core/` 下其它子目录(artifacts/skills/mcp/memory/settings)都是为这条主动脉提供周边能力。
|
||||
|
||||
## 七、一图记住"它在做什么"
|
||||
|
||||
DeerFlow 本质上是一个 **"LangGraph 智能体 + 18 段切面 + 沙箱 + 记忆"** 的组合:
|
||||
|
||||
- **LangGraph** 提供图执行、checkpoint、stream 协议
|
||||
- **18 个中间件** 是 DeerFlow 自己加的"切面层",每个解决一个具体的健壮性/能力问题(错误恢复、上下文压缩、记忆、子代理限流……)
|
||||
- **沙箱+技能+MCP+工具** 是 agent 的"手脚"
|
||||
- **Gateway + IM Channels + 嵌入式 Client** 是同一个 agent 的三种暴露方式(HTTP/聊天/Python 直调)
|
||||
|
||||
> 想继续往里钻的话,建议下一步选三个之一:(a)走读 lead_agent + 中间件链,理解 agent 一轮 think/act 的完整代码路径;(b)走读 sandbox + tools,理解虚拟路径和工具拼装;(c)走读 runtime + StreamBridge,理解 SSE 协议怎么映射回 LangGraph SDK。
|
||||
@@ -0,0 +1,239 @@
|
||||
# ADR-001 · 数据隔离模型
|
||||
|
||||
| 项目 | 内容 |
|
||||
|---|---|
|
||||
| 状态 | 草稿(Draft) · 2026-05-09 据 spike 结果修订 §4.1.1 / §4.1.2 / §4.2 |
|
||||
| 决策日期 | TBD |
|
||||
| 决策者 | CTO + 架构 + 后端 lead |
|
||||
| 关联 ADR | ADR-004 租户层级、ADR-005 存储拓扑、ADR-006 运行时与渠道 |
|
||||
| 关联 spike / 审计 | [adr-vs-code-audit](./adr-vs-code-audit.zh-CN.md) · [langgraph-postgres spike](./adr-spike-langgraph-postgres.zh-CN.md) |
|
||||
| 代码命名 | 本 ADR 写 `tenant_id`,落代码统一读作 `workspace_id`(详 [workspace-schema-design §1](./workspace-schema-design.zh-CN.md#1-命名约定--workspace-vs-tenant)) |
|
||||
|
||||
---
|
||||
|
||||
## 1. 背景
|
||||
|
||||
DeerFlow 当前是 **"多用户单租户"** 模型:所有用户的会话、运行、记忆、产物都共用同一套表,仓储层用 `user_id` WHERE 过滤做个人空间隔离(`runtime/user_context.py:138-167`)。这套机制设计良好——repository 用 ContextVar + `AUTO` 哨兵自动注入当前用户,下层不依赖上层(harness 不能 import app)。
|
||||
|
||||
多租户化需要在 `user_id` 之上再加一层 `tenant_id`。问题是:**用什么物理隔离强度?**
|
||||
|
||||
三种主流方案:
|
||||
|
||||
| 维度 | 行级(tenant_id WHERE) | per-tenant schema | per-tenant DB |
|
||||
|---|---|---|---|
|
||||
| 实现成本 | 低 | 中 | 高 |
|
||||
| 跨租户 bug 爆炸半径 | 高 | 中 | 极低 |
|
||||
| 备份/恢复粒度 | 全量 | 按 schema | 按 DB |
|
||||
| 合规友好度(SOC2/HIPAA) | 一般 | 好 | 最好 |
|
||||
| 跨租户分析查询 | 容易 | 中 | 难 |
|
||||
| 升级 schema | 一次完成 | 要遍历所有 schema | 要遍历所有 DB |
|
||||
| 适用客户规模 | <10k 租户 | 10k–100 大客户 | <100 大客户 |
|
||||
| 运维复杂度 | 低 | 中 | 高 |
|
||||
|
||||
---
|
||||
|
||||
## 2. 决策
|
||||
|
||||
**采用 行级 `tenant_id` + Postgres Row-Level Security(RLS)** 作为双保险。
|
||||
|
||||
理由:
|
||||
|
||||
1. **DeerFlow 仓储层现状几乎平行扩展**——已经有 `resolve_user_id()` 哨兵模式,把 `tenant_id` 按同样模式补一遍,改造面集中、风险可控。
|
||||
2. **Postgres RLS 是 DB 层兜底**——即使应用层有 bug 漏写 `WHERE tenant_id = ...`,DB 也会强制过滤,第二道防线。
|
||||
3. **覆盖目标客户规模**:B2B 中小客户为主、租户数 1k–10k,行级方案足够。
|
||||
4. **不放弃跨租户分析能力**:平台需要做用量统计、监控、健康检查,单库行级最方便。
|
||||
|
||||
---
|
||||
|
||||
## 3. 备选方案与拒绝理由
|
||||
|
||||
### A. per-tenant schema(同库不同 schema)
|
||||
|
||||
**拒绝。** 看似比行级更隔离,实际坑很多:
|
||||
|
||||
- **schema 数量爆炸**:1000 个租户 = 1000 个 schema × 每张表,pg_class 体积膨胀,连接池里 search_path 切换有性能抖动
|
||||
- **schema migration 痛苦**:每发布一次 schema 改动要遍历所有 schema 跑 migration,失败回滚极复杂
|
||||
- **跨租户查询难**:要写 `UNION ALL` 跨所有 schema,运营仪表盘几乎无法实现
|
||||
- **依然需要应用层过滤**:连接进哪个 schema 仍由应用层决定,没真正消除"应用层 bug 跨租户"
|
||||
|
||||
### B. per-tenant database(独立物理库)
|
||||
|
||||
**拒绝(默认场景)。** 隔离最强但成本极高:
|
||||
|
||||
- **运维负担**:1000 个 DB = 1000 套备份/恢复/监控/连接池
|
||||
- **冷启动延迟**:每个租户新建 DB 时间从秒级飙到分钟级
|
||||
- **跨租户操作不可能**:平台级查询、聚合、迁移全部失效
|
||||
- **连接池复杂度爆炸**:每租户独立连接池或者共用动态切库,都是噩梦
|
||||
|
||||
**仅在两种情况切换到此方案:** ① 拿到强合规客户(金融/医疗/政府),合同里写明物理数据隔离;② 客户付费足够覆盖每租户独立 DB 的运维成本(典型企业级订阅)。
|
||||
|
||||
---
|
||||
|
||||
## 4. 落地影响
|
||||
|
||||
### 4.1 表结构改造
|
||||
|
||||
所有业务表加 `tenant_id` 列 + 复合索引(`tenant_id` 作为前导列):
|
||||
|
||||
```sql
|
||||
ALTER TABLE threads_meta ADD COLUMN tenant_id UUID NOT NULL;
|
||||
CREATE INDEX idx_threads_meta_tenant_user ON threads_meta (tenant_id, user_id, updated_at DESC);
|
||||
|
||||
ALTER TABLE runs ADD COLUMN tenant_id UUID NOT NULL;
|
||||
CREATE INDEX idx_runs_tenant_created ON runs (tenant_id, created_at DESC);
|
||||
|
||||
ALTER TABLE run_events ADD COLUMN tenant_id UUID NOT NULL;
|
||||
CREATE INDEX idx_run_events_tenant_run ON run_events (tenant_id, run_id, seq);
|
||||
|
||||
ALTER TABLE feedback ADD COLUMN tenant_id UUID NOT NULL;
|
||||
CREATE INDEX idx_feedback_tenant_run ON feedback (tenant_id, run_id);
|
||||
|
||||
-- ADR-005 引入的新表也要带 tenant_id(建表时就有)
|
||||
-- agent_configs, memory_facts, memory_context,
|
||||
-- tenant_skill_state, tenant_mcp_configs, tenant_secrets, tenant_quotas
|
||||
```
|
||||
|
||||
**关键索引原则**:每个 `tenant_id` 都必须是复合索引的**第一列**——RLS policy 走的就是这条路径,前导列错了 RLS 会全表扫。
|
||||
|
||||
#### 4.1.1 LangGraph 自有表(checkpoints / checkpoint_writes / checkpoint_blobs / checkpoint_migrations)
|
||||
|
||||
`runtime/checkpointer/async_provider.py` 用的是 LangGraph 内置 `AsyncPostgresSaver`,**表结构不在 DeerFlow 控制下**。原稿讨论过两条路(subquery RLS / 列升级),spike([adr-spike-langgraph-postgres](./adr-spike-langgraph-postgres.zh-CN.md))验证后我们改用 **两层隔离模型**:
|
||||
|
||||
| 表归属 | 隔离机制 | 防线性质 |
|
||||
|---|---|---|
|
||||
| **DeerFlow 自有表**(threads_meta、runs、run_events、feedback、users、tenant_*) | RLS + `SET LOCAL app.tenant_id` via SQLAlchemy session(DeerFlow 完全控制 conn pool) | DB 强约束 |
|
||||
| **LangGraph checkpoint 表** | **应用层强校验**——入口路由在调 LangGraph 前必查 `threads_meta` 上的 `(tenant_id, thread_id)` 归属 | 应用层强约束 + 表 unique constraint 兜底 |
|
||||
|
||||
**为何对 LangGraph 表放弃 RLS**:
|
||||
|
||||
- `langgraph-checkpoint-postgres==3.0.5` **不存在 `connection_factory` 参数**(spike §2 实测);其连接池注入路径只有 `__init__(conn=AsyncConnectionPool)` 这一个口子,且 `psycopg_pool` 自带的 `configure` callback 只在物理连接首次创建时跑——拿不到运行期 ContextVar 里的 tenant_id
|
||||
- 子类化 `AsyncConnectionPool` 重写 `getconn` 注入 `SET app.tenant_id`/`RESET` 是可行的 hack,但侵入 psycopg-pool 内部,库升级风险高(spike §3.1)
|
||||
- 给 LangGraph 表 ALTER 加 `tenant_id` 列同样不可取——LangGraph 用 `MIGRATIONS` 数组管理 schema,每次升级都要 diff 防漏(spike §3.4)
|
||||
|
||||
**LangGraph 表的安全模型**(接受的 trade-off):
|
||||
|
||||
- 安全等级从"DB 强约束"降级为"应用层强约束 + 表 unique constraint"
|
||||
- 强约束点是 **`threads_meta` 表上的 `UNIQUE (tenant_id, thread_id)` 复合索引** + 入口路由的强校验:任何代码路径要写 LangGraph 表前必须先在 `threads_meta` 找到对应行,且行的 `tenant_id` 与当前 ContextVar 一致
|
||||
- CI 加 boundary 测试,禁止任何路径绕过 `threads.py` / `thread_runs.py` 直连 LangGraph saver(包括 LangGraph Studio 必须走相同入口或显式审批)
|
||||
- 平台 admin 路径走 `BYPASSRLS` role 时同样必须经过应用层 audit,不直接跳过 thread 归属检查
|
||||
|
||||
**未来可升级路径**(不阻塞 phase-0):若上游接受 PR 加入 `connection_factory`,可平滑切回"DeerFlow 表 + LangGraph 表统一 RLS"模型。
|
||||
|
||||
#### 4.1.2 第一道防线:thread_id ↔ tenant_id 校验
|
||||
|
||||
LangGraph 调用入口在 **`app/gateway/routers/threads.py`**(thread CRUD)和 **`app/gateway/routers/thread_runs.py`**(run 创建/恢复/事件流)。这两个路由在调用 LangGraph 之前**必须先用 `threads_meta` 校验 `(tenant_id, thread_id)` 归属**:
|
||||
|
||||
- **创建路径**:先在 `threads_meta` 写入 `(tenant_id=current, thread_id, user_id=current)`,依赖 `UNIQUE (tenant_id, thread_id)` 防重;再调 LangGraph 创建对应 thread
|
||||
- **读/写路径**:先用 `(current_tenant_id, requested_thread_id)` SELECT `threads_meta`,未命中即 404;命中后才允许调 LangGraph
|
||||
|
||||
> 注:原稿写"在 `AssistantsCompat` 路由强制校验"是错的——`assistants_compat.py:1-50` 只服务 `assistants.search/get` 静态 stub,**不**触达 thread 入口(审计报告 §ADR-001 已修正)。
|
||||
|
||||
应用层校验是第一道防线、`UNIQUE` 约束是 DB 层兜底——任何对 LangGraph 表的访问都经过这一关。
|
||||
|
||||
### 4.2 RLS policy 模板
|
||||
|
||||
所有带 `tenant_id` 的表都加同样形态的 policy:
|
||||
|
||||
```sql
|
||||
ALTER TABLE threads_meta ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE threads_meta FORCE ROW LEVEL SECURITY; -- 即使表所有者也走 policy
|
||||
|
||||
CREATE POLICY tenant_isolation ON threads_meta
|
||||
USING (tenant_id = current_setting('app.tenant_id', true)::uuid)
|
||||
WITH CHECK (tenant_id = current_setting('app.tenant_id', true)::uuid);
|
||||
```
|
||||
|
||||
应用层在每次拿到连接时,先 `SET LOCAL app.tenant_id = '<uuid>'`:
|
||||
|
||||
```python
|
||||
# packages/harness/deerflow/persistence/engine.py
|
||||
async def _set_session_tenant(session: AsyncSession, tenant_id: str) -> None:
|
||||
"""Bind session to tenant; RLS policy enforces filter."""
|
||||
await session.execute(text("SET LOCAL app.tenant_id = :tid"), {"tid": tenant_id})
|
||||
```
|
||||
|
||||
每个仓储方法的开头自动调用,从 ContextVar 取 tenant_id(仿照现有 `resolve_user_id` 模式)。
|
||||
|
||||
**LangGraph 自有连接池不参与 SET LOCAL**:DeerFlow 仓储和 LangGraph checkpointer 是**两套连接池**——前者是 SQLAlchemy `AsyncSession`(DeerFlow 控制),后者是 LangGraph 自己持有的 psycopg 池(DeerFlow 不可控)。按 §4.1.1 的两层模型:
|
||||
|
||||
- **DeerFlow 自有表**:上面 `_set_session_tenant` helper 在每次仓储调用前注入 `SET LOCAL`,RLS 兜底
|
||||
- **LangGraph 表**:**不注入 `SET LOCAL`**——LangGraph 表上不启用 RLS,租户隔离靠应用层强校验(§4.1.2)实现。`AsyncPostgresSaver` 仍按现状用 `from_conn_string`,无侵入
|
||||
|
||||
> 原稿设想的"自定义 `connection_factory` 注入到 saver 构造"在 `langgraph-checkpoint-postgres==3.0.5` 不可行——库不存在该参数([spike](./adr-spike-langgraph-postgres.zh-CN.md) §2.2/2.4)。详细备选方案与拒绝理由见 spike §3。
|
||||
|
||||
**关键约束**:所有"会查 DeerFlow 自有表"的代码路径都必须保证调用栈上已注入 `app.tenant_id`,否则 RLS 会把整个会话过滤成空集。CI 加冒烟测试确认这点。LangGraph 表上的访问则必须经过 §4.1.2 的入口校验。
|
||||
|
||||
### 4.3 ContextVar 扩展
|
||||
|
||||
在 `runtime/user_context.py` 旁边加 `tenant_context.py`:
|
||||
|
||||
```python
|
||||
_current_tenant: Final[ContextVar[CurrentTenant | None]] = ContextVar("deerflow_current_tenant", default=None)
|
||||
|
||||
class _AutoSentinel: ... # 同 user_id 模式
|
||||
AUTO: Final[_AutoSentinel] = _AutoSentinel()
|
||||
|
||||
def resolve_tenant_id(value, *, method_name) -> str:
|
||||
"""与 resolve_user_id 同款三态:AUTO / 显式 str / 显式 None。
|
||||
SaaS 模式下 None 是 forbidden(除非显式 admin override)。"""
|
||||
```
|
||||
|
||||
`AuthMiddleware` 在解析完 JWT 后两个 ContextVar 同时注入。
|
||||
|
||||
### 4.4 SQLite → Postgres 迁移
|
||||
|
||||
**SQLite 不支持 RLS**,多租户上线必须切 Postgres。迁移路径:
|
||||
|
||||
1. 第 0 阶段:在 dev 环境同时跑 SQLite 和 Postgres,用 `aiosqlite` / `asyncpg` 双驱动
|
||||
2. 第 1 阶段:生产切 Postgres,老数据用 `pg_loader` 导入;DeerFlow 现有的 `Base.metadata.create_all()` 直接接 Postgres
|
||||
3. 老用户的 `user_id` 在没有 tenant_id 时归到一个"legacy_tenant",迁移脚本同步把 `tenant_id` 回填
|
||||
|
||||
### 4.5 跨租户操作(平台后台)
|
||||
|
||||
平台 admin / 运维需要跨租户查询时,不能简单"绕过 RLS"——而是用一个**专用 role** 配 `BYPASSRLS`,只给受限运维账号使用,操作审计入库:
|
||||
|
||||
```sql
|
||||
CREATE ROLE deerflow_admin BYPASSRLS;
|
||||
-- 应用层 admin 路由用这个 role 的连接池,并强制审计日志
|
||||
```
|
||||
|
||||
**绝不允许应用主连接池有 `BYPASSRLS`。**
|
||||
|
||||
---
|
||||
|
||||
## 5. 风险与缓解
|
||||
|
||||
| 风险 | 缓解 |
|
||||
|---|---|
|
||||
| 应用层漏写 tenant_id WHERE | RLS 是兜底(DeerFlow 表);CI 加静态检查(detect SQL 不带 tenant_id) |
|
||||
| **LangGraph 表无 RLS,仅应用层强约束**(§4.1.1 trade-off) | `threads_meta` `UNIQUE (tenant_id, thread_id)` 兜底;CI boundary 测试禁止绕过 `threads.py` / `thread_runs.py` 直连 saver;定期审计任何新增的 LangGraph 直连路径 |
|
||||
| 索引前导列错了走全表扫 | DBA 评审所有 EXPLAIN;上线前压测 |
|
||||
| `current_setting('app.tenant_id')` 没设导致 RLS 全过滤掉 | 应用层 fail-closed;监控空集查询率 |
|
||||
| 跨租户分析需求多 | 提供受控的 admin role + 审计日志 |
|
||||
| SQLite 开发 vs Postgres 生产差异 | 测试集成层用 testcontainers 跑 Postgres;不允许用 SQLite 跑 RLS 相关测试 |
|
||||
| **当前不存在 Postgres 测试夹具基础设施**(审计报告 §ADR-001 highest-risk gap) | phase-0 必须先落 testcontainers + RLS 冒烟测试,再做仓储改造;否则 RLS bug 进生产 |
|
||||
| 单 DB 容量上限(>1TB 后维护困难) | 监控 DB 体积;超过阈值切 per-tenant DB(推翻方案) |
|
||||
|
||||
---
|
||||
|
||||
## 6. 推翻条件
|
||||
|
||||
切换到 **per-tenant DB** 当且仅当:
|
||||
|
||||
1. 拿到强合规客户(金融/医疗/政府),合同要求物理数据隔离
|
||||
2. 单 DB 容量 / 写 TPS 触顶,垂直扩展不经济
|
||||
3. 出现一次跨租户数据泄露事故,董事会要求最强隔离
|
||||
|
||||
---
|
||||
|
||||
## 7. 默认假设
|
||||
|
||||
| 项 | 默认 |
|
||||
|---|---|
|
||||
| 数据库 | PostgreSQL 16+ |
|
||||
| RLS 启用 | 所有带 `tenant_id` 的业务表 |
|
||||
| 主键 | UUID v7(时间排序) |
|
||||
| 索引前导列 | `tenant_id` |
|
||||
| Connection pool | 每应用进程 20–50 conns,PgBouncer transaction mode |
|
||||
| 备份 | 每日全量 + WAL streaming,保留 30 天 |
|
||||
| 跨租户查询 | 仅通过 `deerflow_admin` role + 审计 |
|
||||
@@ -0,0 +1,372 @@
|
||||
# ADR-002 · 沙箱隔离模型
|
||||
|
||||
| 项目 | 内容 |
|
||||
|---|---|
|
||||
| 状态 | 草稿(Draft) |
|
||||
| 决策日期 | TBD |
|
||||
| 决策者 | 安全 + 架构 + SRE |
|
||||
| 关联 ADR | ADR-001 数据隔离、ADR-005 存储拓扑 |
|
||||
| 关联审计 | [adr-vs-code-audit](./adr-vs-code-audit.zh-CN.md) — 注意:现有 `AioSandboxProvider` 出网/资源/cosign 缺位;K8sSandboxProvider 几乎从零开工(实际工作量大于本 ADR §5 估算) |
|
||||
| 代码命名 | 本 ADR 写 `tenant_id` / `tenant-{tenant_id}` namespace,落代码统一读作 `workspace_id` / `ws-{workspace_id}`(详 [workspace-schema-design §1](./workspace-schema-design.zh-CN.md#1-命名约定--workspace-vs-tenant)) |
|
||||
|
||||
---
|
||||
|
||||
## 0. 概念前提
|
||||
|
||||
本 ADR 反复出现 **K8s Namespace + gVisor/Kata 运行时 + NetworkPolicy** 三件套——它们在不同层把租户的代码运行环境关起来,缺一不可。先用一段话讲清楚是什么、防什么、不防什么,再读后面的决策细节会顺很多。
|
||||
|
||||
### 0.1 K8s Namespace —— 资源/视图隔离
|
||||
|
||||
Kubernetes 的逻辑分区。一个集群里跑多租户,每租户分一个 namespace(如 `tenant-acme`、`tenant-bigco`),其中的 Pod / Service / Secret / ConfigMap 互相看不见。配套:
|
||||
|
||||
- **ResourceQuota**:限制 namespace 总用量(CPU、内存、Pod 数、存储)
|
||||
- **LimitRange**:单 Pod 兜底(默认 request/limit、单 Pod 上限)
|
||||
- **RBAC**:把租户管理员权限只绑到自己的 namespace
|
||||
|
||||
⚠️ **不是安全边界**。namespace 只让你"看不到",不是"碰不到"。两租户的 Pod 若都跑在默认 `runc` 上、共享同一个 Linux 内核,**任何一个内核 0day 都能让 A 容器逃逸到宿主,进而看到 B 容器**。所以需要下一层。
|
||||
|
||||
> 类比:办公楼里不同公司的门禁卡。同事进不了别人的工位,但墙不会自己变厚。
|
||||
|
||||
### 0.2 gVisor / Kata —— 内核级隔离
|
||||
|
||||
把容器从宿主内核上"再隔一层"。DeerFlow 沙箱要跑租户上传的任意 Python 代码,必须比 runc 更硬。
|
||||
|
||||
**gVisor(Google)**
|
||||
- 用户态实现一个沙箱内核(Sentry),拦截容器所有 syscall 自己模拟,再用极少几个 syscall 跟真内核打交道
|
||||
- 攻击面:先攻破 Sentry,再攻破真内核——多一道
|
||||
- 代价:每个 syscall 中转,IO 密集型 workload 慢 ~10-30%
|
||||
- 集成:Pod spec 写 `runtimeClassName: gvisor`
|
||||
|
||||
**Kata Containers**
|
||||
- 给每个 Pod 起一个轻量虚拟机(QEMU 或 Firecracker 后端),容器跑在 VM 内独立内核里
|
||||
- 隔离强度 ≈ 真 VM,启动几百毫秒
|
||||
- 代价:每 Pod 多占 ~50-150 MB 内存、冷启动比 gVisor 慢一点
|
||||
- 集成:`runtimeClassName: kata-qemu` / `kata-fc`
|
||||
|
||||
> 本 ADR 取舍:默认 **gVisor**(性价比平衡),监管/付费档位切 **Kata-Firecracker**(接近 VM 强度,单独定价)。配套 Cosign 镜像签名 + 只读根文件系统 + drop ALL caps 是纵深防御。
|
||||
|
||||
### 0.3 NetworkPolicy —— 出/入流量白名单
|
||||
|
||||
K8s 原生防火墙,按 namespace / Pod label 控制谁能跟谁通信。**默认拒绝 + 显式放行**是标准姿势:
|
||||
|
||||
```yaml
|
||||
spec:
|
||||
podSelector: {} # 命中 namespace 内所有 Pod
|
||||
policyTypes: [Egress]
|
||||
egress: [] # 空白名单 = 全部禁止
|
||||
```
|
||||
|
||||
为什么对 DeerFlow 是关键——租户代码可能尝试访问:
|
||||
|
||||
| 目标 | 风险 |
|
||||
|---|---|
|
||||
| `169.254.169.254`(云元数据) | 偷 IAM 凭据、节点 token,直接拿下集群 |
|
||||
| 平台内网 DB / Redis | 横向打到其他租户的数据 |
|
||||
| 其他租户的 namespace IP | 跨租户监听/嗅探 |
|
||||
| 互联网 C2 服务器 | 数据外泄、挖矿、僵尸网络 |
|
||||
|
||||
默认全部拒绝后,仅放行:DNS(CoreDNS)+ 出口走 **Egress Gateway**(Envoy/Squid 做域名白名单,允许 `api.openai.com`、`pypi.org` 等,拒绝其余)。
|
||||
|
||||
> 执行靠 CNI 插件(Calico / Cilium)。Cilium 还支持 L7 策略(如"允许 GET /v1/chat/completions、禁 POST /admin"),是更强的备选。
|
||||
|
||||
### 0.4 三件套合起来看
|
||||
|
||||
```
|
||||
租户的 Python 代码
|
||||
│
|
||||
▼
|
||||
┌────────────────────────────────────────┐
|
||||
│ Pod (tenant-acme namespace) │ ← K8s Namespace:逻辑隔离 + 配额
|
||||
│ ├─ runtimeClassName: gvisor │ ← gVisor:内核级攻击面隔离
|
||||
│ ├─ readOnlyRootFilesystem │
|
||||
│ └─ capabilities.drop: ["ALL"] │
|
||||
└────────────────────────────────────────┘
|
||||
│ egress
|
||||
▼
|
||||
┌────────────────────────────────────────┐
|
||||
│ NetworkPolicy: default-deny │ ← NetworkPolicy:网络层白名单
|
||||
│ → 仅允许 Egress Gateway / DNS │
|
||||
└────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
Egress Gateway(域名白名单)
|
||||
```
|
||||
|
||||
- 没有 namespace:租户互相能看见对方的资源对象
|
||||
- 没有 gVisor:一个内核 0day 全集群陪葬
|
||||
- 没有 NetworkPolicy:租户代码 `curl 169.254.169.254` 就能拿走节点凭据
|
||||
|
||||
三层都套上,才是本 ADR 想要的"对抗任意租户代码"的最低防御姿势。
|
||||
|
||||
---
|
||||
|
||||
## 1. 背景
|
||||
|
||||
> **分期落地提示**:本 ADR 描述的 K8s + gVisor + NetworkPolicy 全套架构是**目标态**。按 [phased-rollout-by-scale](../02-rollout/phased-rollout-by-scale.zh-CN.md) 实际落地节奏:
|
||||
> - **Stage 1**:仅做 §3 威胁模型的"出网默认禁 + cgroup CPU/memory 限额",落到现有 `AioSandboxProvider` 上(轻量补丁版)。**不上 K8s**。
|
||||
> - **Stage 3**:才换 `K8sSandboxProvider`,引入 namespace + gVisor + NetworkPolicy + Pod Security Standard 全套(§5 全文落地)。
|
||||
> - **Stage 4 / premium**:按合同切 Kata-Firecracker 或独立 nodepool。
|
||||
>
|
||||
> 读 §2~§9 时把它当作"Stage 3 完成态",Stage 1 落地时只摘 §3 出网/资源那两行就好。
|
||||
|
||||
沙箱是多租户里**爆炸半径最大**的组件:客户的 agent 可以跑任意 bash 命令、读写文件、调用 MCP 工具。如果隔离不够强,一个客户能:
|
||||
|
||||
- **读到其他客户的数据**(容器逃逸 / 共享卷误用)
|
||||
- **薅云元数据**(`curl http://169.254.169.254/...` 偷 IAM 凭证)
|
||||
- **横向移动**(同 namespace 的其他容器、同节点的 hostNetwork)
|
||||
- **耗尽资源**(fork bomb、无限循环、磁盘填满)
|
||||
|
||||
DeerFlow 现状有两个 sandbox provider:
|
||||
|
||||
| Provider | 强度 | 多租户可用 |
|
||||
|---|---|---|
|
||||
| `LocalSandboxProvider` | bash/fs **直接落主机**,零隔离 | ❌ 绝对不能用 |
|
||||
| `AioSandboxProvider` | Docker 容器(社区实现,`packages/harness/deerflow/community/aio_sandbox/`) | ⚠️ 当前配置不够 |
|
||||
|
||||
`AioSandboxProvider` 起点不错(每 thread 一个容器、虚拟路径翻译已有),但默认配置缺少多租户必需的几条隔离:默认出网未禁、CPU/内存 limits 未强制、根文件系统未只读、镜像未签名校验。
|
||||
|
||||
---
|
||||
|
||||
## 2. 决策
|
||||
|
||||
**采用 K8s + 强隔离运行时(gVisor 或 Kata Containers)+ NetworkPolicy 默认禁出网 + per-tenant Namespace。**
|
||||
|
||||
| 层 | 作用 |
|
||||
|---|---|
|
||||
| **K8s Namespace per tenant** | 资源逻辑隔离;NetworkPolicy 起效边界 |
|
||||
| **gVisor (runsc) 运行时** | 用户态系统调用拦截,容器逃逸到宿主难度大幅提升;性能损失 ~5-15%(多数 agent 任务可接受) |
|
||||
| **NetworkPolicy 默认 DENY** | 出网白名单:只允许到 LLM endpoint、配置的 MCP servers、搜索 API |
|
||||
| **ResourceQuota + LimitRange** | per-namespace CPU/内存上限;单 pod CPU/内存上限 |
|
||||
| **Pod Security Standard: restricted** | 禁 root、禁 privileged、只读根文件系统、drop ALL caps |
|
||||
| **emptyDir 临时卷** | 数据靠 ADR-005 同步对象存储,pod 销毁即清 |
|
||||
| **镜像签名校验**(Cosign) | 启动 pod 前验证 sandbox 镜像签名,防供应链攻击 |
|
||||
|
||||
**Premium 客户档位**:在此之上再加 per-tenant **物理节点池** + **Firecracker microVM**(kata-fc),把"租户 X 的沙箱永远不和别人共享物理节点"做到 SLA 里。
|
||||
|
||||
---
|
||||
|
||||
## 3. 威胁模型
|
||||
|
||||
| 攻击场景 | 共享 Docker(现状) | per-tenant K8s NS + gVisor(决策) | per-tenant Firecracker |
|
||||
|---|---|---|---|
|
||||
| 容器逃逸到宿主 | 全员沦陷 | 单租户沦陷(gVisor 大幅降低成功率) | 单租户沦陷(VM 边界,逃逸难度极高) |
|
||||
| 容器间横向移动(同节点) | 可行 | NetworkPolicy 禁止 + namespace 隔离 | 不可能(独立 VM) |
|
||||
| 出网到云元数据 169.254.169.254 | 默认可行 | NetworkPolicy 禁 + IMDSv2 强制 token | 同左 |
|
||||
| 出网到内部服务(DB / 内网) | 可行 | NetworkPolicy 禁 + egress gateway 白名单 | 同左 |
|
||||
| 侧信道(CPU 缓存 / Spectre) | 可行 | 减弱(gVisor 隔离系统调用,但 CPU 共享仍有风险) | 显著减弱(独立 VM、独立 vCPU) |
|
||||
| 资源耗尽(fork bomb / OOM) | 影响同节点全部容器 | LimitRange 强制 cgroup;超限 OOMKill | VM 内独立调度 |
|
||||
| 持久化攻击(写定时任务) | 可写主机 cron | 只读根文件系统 + ephemeral pod 重建即清 | 同左 |
|
||||
| 提权 | 看 Docker 配置(默认 root) | restricted PSS 禁 root、禁 capabilities | 同左 |
|
||||
| 镜像被替换(供应链) | 不校验 | Cosign 验证签名 + admission controller 拦截 | 同左 |
|
||||
|
||||
---
|
||||
|
||||
## 4. 备选方案与拒绝理由
|
||||
|
||||
### A. 共享 Docker(保留 AioSandboxProvider 当前模式)
|
||||
|
||||
**拒绝。** 即使加固到极致,根本问题仍在:
|
||||
|
||||
- 所有租户共享 dockerd / containerd,一次容器逃逸 = 全员沦陷
|
||||
- Docker 的 NetworkPolicy 等价物(user-defined network)粒度粗
|
||||
- 资源限制依赖 cgroup v1/v2 配置一致,运维难统一
|
||||
|
||||
### B. per-tenant Firecracker microVM(默认)
|
||||
|
||||
**拒绝(作为默认)。** 隔离最强但成本最高:
|
||||
|
||||
- 冷启动比 K8s pod 慢 2-5×(500ms vs 几秒)
|
||||
- 运维需要专门团队(kata-fc / cloud-hypervisor 都不是开箱即用)
|
||||
- AWS / 阿里云的部分托管 K8s 不直接支持 Firecracker,需要自建 nodepool
|
||||
|
||||
**保留作为 premium 档位**:把 Firecracker 当付费 SLA 卖给监管类客户。
|
||||
|
||||
### C. 共享 K8s Namespace + 仅靠 NetworkPolicy + cgroup
|
||||
|
||||
**拒绝。** namespace 不分隔意味着:
|
||||
|
||||
- pod-to-pod 通信默认开放(NetworkPolicy 是白名单制,漏一条就全开)
|
||||
- ServiceAccount 共享,越权读 secret 风险大
|
||||
- ResourceQuota 是 namespace 级别,没法精细分配到租户
|
||||
|
||||
---
|
||||
|
||||
## 5. 落地影响
|
||||
|
||||
### 5.1 新建 `K8sSandboxProvider`
|
||||
|
||||
```python
|
||||
# packages/harness/deerflow/sandbox/k8s/provider.py
|
||||
class K8sSandboxProvider(SandboxProvider):
|
||||
"""每 thread 一个 Pod,按 tenant 落到对应 Namespace。
|
||||
|
||||
生命周期:
|
||||
acquire(thread_id) → 创建 Pod(gVisor runtime, restricted PSS, NetworkPolicy 已挂)
|
||||
get(sandbox_id) → 返回与运行中 Pod 通信的客户端(kubectl exec / WebSocket)
|
||||
release(sandbox_id) → delete Pod(emptyDir 自动回收)
|
||||
"""
|
||||
```
|
||||
|
||||
替换 `LocalSandboxProvider`(开发/测试用)和 `AioSandboxProvider`(保留作为单机部署 fallback)。
|
||||
|
||||
### 5.2 K8s 资源(每租户 Namespace 一份)
|
||||
|
||||
```yaml
|
||||
# tenant onboarding 时自动渲染、apply
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: tenant-{tenant_id}
|
||||
labels:
|
||||
pod-security.kubernetes.io/enforce: restricted
|
||||
deerflow.io/tenant-id: {tenant_id}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ResourceQuota
|
||||
metadata:
|
||||
namespace: tenant-{tenant_id}
|
||||
spec:
|
||||
hard:
|
||||
cpu: "16"
|
||||
memory: 32Gi
|
||||
pods: "20"
|
||||
requests.storage: 100Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: LimitRange
|
||||
metadata:
|
||||
namespace: tenant-{tenant_id}
|
||||
spec:
|
||||
limits:
|
||||
- type: Container
|
||||
default: { cpu: "500m", memory: 1Gi }
|
||||
defaultRequest: { cpu: "100m", memory: 256Mi }
|
||||
max: { cpu: "2", memory: 4Gi }
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: deny-all-egress
|
||||
namespace: tenant-{tenant_id}
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes: [Egress]
|
||||
egress: # 仅允许下面这些
|
||||
- to:
|
||||
- namespaceSelector: { matchLabels: { name: kube-system } }
|
||||
podSelector: { matchLabels: { k8s-app: kube-dns } }
|
||||
ports: [{ protocol: UDP, port: 53 }]
|
||||
- to: # 通过 egress gateway 出外网
|
||||
- podSelector: { matchLabels: { app: deerflow-egress-gateway } }
|
||||
```
|
||||
|
||||
### 5.3 Egress gateway
|
||||
|
||||
放一个集中的出口代理(envoy / squid)做:
|
||||
|
||||
- LLM endpoint 白名单(OpenAI / Anthropic / vLLM 内网)
|
||||
- MCP server 白名单(per-tenant 启用列表)
|
||||
- 搜索 API 白名单(Tavily / Jina / Brave / DuckDuckGo)
|
||||
- **黑名单**:169.254.169.254(cloud metadata)、10.0.0.0/8 / 172.16.0.0/12 / 192.168.0.0/16(内部网络,除非白名单)
|
||||
- 全量审计(按租户记录每次出网)
|
||||
|
||||
### 5.4 Pod Spec 关键字段
|
||||
|
||||
```yaml
|
||||
spec:
|
||||
runtimeClassName: gvisor # 或 kata-fc(premium)
|
||||
automountServiceAccountToken: false # 沙箱不该有 SA token
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: sandbox
|
||||
image: registry.deerflow.io/sandbox:v2.3.4@sha256:... # 强制 digest pin
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: { drop: [ALL] }
|
||||
volumeMounts:
|
||||
- { name: workspace, mountPath: /mnt/user-data }
|
||||
- { name: skills, mountPath: /mnt/skills, readOnly: true }
|
||||
volumes:
|
||||
- { name: workspace, emptyDir: { sizeLimit: 10Gi } }
|
||||
- { name: skills, emptyDir: { sizeLimit: 5Gi } } # 启动时从 S3 拉
|
||||
```
|
||||
|
||||
### 5.5 镜像签名
|
||||
|
||||
- CI 用 cosign sign 给 sandbox 镜像签名
|
||||
- K8s admission controller(policy-controller / Kyverno)启动 pod 前验证 cosign signature
|
||||
- 拒绝任何未签名 / 签名不匹配的镜像
|
||||
|
||||
### 5.6 SandboxAuditMiddleware 增强
|
||||
|
||||
现有 `SandboxAuditMiddleware` 记录了工具调用,多租户必须:
|
||||
|
||||
- 每条审计带 `(tenant_id, user_id, thread_id, tool_name, args_hash)`
|
||||
- 异步推到独立审计存储(不与业务库共用)
|
||||
- 保留至少 90 天(合规要求常见值)
|
||||
|
||||
---
|
||||
|
||||
## 6. 性能影响
|
||||
|
||||
| 指标 | 估计 |
|
||||
|---|---|
|
||||
| Pod 冷启动(gVisor + 镜像 pre-warm) | 1.5-3s |
|
||||
| Pod 冷启动(Firecracker) | 3-8s |
|
||||
| Bash 命令延迟(vs 宿主) | gVisor +5-15%;Firecracker +10-30% |
|
||||
| 文件 I/O(小文件) | gVisor 显著慢(系统调用拦截);用 emptyDir tmpfs 缓解 |
|
||||
| 网络(egress gateway) | +1-3ms 单跳 |
|
||||
|
||||
**冷启动是最敏感的指标**——靠两条路径优化:
|
||||
|
||||
1. **Pod prewarm**:每个 namespace 维护 N 个空闲 pod 池(`PodReadinessProbe` 通过即可服用,按需绑定 thread)
|
||||
2. **镜像层缓存**:每节点预拉镜像(DaemonSet image-puller)
|
||||
|
||||
---
|
||||
|
||||
## 7. 风险与缓解
|
||||
|
||||
| 风险 | 缓解 |
|
||||
|---|---|
|
||||
| gVisor 与某些 syscall 不兼容(agent bash 跑不动某些工具) | sandbox 镜像里预装常用工具;CI 跑兼容性测试集 |
|
||||
| 出网白名单维护负担 | Per-tenant MCP/搜索配置自动生成 NetworkPolicy;运维 admin UI 一键加 |
|
||||
| Firecracker 运维复杂 | 仅作为 premium 档位,不强制全量上 |
|
||||
| 节点 noisy neighbor(CPU 共享导致侧信道) | 高敏租户走专属 nodepool(taints/tolerations) |
|
||||
| Pod prewarm 池资源浪费 | 按租户活跃度动态调节池大小;闲置超过阈值缩到 0 |
|
||||
| 镜像供应链攻击 | Cosign 强制 + SBOM + 漏洞扫描 |
|
||||
|
||||
---
|
||||
|
||||
## 8. 推翻条件
|
||||
|
||||
切换到 **per-tenant Firecracker(默认)** 当且仅当:
|
||||
|
||||
1. 实测 gVisor 在某条关键 syscall 上有不可绕过的兼容性问题(且 sandbox 镜像无法预装替代品)
|
||||
2. 拿到合同要求"强物理隔离"的监管客户,付费档位要求覆盖运维成本
|
||||
3. 出现一次容器逃逸 PoC 影响多租户
|
||||
|
||||
切换到 **共享 Docker(极端简化)** 当且仅当:
|
||||
|
||||
- 公司决定退回单租户产品形态——多租户上线后基本不应回退
|
||||
|
||||
---
|
||||
|
||||
## 9. 默认假设
|
||||
|
||||
| 项 | 默认 |
|
||||
|---|---|
|
||||
| 集群 | EKS / ACK / GKE,K8s 1.29+ |
|
||||
| 沙箱 runtime | gVisor (runsc) |
|
||||
| Premium runtime | Kata Containers + Firecracker |
|
||||
| Pod 隔离粒度 | per-thread(不复用) |
|
||||
| 冷启动 SLO | P50 < 2s, P99 < 5s |
|
||||
| Pod CPU/Mem 上限 | 2 CPU / 4 GiB(单 pod) |
|
||||
| Namespace 配额 | 16 CPU / 32 GiB / 20 pods(按 plan 调) |
|
||||
| Egress 白名单数量 | <30 个域名 / 租户 |
|
||||
| 审计保留期 | 90 天热 + 1 年冷 |
|
||||
| 镜像签名 | cosign + Kyverno 强制 |
|
||||
@@ -0,0 +1,289 @@
|
||||
# ADR-003 · LLM Key 与计费模型
|
||||
|
||||
| 项目 | 内容 |
|
||||
|---|---|
|
||||
| 状态 | 草稿(Draft) · 2026-05-09 据审计修订 §4.2 / §4.3 / §4.4.2(`create_chat_model` 是 sync、`TokenUsageMiddleware` 不持久化) |
|
||||
| 决策日期 | TBD |
|
||||
| 决策者 | 产品 + CTO + 财务 |
|
||||
| 关联 ADR | ADR-001 数据隔离、ADR-005 存储拓扑、ADR-006 运行时与渠道 |
|
||||
| 关联审计 | [adr-vs-code-audit](./adr-vs-code-audit.zh-CN.md) |
|
||||
| 代码命名 | 本 ADR 写 `tenant_*` 表 / `tenant_id` 列,落代码统一读作 `workspace_*` / `workspace_id`(详 [workspace-schema-design §1](./workspace-schema-design.zh-CN.md#1-命名约定--workspace-vs-tenant)) |
|
||||
|
||||
---
|
||||
|
||||
## 1. 背景
|
||||
|
||||
DeerFlow 当前的 LLM 配置是**进程级全局**:`config.yaml` 写 `api_key: $OPENAI_API_KEY`,环境变量在容器启动时注入,所有用户共用同一个 key。`models/factory.py` 在 `create_chat_model()` 时做反射 + 环境变量替换。
|
||||
|
||||
多租户场景下,这套模式有四个问题:
|
||||
|
||||
1. **成本归因不清**:所有租户的 token 消费混在同一个 key 上,月底无法精确按租户计费
|
||||
2. **滥用风险大**:一个租户写循环 prompt 把 key 刷爆,所有租户一起被拉黑
|
||||
3. **客户合规问题**:部分企业要求"我的数据只能走我的 LLM 账户"(数据驻留 / 审计闭环)
|
||||
4. **MCP / 第三方 API key 同样问题**:Tavily / Firecrawl / Jina 的 key 也是全局的
|
||||
|
||||
需要决定:**LLM Key 由谁出?怎么算账?**
|
||||
|
||||
---
|
||||
|
||||
## 2. 决策
|
||||
|
||||
**采用混合模式(Hybrid BYO)**:
|
||||
|
||||
- **Free / Pro 套餐**:默认走 **平台 key**,按月配额限制(token / 并发 / 沙箱 CPU 秒)
|
||||
- **Enterprise / BYO 套餐**:客户上传自己的 LLM key(OpenAI / Anthropic / Azure / Bedrock),不限平台 quota,按"管理费 + 沙箱用量"收费
|
||||
|
||||
理由:
|
||||
|
||||
1. **降低小客户上手摩擦**——他们不想注册 OpenAI 账号、不想填信用卡,平台 key 体验最顺
|
||||
2. **大客户控制权要求**——他们已经有 OpenAI / Anthropic 企业合同(折扣 / 数据条款),希望复用
|
||||
3. **平台风险可控**——平台 key 有 quota 兜底;BYO key 客户自己滥用自己的额度
|
||||
4. **现有 `models/factory.py` 改造可控**——加一层 tenant 上下文 + secret vault 即可
|
||||
|
||||
---
|
||||
|
||||
## 3. 备选方案与拒绝理由
|
||||
|
||||
### A. 纯平台 key(所有租户共用,按 token 加价转售)
|
||||
|
||||
**拒绝。** 看似简单实则雷区遍地:
|
||||
|
||||
- 大客户必拒——数据合规审查过不去(数据流过你的 key 等于过你的账户)
|
||||
- 一旦 OpenAI 临时封号 / rate limit,全员宕机
|
||||
- 加价定价天花板低(客户自己注册也能买,溢价空间小)
|
||||
- 你要替每个客户做信用评估和反滥用,运维成本飙升
|
||||
|
||||
### B. 纯 BYO key(强制客户自带)
|
||||
|
||||
**拒绝。** 与早期增长冲突:
|
||||
|
||||
- Onboarding 多一步"去注册 OpenAI"——转化率显著下降
|
||||
- 试用客户体验差("还没用就要填卡")
|
||||
- 小客户会嫌烦直接放弃
|
||||
|
||||
### C. 平台 key + 严苛 quota(不开 BYO)
|
||||
|
||||
**部分采纳,作为 Free/Pro 默认。** 但不开 BYO 会卡企业客户,不能作为唯一选项。
|
||||
|
||||
---
|
||||
|
||||
## 4. 落地影响
|
||||
|
||||
### 4.1 Tenant Secrets 表(ADR-005 已含)
|
||||
|
||||
```sql
|
||||
tenant_secrets (
|
||||
tenant_id UUID FK,
|
||||
key VARCHAR(64), -- 'OPENAI_API_KEY' / 'ANTHROPIC_API_KEY' / 'TAVILY_API_KEY' / ...
|
||||
encrypted_value BYTEA, -- KMS 加密
|
||||
rotated_at TIMESTAMP,
|
||||
created_at TIMESTAMP,
|
||||
PRIMARY KEY (tenant_id, key)
|
||||
)
|
||||
```
|
||||
|
||||
加密策略:每个租户一个 KMS data key(DEK),KEK 在 AWS KMS / 阿里云 KMS 集中管。
|
||||
|
||||
### 4.2 改造 `create_chat_model()`
|
||||
|
||||
当前签名(伪):
|
||||
|
||||
```python
|
||||
def create_chat_model(name: str = None, *, thinking_enabled: bool, app_config: AppConfig = None) -> BaseChatModel:
|
||||
model_config = app_config.get_model_config(name)
|
||||
api_key = resolve_env_var(model_config.api_key) # 从 process env 取
|
||||
return reflect(model_config.use)(api_key=api_key, ...)
|
||||
```
|
||||
|
||||
改造后:
|
||||
|
||||
```python
|
||||
async def create_chat_model(
|
||||
name: str = None,
|
||||
*,
|
||||
thinking_enabled: bool,
|
||||
tenant_id: str = AUTO, # 从 ContextVar 取
|
||||
app_config: AppConfig = None,
|
||||
) -> BaseChatModel:
|
||||
tenant = resolve_tenant_id(tenant_id, method_name="create_chat_model")
|
||||
model_config = app_config.get_model_config(name)
|
||||
|
||||
# 优先级:tenant 自带 key > 平台 key(带 quota)
|
||||
api_key = await secret_vault.get(tenant, model_config.api_key_secret_name)
|
||||
if api_key is None:
|
||||
api_key = await platform_keys.get(model_config.api_key_secret_name)
|
||||
# 走平台 key 的 LLM 调用必须挂 QuotaMiddleware(见 4.4)
|
||||
|
||||
return reflect(model_config.use)(api_key=api_key, ...)
|
||||
```
|
||||
|
||||
> **sync → async 的连带影响**:当前 `create_chat_model` 是同步函数(`models/factory.py:50`)。改成 async 后所有调用点(lead_agent factory、`MemoryMiddleware` / `TitleMiddleware` / `SummarizationMiddleware` 等)都要同步改 await——这是一次跨多个文件的改动,不是单点 patch。phase-1 实现时按"factory 改 async + 一次性扫所有调用点 await"作为单个 PR 落地,不要分批,避免中间态不可运行。
|
||||
|
||||
### 4.3 Quota 表与 Usage 表(ADR-005 已含)
|
||||
|
||||
```sql
|
||||
tenant_quotas (
|
||||
tenant_id UUID,
|
||||
metric VARCHAR(32), -- tokens_monthly / runs_concurrent / sandbox_cpu_seconds_daily
|
||||
hard_limit BIGINT, -- 超过即拒绝
|
||||
soft_limit BIGINT, -- 超过即告警 / 降速
|
||||
PRIMARY KEY (tenant_id, metric)
|
||||
)
|
||||
|
||||
tenant_usage_daily (
|
||||
tenant_id UUID,
|
||||
date DATE,
|
||||
metric VARCHAR(32),
|
||||
model_name VARCHAR(64), -- 区分 OpenAI / Anthropic / 本地 vLLM
|
||||
value BIGINT,
|
||||
PRIMARY KEY (tenant_id, date, metric, model_name)
|
||||
)
|
||||
```
|
||||
|
||||
写入时机:
|
||||
|
||||
- token:**当前 `TokenUsageMiddleware` 只 log 不持久化**(`agents/middlewares/token_usage_middleware.py:268-275`);本 ADR 要求新增持久化路径,按 (tenant_id, model, date) 累加,并配合 `usage_category` 区分主对话 / 内部任务(参 ADR-006 §2.5)
|
||||
- 沙箱 CPU 秒:K8s metrics-server / Prometheus 抓取,每 5 min 聚合写入
|
||||
- 并发 runs:`RunManager` 启动/结束时增减计数器
|
||||
|
||||
### 4.4 新增 `QuotaMiddleware`
|
||||
|
||||
放在 lead_agent 中间件链最前(在 ThreadDataMiddleware 之后、SandboxMiddleware 之前),LLM 调用前检查:
|
||||
|
||||
```python
|
||||
class QuotaMiddleware(AgentMiddleware):
|
||||
async def before_model(self, state, runtime):
|
||||
tenant = get_current_tenant()
|
||||
usage = await usage_repo.current_month(tenant.id, "tokens_monthly")
|
||||
quota = await quota_repo.get(tenant.id, "tokens_monthly")
|
||||
|
||||
if quota.hard_limit and usage >= quota.hard_limit:
|
||||
raise QuotaExceeded(
|
||||
"Monthly token quota exhausted. Upgrade plan or wait for reset.",
|
||||
next_reset=first_day_of_next_month(),
|
||||
)
|
||||
if quota.soft_limit and usage >= quota.soft_limit:
|
||||
# 软限:仍允许调用,但触发告警 + 在 UI 显示警告
|
||||
await alert_soft_limit(tenant.id)
|
||||
```
|
||||
|
||||
`QuotaExceeded` 通过 `LLMErrorHandlingMiddleware` 转成 user-facing 错误(不是 500),保持一致性。
|
||||
|
||||
#### 4.4.1 悲观预扣 vs 事后结算("幽灵 token"问题)
|
||||
|
||||
`before_model` 只能看到"调用前累计用量",但 token 消耗是 `after_model` 才知道的——硬限到达时**最后一次调用一定超额**(典型可超 32k–200k token,单次可达 quota 的 1-5%)。处理:
|
||||
|
||||
| 阶段 | 动作 |
|
||||
|---|---|
|
||||
| `before_model` | **悲观预扣**:按 `model_max_input_tokens` 估上限(按 model 配置查表),累加到 "reserved" 列。reserved + actual ≥ hard_limit 时拒绝 |
|
||||
| `after_model`(非流式) | 拿到真实 token,把对应 reservation 从 reserved 移到 actual,差额返还 |
|
||||
| `after_model`(流式) | 流尾 `usage` event 到达时同上;流被 cancel 时按已收到的增量扣,剩余 reservation 释放 |
|
||||
|
||||
```sql
|
||||
tenant_usage_daily (
|
||||
...,
|
||||
metric VARCHAR(32), -- tokens_input / tokens_output / tokens_reserved
|
||||
value BIGINT,
|
||||
...
|
||||
)
|
||||
```
|
||||
|
||||
**reserved 不进账单**,只用于 quota gate。月底对账只看 `tokens_input` + `tokens_output`。
|
||||
|
||||
#### 4.4.2 流式 token 的提交时机
|
||||
|
||||
LangGraph SDK 的 `messages-tuple` 流模式按 chunk 推 delta。当前 `TokenUsageMiddleware` 在 `after_model` 一次性 **log**(不持久化)——多租户加上持久化后,这有两个隐患:
|
||||
|
||||
1. **客户端 abort 流时漏记**:用户关浏览器、SSE 断开 → middleware 没收到 `after_model` → token 漏算
|
||||
2. **provider 本身的 usage 帧晚到**:OpenAI/Anthropic 把 usage 放在最后一个 chunk;StreamBridge 必须在 finalizing 时强制等这一帧
|
||||
|
||||
强约束:
|
||||
|
||||
- StreamBridge 收到 abort/disconnect 信号时,**仍需等 LLM provider 流自然结束** + 把 usage 提交后再断 client(限超时 5s 兜底)
|
||||
- 提交时机:`after_model` 终态(成功/失败/abort)三选一时立刻 commit;不允许"等会话结束再批量"
|
||||
- 流式调用的 `usage_category`(见 ADR-006 §2.5)按触发中间件分类
|
||||
- 测试覆盖:`test_billing_stream_abort.py` 模拟客户端断连,断言 token 仍被记录
|
||||
|
||||
#### 4.4.3 内部 LLM 调用的归属
|
||||
|
||||
详见 ADR-006 §2.5。简表如下:
|
||||
|
||||
| 触发 | 计费归属 | usage_category |
|
||||
|---|---|---|
|
||||
| 主对话 | tenant | `main` |
|
||||
| MemoryMiddleware 抽取 | tenant | `memory` |
|
||||
| TitleMiddleware 起标题 | tenant | `title` |
|
||||
| SummarizationMiddleware 历史压缩 | tenant | `summarization` |
|
||||
| 平台 admin 主动 LLM 工具(健康检查等) | platform | `platform` |
|
||||
|
||||
`tenant_usage_daily` schema 在 §4.3 基础上补 `usage_category` 列。报表 UI 展示这五类分项,避免"为什么我没说话也产生 token"这类客户投诉。
|
||||
|
||||
### 4.5 计费对账
|
||||
|
||||
平台 key 模式下,"实际成本"和"客户账单"要分清:
|
||||
|
||||
| 维度 | 数据来源 | 用途 |
|
||||
|---|---|---|
|
||||
| **OpenAI 实际账单** | OpenAI API usage report(每日拉) | 与平台财务对账 |
|
||||
| **客户应付** | `tenant_usage_daily`(你自己记的) | 月底生成账单 |
|
||||
| **差额** | OpenAI 实际 - sum(客户应付) | 监控异常(>5% 触发审计) |
|
||||
|
||||
差额监控很重要——如果你少记了 token(比如 streaming 异常时漏记),平台会替客户埋单。每月对账。
|
||||
|
||||
### 4.6 BYO key 验证流程
|
||||
|
||||
客户填 key 时:
|
||||
|
||||
1. 加密前先做一次 test call(小 prompt,验证 key 有效)
|
||||
2. 通过后加密入库
|
||||
3. UI 显示"已配置"但永远不回显原始 key(防泄露)
|
||||
4. 提供轮换流程(rotate)和撤销流程(revoke)
|
||||
|
||||
---
|
||||
|
||||
## 5. 套餐建议(产品决策,仅参考)
|
||||
|
||||
| 套餐 | LLM Key | 月度 token quota | 并发 runs | 沙箱 CPU 秒/月 | 价格 |
|
||||
|---|---|---|---|---|---|
|
||||
| **Free** | 平台 key | 100k | 1 | 1k | $0 |
|
||||
| **Pro** | 平台 key | 5M | 5 | 50k | $X |
|
||||
| **Team** | 平台 key(按 token 转售) | 50M | 20 | 500k | $XX |
|
||||
| **Enterprise BYO** | 客户自带 | 不限 | 协商 | 协商 | $XXX 管理费 + 沙箱用量 |
|
||||
|
||||
具体数字由 PMM 和财务定,不在本 ADR 范围。
|
||||
|
||||
---
|
||||
|
||||
## 6. 风险与缓解
|
||||
|
||||
| 风险 | 缓解 |
|
||||
|---|---|
|
||||
| 平台 key 被某租户刷爆 | QuotaMiddleware 硬限 + 异常用量告警(>3σ)+ 单 run token 上限 |
|
||||
| 平台 key 被 OpenAI 临时封禁 | 多备份 key 轮询(多 tier API key)+ 多 provider 兜底(OpenAI 挂了切 Anthropic) |
|
||||
| BYO key 在 DB 泄露 | KMS 加密 + 审计每次 decrypt + 仅在沙箱 pod 启动时 inject 进环境,不出 pod |
|
||||
| 客户 BYO key 滥用导致他自己被 OpenAI 封 | 不归我们管(合同里写明) |
|
||||
| 计费错算(少记 token) | 月度对账 + 5% 阈值告警 + 流式调用结束时强制 commit |
|
||||
| 客户跨币种 / 退款 | 接 Stripe 完整闭环,不要自己手搓 |
|
||||
|
||||
---
|
||||
|
||||
## 7. 推翻条件
|
||||
|
||||
- **退回纯平台 key**:如果 BYO 流量 < 5%,可考虑下线 BYO 简化运维(但企业客户已签合同的不能强制迁回)
|
||||
- **强制 BYO**:如果平台 key 滥用/欺诈损失年化 > $X,关掉免费档
|
||||
- **per-tenant 物理 LLM 资源**:如果监管客户要求"专属推理实例",需要专用 vLLM / Bedrock provisioned throughput
|
||||
|
||||
---
|
||||
|
||||
## 8. 默认假设
|
||||
|
||||
| 项 | 默认 |
|
||||
|---|---|
|
||||
| Secret 加密 | 信封加密:DEK(per-tenant)+ KEK(KMS) |
|
||||
| Quota 维度 | tokens_monthly + runs_concurrent + sandbox_cpu_seconds_daily |
|
||||
| Quota 重置 | 月度 token 按 UTC 月初;并发是实时;沙箱秒按 UTC 日初 |
|
||||
| 软限:硬限比例 | soft = 0.8 × hard |
|
||||
| BYO 支持的 provider | OpenAI / Anthropic / Azure / AWS Bedrock / vLLM-compatible |
|
||||
| 计费货币 | USD(多币种由 Stripe 处理) |
|
||||
| 对账周期 | 每日抓取 OpenAI usage,月度核对 |
|
||||
| 单 run token 上限 | 平台 key:100k tokens;BYO:不限 |
|
||||
@@ -0,0 +1,374 @@
|
||||
# ADR-004 · 租户内层级与 RBAC
|
||||
|
||||
| 项目 | 内容 |
|
||||
|---|---|
|
||||
| 状态 | 草稿(Draft) |
|
||||
| 决策日期 | TBD |
|
||||
| 决策者 | 产品 + 后端 lead |
|
||||
| 关联 ADR | ADR-001 数据隔离、ADR-003 LLM Key 与计费 |
|
||||
| 关联审计 | [adr-vs-code-audit](./adr-vs-code-audit.zh-CN.md) — 现状:`users.token_version` + JWT `ver` claim 已存在;`@require_permission` 装饰器存在但 `owner_check` 是 bool 需扩为 enum;`MembershipCache` 30s LRU 全新建 |
|
||||
| 代码命名 | 本 ADR 写 `tenant_id` / `tenant_memberships`,落代码读作 `workspace_id` / `workspace_memberships`(详 [workspace-schema-design §1](./workspace-schema-design.zh-CN.md#1-命名约定--workspace-vs-tenant)) |
|
||||
|
||||
---
|
||||
|
||||
## 1. 背景
|
||||
|
||||
DeerFlow 当前的角色模型在 `users.system_role` 字段(`backend/packages/harness/deerflow/persistence/user/model.py:33`),只有两级:
|
||||
|
||||
- `admin`:首启账户、平台运维
|
||||
- `user`:普通注册用户
|
||||
|
||||
并且 admin 是**全平台级别**的——它能管所有用户。多租户化后,"管理员"概念要拆成两层:
|
||||
|
||||
- **平台 admin**:你(运营 DeerFlow SaaS 的人)的运维账号
|
||||
- **租户内角色**:客户公司内部的角色(owner / admin / member)
|
||||
|
||||
需要决定:**租户内要不要分角色?分多细?**
|
||||
|
||||
---
|
||||
|
||||
## 2. 决策
|
||||
|
||||
**采用二级 RBAC**:每个租户内有 `owner` / `admin` / `member` 三种角色。最小化但够用。
|
||||
|
||||
| 角色 | 数量 | 核心权限 |
|
||||
|---|---|---|
|
||||
| **owner** | 1 个/租户(可转让) | 删除租户、修改计费、转让所有权、管理 admin |
|
||||
| **admin** | 多个 | 邀请/移除 member、安装/启用 skill、配置 MCP、查看用量 |
|
||||
| **member** | 多个 | 跑 agent、上传文件、看自己的 thread、看本租户共享 skill |
|
||||
|
||||
---
|
||||
|
||||
## 3. 备选方案与拒绝理由
|
||||
|
||||
### A. 扁平(租户内不分角色)
|
||||
|
||||
**拒绝。** 简单但企业客户会立刻不满:
|
||||
|
||||
- 没法满足"我让员工用,但不让他们改 skill 配置"的需求
|
||||
- 没法做 SSO / SCIM 集成(IT 管理员要能批量管成员)
|
||||
- 客户拉新时(员工增多)会出现"谁负责清理"的问题
|
||||
|
||||
### B. 三级以上 RBAC(自定义角色 / 项目级权限)
|
||||
|
||||
**拒绝(起步阶段)。** 复杂度爆炸:
|
||||
|
||||
- 自定义角色 = 完整的 RBAC engine(角色继承、权限矩阵、UI 编辑器)
|
||||
- 项目/工作区级权限 = 还要再加一层组织
|
||||
- 这是 Enterprise 后期需求,第一年用不到
|
||||
|
||||
**保留作为后续扩展点**:等"自定义角色"成为销售卡点再做。
|
||||
|
||||
### C. 单一 admin(owner = admin = 同一个)
|
||||
|
||||
**拒绝。** owner 和 admin 必须分开:
|
||||
|
||||
- owner 是"账户所有权"——负责计费和租户存亡
|
||||
- admin 是"日常管理"——可以授权多个
|
||||
- 不分开会导致"所有 admin 都能删租户",运营失控
|
||||
|
||||
---
|
||||
|
||||
## 4. 权限矩阵
|
||||
|
||||
按"资源 × 动作"展开,下表是 MVP 矩阵(可扩展):
|
||||
|
||||
| 资源 | 动作 | owner | admin | member |
|
||||
|---|---|---|---|---|
|
||||
| **tenant** | view | ✓ | ✓ | ✓ |
|
||||
| | update_settings | ✓ | ✓ | |
|
||||
| | delete | ✓ | | |
|
||||
| | transfer_ownership | ✓ | | |
|
||||
| **billing** | view | ✓ | ✓ | |
|
||||
| | update_payment | ✓ | | |
|
||||
| | manage_byo_key | ✓ | ✓ | |
|
||||
| **members** | invite | ✓ | ✓ | |
|
||||
| | remove | ✓ | ✓ | |
|
||||
| | change_role | ✓ | | |
|
||||
| **threads** | create | ✓ | ✓ | ✓ |
|
||||
| | read_own | ✓ | ✓ | ✓ |
|
||||
| | read_others | ✓ | ✓ | |
|
||||
| | delete_own | ✓ | ✓ | ✓ |
|
||||
| | delete_others | ✓ | ✓ | |
|
||||
| **skills** | install | ✓ | ✓ | |
|
||||
| | enable_disable | ✓ | ✓ | |
|
||||
| | use | ✓ | ✓ | ✓ |
|
||||
| **mcp_servers** | configure | ✓ | ✓ | |
|
||||
| | use | ✓ | ✓ | ✓ |
|
||||
| **custom_agents** | create_for_self | ✓ | ✓ | ✓ |
|
||||
| | create_shared | ✓ | ✓ | |
|
||||
| | edit_others | ✓ | ✓ | |
|
||||
| **usage_reports** | view | ✓ | ✓ | |
|
||||
| **audit_log** | view | ✓ | ✓ | |
|
||||
|
||||
注意几个设计选择:
|
||||
|
||||
- **`threads:read_others`**:默认 admin 能看(合规审计需要),但建议加配置项让租户 owner 可关掉
|
||||
- **`custom_agents:create_for_self` 给 member**:每个成员可以建私人 agent,但要 admin 才能"共享给整租户"
|
||||
- **`skills:use` 给 member**:使用 admin 启用的 skill;不能自己装 / 关
|
||||
|
||||
---
|
||||
|
||||
## 5. 落地影响
|
||||
|
||||
### 5.1 数据模型(ADR-005 phase 0 plan 已含)
|
||||
|
||||
```sql
|
||||
-- 1 个用户可属于多个租户
|
||||
tenant_memberships (
|
||||
tenant_id UUID FK,
|
||||
user_id UUID FK,
|
||||
role VARCHAR(16), -- 'owner' | 'admin' | 'member'
|
||||
invited_by UUID,
|
||||
joined_at TIMESTAMP,
|
||||
PRIMARY KEY (tenant_id, user_id)
|
||||
)
|
||||
|
||||
-- 1 个租户必须有且仅有 1 个 owner
|
||||
CREATE UNIQUE INDEX idx_one_owner_per_tenant
|
||||
ON tenant_memberships (tenant_id) WHERE role = 'owner';
|
||||
|
||||
-- 当前用户的"默认租户"(用户登录时落进哪个 tenant context)
|
||||
ALTER TABLE users ADD COLUMN default_tenant_id UUID;
|
||||
```
|
||||
|
||||
### 5.2 JWT Payload 改造
|
||||
|
||||
当前 JWT 只有 `sub: user_id`。改造后:
|
||||
|
||||
```json
|
||||
{
|
||||
"sub": "<user_id>",
|
||||
"tid": "<tenant_id>", // 当前激活的租户
|
||||
"role": "admin", // 当前租户内的角色
|
||||
"tv": 5, // token_version(保留现有撤销机制)
|
||||
"iat": ...,
|
||||
"exp": ...
|
||||
}
|
||||
```
|
||||
|
||||
**为什么把 role 放 JWT**:避免每次请求都查 `tenant_memberships` 表;权限变化时通过 `token_version++` 强制踢人重登。
|
||||
|
||||
**租户切换**:用户在 UI 切换租户时,调 `POST /api/v1/auth/switch-tenant` → 服务端校验 membership → 重发 JWT(新 tid + role)+ 新 cookie。
|
||||
|
||||
#### 5.2.1 token_version 的存储与失效路径
|
||||
|
||||
`token_version` 不是 JWT 自带字段——需要在 `users` 表加一列:
|
||||
|
||||
```sql
|
||||
ALTER TABLE users ADD COLUMN token_version INT NOT NULL DEFAULT 0;
|
||||
```
|
||||
|
||||
签发 JWT 时把当前 `token_version` 写进 `tv` claim;验证 JWT 时**只读一次**(不每请求查 DB):
|
||||
|
||||
- 命中**短 cache**(30s LRU,key=user_id)→ 比对 cache 里的 token_version
|
||||
- cache miss → 查 DB 一次,写入 cache
|
||||
- DB 里的 `token_version` 与 JWT 内 `tv` 不一致 → 401,cookie 强制失效,重登
|
||||
|
||||
谁会 bump `token_version`:
|
||||
|
||||
| 触发 | 谁写 |
|
||||
|---|---|
|
||||
| Membership 撤销 | `DELETE /tenants/{tid}/members/{uid}` 时 `users.token_version += 1` |
|
||||
| 角色变更(admin → member 等) | 同上 |
|
||||
| 主动注销所有设备("踢出所有会话") | `POST /auth/sign-out-all` |
|
||||
| 密码修改 | `POST /auth/change-password` |
|
||||
| owner 转让冷静期满 | 双方都 bump |
|
||||
|
||||
#### 5.2.2 JWT 内 role 与每请求查 DB 的取舍
|
||||
|
||||
这两条**只能选一条**,本 ADR 选 **A:JWT 内 role + 30s cache + 敏感操作必查 DB**:
|
||||
|
||||
- **常规请求**(read thread、list skills 等):信 JWT,30s 内可能拿到过期权限——可接受
|
||||
- **写敏感资源**(删除 thread、修改 billing、邀请成员、安装 skill):装饰器 `@require_permission(strict=True)` 强制查 `tenant_memberships`,不走 cache
|
||||
- **撤销/降权后**:bump `token_version` 让 cache miss → 下一次请求 401 → 用户重登拿新 JWT
|
||||
|
||||
这等于"读路径乐观、写路径悲观",与 ADR-001 RLS 是双层兜底(JWT role 错了 RLS 还在 tenant 维度过滤,跨租户绝不会泄露)。
|
||||
|
||||
#### 5.2.3 cache 实现要点
|
||||
|
||||
```python
|
||||
# packages/harness/deerflow/persistence/membership/cache.py
|
||||
class MembershipCache:
|
||||
"""Per-process LRU, 30s TTL, key = (user_id,) → token_version."""
|
||||
_cache: TTLCache = TTLCache(maxsize=10_000, ttl=30)
|
||||
|
||||
async def get_token_version(self, user_id: str) -> int:
|
||||
cached = self._cache.get(user_id)
|
||||
if cached is not None:
|
||||
return cached
|
||||
version = await user_repo.get_token_version(user_id)
|
||||
self._cache[user_id] = version
|
||||
return version
|
||||
|
||||
def invalidate(self, user_id: str) -> None:
|
||||
"""主动失效——bump token_version 时调用。"""
|
||||
self._cache.pop(user_id, None)
|
||||
```
|
||||
|
||||
进程间不共享(K8s 多 pod 时 30s 内可能不一致——可接受,超过 30s 全部 pod 自动同步)。
|
||||
|
||||
**禁忌**:把 membership cache 做到 Redis 之类共享层。理由:失效广播复杂、租户隔离混乱、收益小。30s 不一致窗口比这个复杂度划算。
|
||||
|
||||
### 5.3 AuthMiddleware 注入两层 ContextVar
|
||||
|
||||
```python
|
||||
async def dispatch(request, call_next):
|
||||
if _is_public(request.url.path):
|
||||
return await call_next(request)
|
||||
|
||||
user = await get_current_user_from_request(request)
|
||||
jwt_payload = await verify_jwt(request)
|
||||
tenant_id = jwt_payload["tid"]
|
||||
role = jwt_payload["role"]
|
||||
jwt_tv = jwt_payload["tv"]
|
||||
|
||||
# 走 cache 比对 token_version:通常不查 DB
|
||||
current_tv = await membership_cache.get_token_version(user.id)
|
||||
if jwt_tv != current_tv:
|
||||
raise HTTPException(401, "Token revoked, please sign in again")
|
||||
|
||||
# ContextVar + request.state 注入
|
||||
set_current_user(user)
|
||||
set_current_tenant(tenant_id, role)
|
||||
request.state.user = user
|
||||
request.state.tenant_id = tenant_id
|
||||
request.state.role = role
|
||||
|
||||
return await call_next(request)
|
||||
```
|
||||
|
||||
**这里**只比对 `token_version`(30s cache),**不**每请求查 `tenant_memberships`。membership 状态变化通过 §5.2.1 的 bump 机制传导。
|
||||
|
||||
敏感写操作另走一层(`@require_permission(strict=True)`)—— 在路由 handler 里查 DB,详见 §5.2.2。
|
||||
|
||||
### 5.4 权限装饰器升级
|
||||
|
||||
现有 `@require_permission("threads", "read", owner_check=True)` 沿用,但语义升级:
|
||||
|
||||
```python
|
||||
@router.get("/{thread_id}")
|
||||
@require_auth
|
||||
@require_permission("threads", "read", owner_check="self_or_admin")
|
||||
# self_or_admin: 自己拥有的 thread 自由读;他人的需要 admin/owner
|
||||
async def get_thread(thread_id: str, request: Request):
|
||||
...
|
||||
|
||||
@router.delete("/tenants/{tid}/members/{uid}")
|
||||
@require_auth
|
||||
@require_permission("members", "remove", owner_check="admin_only", strict=True)
|
||||
# strict=True: 不走 cache,强制查最新 membership
|
||||
async def remove_member(...):
|
||||
...
|
||||
```
|
||||
|
||||
`owner_check` 的几种模式:
|
||||
|
||||
- `"self"`:必须是该资源的 user_id 创建者
|
||||
- `"self_or_admin"`:自己 / 当前租户的 admin/owner 都行
|
||||
- `"admin_only"`:仅 admin/owner
|
||||
- `"owner_only"`:仅 owner
|
||||
|
||||
`strict=True` 何时必加(不走 30s cache,强制查 DB):
|
||||
|
||||
- 任何修改 RBAC 的操作(邀请/移除/改角色/转让)
|
||||
- 任何修改计费 / billing 的操作
|
||||
- 删除 thread / 删除 skill / 删除 MCP server
|
||||
- owner 才能做的危险操作(删租户)
|
||||
|
||||
读操作和普通写操作(创建 thread、改自己的 memory)不需要 strict——cache 不一致最多窗口 30s,足够。
|
||||
|
||||
### 5.5 邀请流程
|
||||
|
||||
```
|
||||
1. admin 在 UI 输入 email + role → POST /api/v1/tenants/{tid}/invitations
|
||||
2. 后端写 invitations 表 → 发邮件(链接含 invitation_token)
|
||||
3. 受邀用户点击:
|
||||
a. 已注册 → 直接 attach membership
|
||||
b. 未注册 → 跳注册流程,注册成功后 attach
|
||||
4. 邀请 7 天过期,admin 可重发或撤销
|
||||
```
|
||||
|
||||
`invitations` 表已在 phase 0 plan 设计中。
|
||||
|
||||
### 5.6 Owner 转让
|
||||
|
||||
owner 是单一的,转让流程要谨慎:
|
||||
|
||||
```
|
||||
1. owner 在 UI 选定新 owner(必须是当前 admin)
|
||||
2. 系统给原 owner 发确认邮件 + 二次密码确认
|
||||
3. 24h 冷静期内可撤销
|
||||
4. 冷静期满 → membership 表事务交换:
|
||||
原 owner.role := 'admin'
|
||||
新 owner.role := 'owner'
|
||||
(单事务 + 唯一索引保证不会出现 2 个 owner)
|
||||
```
|
||||
|
||||
### 5.7 SSO / SCIM 预留
|
||||
|
||||
**SSO(SAML / OIDC)**:
|
||||
|
||||
- 复用现有 `oauth_provider` / `oauth_id` 字段(`UserRow`)
|
||||
- 新增 `tenant_sso_configs(tenant_id, provider, idp_url, cert, ...)`
|
||||
- 用户首次 SSO 登录时,自动 attach 到 IdP 配置的默认 tenant + 默认角色(通常 member)
|
||||
|
||||
**SCIM**(自动用户配置 / 撤销):
|
||||
|
||||
- 实现 `/api/scim/v2/Users` + `/api/scim/v2/Groups` 标准接口
|
||||
- IdP(Okta / Azure AD)push 用户增删 → 自动同步 `tenant_memberships`
|
||||
- **第一年可不实现**——SSO 已能覆盖大多数企业需求
|
||||
|
||||
---
|
||||
|
||||
## 6. 与现有 system_role 的关系
|
||||
|
||||
| 字段 | 含义 | 是否保留 |
|
||||
|---|---|---|
|
||||
| `users.system_role` | **平台级**角色:`platform_admin` / `user` | 保留 |
|
||||
| `tenant_memberships.role` | **租户内**角色:`owner` / `admin` / `member` | 新增 |
|
||||
|
||||
`platform_admin`(DeerFlow 运营人员)可以跨租户操作(结合 ADR-001 的 `BYPASSRLS` role),但操作必须审计。普通用户的 `system_role = 'user'`。
|
||||
|
||||
注意 **首次启动**逻辑改动:
|
||||
|
||||
- 旧:`/setup` 创建第一个 admin 用户
|
||||
- 新:`/setup` 创建第一个 `platform_admin` + 同时建一个名为 `default` 的租户,把这个用户设为 owner(兼容老部署)
|
||||
|
||||
---
|
||||
|
||||
## 7. 风险与缓解
|
||||
|
||||
| 风险 | 缓解 |
|
||||
|---|---|
|
||||
| 最后一个 owner 离职导致租户失控 | 不允许 owner 直接退出,必须先转让;提供平台 admin 强制转让的运维接口(带审计) |
|
||||
| 权限矩阵越改越复杂 | MVP 矩阵冻结半年;新需求先走"是否扁平角色能解决"评审 |
|
||||
| JWT 里 role 缓存与 DB 不一致 | membership 撤销时 bump `token_version`,cookie 失效 |
|
||||
| 跨租户用户切换场景容易误操作 | UI 显著标识当前激活租户(顶栏色块 + 租户名);危险操作再校验 |
|
||||
| SSO 接入工作量被低估 | SSO 列入 v2 路线图,明确不阻塞 v1 上线 |
|
||||
|
||||
---
|
||||
|
||||
## 8. 推翻条件
|
||||
|
||||
- **走向自定义角色**:销售反馈明确"我们大客户必须自定义 admin/operator/auditor"——升级到完整 RBAC engine
|
||||
- **走向项目/工作区**:客户内部需要"多个团队互相不可见"——加 workspace 层(tenant > workspace > user)
|
||||
- **退回扁平**:极简产品形态变化,弃用企业销售路线(不太可能)
|
||||
|
||||
---
|
||||
|
||||
## 9. 默认假设
|
||||
|
||||
| 项 | 默认 |
|
||||
|---|---|
|
||||
| 角色数 | 3 (owner/admin/member) |
|
||||
| Owner 数 | 严格 1 个/租户 |
|
||||
| Admin 数 | 不限(按 plan 可设上限) |
|
||||
| 跨租户成员 | 同一 user 可属于多个租户 |
|
||||
| JWT role claim | 缓存到 token 里,membership 变更走 token_version 失效 |
|
||||
| 默认新成员角色 | `member` |
|
||||
| Invitation TTL | 7 天 |
|
||||
| Owner 转让冷静期 | 24h |
|
||||
| SSO | v2 路线图(非 v1 阻塞) |
|
||||
| SCIM | 暂不实现 |
|
||||
| 自定义角色 | 暂不实现 |
|
||||
@@ -0,0 +1,406 @@
|
||||
# ADR-005 · 存储拓扑与持久化策略
|
||||
|
||||
| 项目 | 内容 |
|
||||
|---|---|
|
||||
| 状态 | 草稿(Draft) |
|
||||
| 决策日期 | TBD |
|
||||
| 决策者 | 架构 + 后端 lead + SRE |
|
||||
| 关联 ADR | ADR-001 数据隔离、ADR-002 沙箱隔离、ADR-006 运行时与渠道 |
|
||||
| 关联审计 | [adr-vs-code-audit](./adr-vs-code-audit.zh-CN.md) — 现状:`ObjectStorage` Protocol、7 张新表、KMS 抽象**全部不存在**,本 ADR 描述的是从 0 起的设计;phase-0 §3.5 已加"底座先行"骨架要求 |
|
||||
| 代码命名 | 本 ADR 写 `tenants/{tid}/...` prefix 与 `tenant_*` 表名,落代码统一读作 `workspaces/{wid}/...` 与 `workspace_*`(详 [workspace-schema-design §1](./workspace-schema-design.zh-CN.md#1-命名约定--workspace-vs-tenant)) |
|
||||
|
||||
---
|
||||
|
||||
## 1. 背景
|
||||
|
||||
DeerFlow 当前所有持久化数据都在容器/主机的本地文件系统下(`{base_dir}/...`,默认 `./.deer-flow`,可用 `DEER_FLOW_HOME` 覆盖)。这套布局是为"单机/单副本本地开发"设计的——多租户上线后会出现三类问题:
|
||||
|
||||
1. **多副本不一致**:K8s 横向扩展后,副本 A 写的 `memory.json` 副本 B 看不到。
|
||||
2. **容器重建丢数据**:沙箱 pod 销毁、Gateway pod 重启即丢失自定义 skill / agent / memory / 上传 / 产物。
|
||||
3. **没有备份/灾备**:磁盘损坏即客户数据全失。
|
||||
|
||||
现状盘点(详见 `paths.py` 与 `skills/storage/`):
|
||||
|
||||
| 数据 | 现位置 | 容器重建会丢 |
|
||||
|---|---|---|
|
||||
| 公共 skills | repo 内 `skills/public/`(镜像里) | 否 |
|
||||
| 自定义 skills(用户安装) | `skills/custom/`(**全局共享**,非 per-user) | **是** |
|
||||
| 自定义 agent(SOUL.md + config.yaml) | `{base_dir}/users/{uid}/agents/{name}/` | **是** |
|
||||
| 长期记忆 | `{base_dir}/users/{uid}/memory.json` | **是** |
|
||||
| 用户上传 | `{base_dir}/users/{uid}/threads/{tid}/user-data/uploads/` | **是** |
|
||||
| Agent 工作区(草稿) | `.../user-data/workspace/` | 是(可接受) |
|
||||
| Agent 产物 | `.../user-data/outputs/` | **是** |
|
||||
| 配置开关 | `extensions_config.json`(仓库根,全局) | **是** |
|
||||
| 会话/运行/反馈 | SQLAlchemy 表(DB) | 否 |
|
||||
| 用户/认证 | `users` 表(DB) | 否 |
|
||||
|
||||
---
|
||||
|
||||
## 2. 决策
|
||||
|
||||
**采用三层存储拓扑**:结构化数据进 Postgres,二进制大对象进对象存储,沙箱本地只保留运行期临时区。**沙箱 pod 因此是真正无状态的**——可被任意调度、滚动升级、销毁重建。
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Postgres (RLS) │
|
||||
│ ┌────────────────┐ ┌──────────────────┐ │
|
||||
│ │ users │ │ agent_configs │ ← SOUL.md 入库 │
|
||||
│ │ tenants │ │ memory_facts │ ← memory.json 入库│
|
||||
│ │ memberships │ │ memory_context │ │
|
||||
│ │ threads_meta │ │ tenant_secrets │ ← API key 加密入库│
|
||||
│ │ runs │ │ tenant_skill_state│ │
|
||||
│ │ run_events │ │ tenant_mcp_configs│ │
|
||||
│ │ feedback │ │ tenant_quotas │ │
|
||||
│ └────────────────┘ └──────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ 对象存储(S3 / OSS / MinIO) │
|
||||
│ deerflow-data/{tenant_id}/... │
|
||||
│ uploads/{thread_id}/... ← 客户上传 │
|
||||
│ threads/{thread_id}/outputs/...← agent 产物 │
|
||||
│ deerflow-skills/{tenant_id}/ │
|
||||
│ {skill_name}-{version}.skill ← 技能包源 │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ 沙箱 Pod 本地(容器 emptyDir / 临时卷) │
|
||||
│ /mnt/user-data/workspace/ ← 中间草稿,run 结束清掉 │
|
||||
│ /mnt/user-data/uploads/ ← 启动时从 S3 拉,按需 │
|
||||
│ /mnt/user-data/outputs/ ← 写完后同步到 S3 │
|
||||
│ /mnt/skills/ ← 启动时按租户 enabled 列表拉│
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 2.1 各类数据的归属
|
||||
|
||||
**A. 进 Postgres(结构化、要查询、要并发更新、不大)**
|
||||
|
||||
| 数据 | 表 | 备注 |
|
||||
|---|---|---|
|
||||
| 自定义 agent 的 `SOUL.md` + `config.yaml` | `agent_configs(tenant_id, user_id, agent_name, soul_md TEXT, config_yaml TEXT, version, updated_at)` | 文本不大;要并发改、版本回滚 |
|
||||
| memory.json 中的事实 | `memory_facts(tenant_id, user_id, fact_id, content, category, confidence, created_at, source)` | 要按 confidence/category 查询、去重、流式追加 |
|
||||
| memory.json 中的上下文 | `memory_context(tenant_id, user_id, work_context, personal_context, top_of_mind, ...)` | 1 行 / (tenant, user) |
|
||||
| skill 启用状态 | `tenant_skill_state(tenant_id, skill_name, enabled, source)` | 替代全局 `extensions_config.json` |
|
||||
| MCP 配置 | `tenant_mcp_configs(tenant_id, server_name, transport, url, encrypted_config)` | 替代全局 `extensions_config.json` |
|
||||
| Tenant secrets(LLM key 等) | `tenant_secrets(tenant_id, key, encrypted_value)` | KMS 加密 |
|
||||
|
||||
**B. 进对象存储(大、二进制、写一次读多次、版本化天然)**
|
||||
|
||||
| 数据 | Object key | 备注 |
|
||||
|---|---|---|
|
||||
| 用户上传文件 | `tenants/{tid}/uploads/{thread_id}/{filename}` | presigned PUT 直传,沙箱按需拉 |
|
||||
| Agent 产物 | `tenants/{tid}/threads/{thread_id}/outputs/{path}` | `present_files` 触发上传 |
|
||||
| 技能包 `.skill` | `tenants/{tid}/skills/{skill_name}/{version}.skill` | SHA256 校验,做版本控制 |
|
||||
| 公共技能包(平台) | `platform/skills/{skill_name}/{version}.skill` | 跨租户共享 |
|
||||
|
||||
**C. 不持久化(彻底丢弃)**
|
||||
|
||||
| 数据 | 为什么 |
|
||||
|---|---|
|
||||
| 沙箱 `workspace/` 中间产物 | agent 的"草稿纸"——半成品脚本、临时调试输出。强行同步浪费带宽 + 增加爆炸半径。让 agent 主动 `present_files` 到 outputs 才同步 |
|
||||
| 解压后的 skill 目录 | 当本地缓存(LRU)。启动时从 S3 拉 .skill 包解压;缓存命中则跳过 |
|
||||
| Gateway 进程的本地缓存(mtime 失效那些) | 进程级,不需要持久化 |
|
||||
|
||||
---
|
||||
|
||||
## 3. 备选方案
|
||||
|
||||
### A. 全部进 PVC(共享文件存储 RWX,例如 EFS / NFS)
|
||||
**拒绝。** 看似最小改动,但:
|
||||
- 读写竞态依然存在(DeerFlow 的 atomic rename 在 NFS 上行为不一致)
|
||||
- 大量小文件读写性能差(memory.json 每次写需要 fsync)
|
||||
- 备份策略复杂(NFS 快照 vs 增量备份)
|
||||
- 退订清理很慢(递归删大量小文件)
|
||||
|
||||
### B. 全部进对象存储(连 metadata 都用 S3)
|
||||
**拒绝。** 用对象存储模拟文件系统:
|
||||
- 强一致性差(多数对象存储是 read-after-write,没有 CAS)
|
||||
- 对小文件高频写延迟太高(每次写 50–200ms)
|
||||
- 没有事务,跨对象一致性靠应用层补
|
||||
- 列表操作慢(list-objects 是分页拉取)
|
||||
|
||||
### C. 分层:DB(结构化)+ S3(二进制)+ 临时区(中间产物)
|
||||
**采纳。** 各取所长,与 ADR-001(行级隔离 + RLS)天然契合,并让沙箱 pod 真正无状态。
|
||||
|
||||
---
|
||||
|
||||
## 4. ObjectStorage 接口草稿
|
||||
|
||||
抽象层放在 `backend/packages/harness/deerflow/storage/`。所有调用方写抽象接口,**不直接 import boto3**。
|
||||
|
||||
```python
|
||||
# packages/harness/deerflow/storage/protocol.py
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ObjectMetadata(BaseModel):
|
||||
key: str
|
||||
size: int
|
||||
etag: str
|
||||
content_type: str
|
||||
last_modified: datetime
|
||||
metadata: dict[str, str] = {} # 自定义元数据(x-amz-meta-*)
|
||||
|
||||
|
||||
class ObjectNotFound(Exception):
|
||||
"""对象不存在(幂等删除/读取时由实现转换为此异常或返回 None)。"""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ObjectStorage(Protocol):
|
||||
"""租户感知的对象存储抽象。
|
||||
|
||||
实现:
|
||||
- LocalObjectStorage 开发/单元测试用,落本地目录
|
||||
- S3ObjectStorage AWS S3 / 兼容接口(OSS / OBS / R2)
|
||||
- MinIOObjectStorage 自部署 MinIO(S3 协议)
|
||||
|
||||
Key 命名规范(强约束,便于按 prefix 退订清理):
|
||||
tenants/{tenant_id}/...
|
||||
platform/... ← 跨租户共享资源(公共技能包)
|
||||
"""
|
||||
|
||||
# ── 同步 / 异步统一为 async(实现里用 aioboto3 / 自托管 thread pool) ──
|
||||
|
||||
async def put(
|
||||
self,
|
||||
key: str,
|
||||
data: bytes,
|
||||
*,
|
||||
content_type: str = "application/octet-stream",
|
||||
metadata: dict[str, str] | None = None,
|
||||
) -> ObjectMetadata: ...
|
||||
|
||||
async def put_stream(
|
||||
self,
|
||||
key: str,
|
||||
stream: AsyncIterator[bytes],
|
||||
*,
|
||||
content_type: str = "application/octet-stream",
|
||||
content_length: int | None = None,
|
||||
metadata: dict[str, str] | None = None,
|
||||
) -> ObjectMetadata: ...
|
||||
|
||||
async def get(self, key: str) -> bytes: ...
|
||||
|
||||
async def get_stream(self, key: str) -> AsyncIterator[bytes]: ...
|
||||
|
||||
async def stat(self, key: str) -> ObjectMetadata | None:
|
||||
"""读取元数据;不存在返回 None(不抛异常)。"""
|
||||
|
||||
async def exists(self, key: str) -> bool: ...
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
"""幂等删除——不存在视为成功。"""
|
||||
|
||||
async def delete_prefix(self, prefix: str) -> int:
|
||||
"""按 prefix 批量删除,返回删除数量。
|
||||
|
||||
重要:仅供 tenant 退订 / GC 任务使用。生产实现应:
|
||||
- 校验 prefix 必须以 ``tenants/{tenant_id}/`` 开头
|
||||
- 限制最大并发与最大删除数(避免一次误删全库)
|
||||
- 异步分批,超时可继续
|
||||
"""
|
||||
|
||||
async def list(
|
||||
self,
|
||||
prefix: str,
|
||||
*,
|
||||
limit: int = 1000,
|
||||
continuation_token: str | None = None,
|
||||
) -> tuple[list[ObjectMetadata], str | None]:
|
||||
"""分页列出。第二个返回值是下一页 token,None 表示结束。"""
|
||||
|
||||
async def copy(self, src_key: str, dst_key: str) -> ObjectMetadata: ...
|
||||
|
||||
async def presigned_put_url(
|
||||
self,
|
||||
key: str,
|
||||
*,
|
||||
expires_in: int = 3600,
|
||||
content_type: str = "application/octet-stream",
|
||||
max_size_bytes: int | None = None,
|
||||
) -> str:
|
||||
"""给客户端生成预签名上传 URL。max_size_bytes 用于 S3 POST policy。"""
|
||||
|
||||
async def presigned_get_url(
|
||||
self,
|
||||
key: str,
|
||||
*,
|
||||
expires_in: int = 3600,
|
||||
content_disposition: str | None = None,
|
||||
) -> str:
|
||||
"""给客户端生成预签名下载 URL。
|
||||
|
||||
重要:对 HTML/SVG 等活动内容,调用方必须传入
|
||||
content_disposition='attachment; filename="..."'
|
||||
以保留当前 Gateway artifacts 路由的 XSS 防护策略。
|
||||
"""
|
||||
```
|
||||
|
||||
### 4.1 调用边界
|
||||
|
||||
```python
|
||||
# 应用层只 import 抽象,不知道底下是 S3 还是 local
|
||||
from deerflow.storage import ObjectStorage, get_storage
|
||||
|
||||
storage: ObjectStorage = get_storage() # 单例工厂,按 config.yaml 选实现
|
||||
|
||||
# 上传:从 sandbox outputs 同步到 S3
|
||||
await storage.put_stream(
|
||||
f"tenants/{tid}/threads/{thread_id}/outputs/{filename}",
|
||||
stream=open_async(local_path),
|
||||
content_type=guessed_mime,
|
||||
)
|
||||
|
||||
# 退订:清理整个租户(GC 任务)
|
||||
await storage.delete_prefix(f"tenants/{tid}/")
|
||||
|
||||
# 给前端 presigned upload
|
||||
url = await storage.presigned_put_url(
|
||||
f"tenants/{tid}/uploads/{thread_id}/{filename}",
|
||||
expires_in=900,
|
||||
content_type=mime,
|
||||
max_size_bytes=100 * 1024 * 1024,
|
||||
)
|
||||
```
|
||||
|
||||
### 4.2 配置
|
||||
|
||||
```yaml
|
||||
# config.yaml
|
||||
storage:
|
||||
use: deerflow.storage.s3:S3ObjectStorage # 反射加载,沿用现有模式
|
||||
bucket: deerflow-data
|
||||
region: ap-northeast-1
|
||||
endpoint_url: null # MinIO 时填自部署地址
|
||||
access_key_id: $S3_ACCESS_KEY_ID
|
||||
secret_access_key: $S3_SECRET_ACCESS_KEY
|
||||
encryption: SSE-KMS # SSE-S3 / SSE-KMS / null
|
||||
kms_key_id: $S3_KMS_KEY_ID # 用 KMS 时填
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. 落地改造点(实施清单)
|
||||
|
||||
按优先级排序,每条对应一个 PR。
|
||||
|
||||
> **分期映射**(与 [phased-rollout-by-scale](../02-rollout/phased-rollout-by-scale.zh-CN.md) 对齐):本 §5 的"第 1 阶段"≈ rollout 的 **Stage 2**(KMS + ObjectStorage + RLS 同期落地);"第 2 阶段"≈ rollout 的 **Stage 3**。Stage 0 / Stage 1 仅复用现有本地文件系统,不动 storage 拓扑。
|
||||
|
||||
### 第 1 阶段(必须;对应 rollout Stage 2)
|
||||
|
||||
1. **抽象 `ObjectStorage` 接口 + `LocalObjectStorage` 实现** — 走通端到端,开发/测试用本地目录跑,不阻塞迁移
|
||||
2. **memory.json 迁库**
|
||||
- 新增 `memory_facts` / `memory_context` 表
|
||||
- 改写 `agents/memory/storage.py` 用 repository
|
||||
- 移除 30s debounce 队列的"合并写"逻辑(DB 层不需要了,保留外部 LLM 抽取的 debounce)
|
||||
- 数据迁移脚本:扫现有 `memory.json` 文件 → 入库 → 校验 → 删源文件
|
||||
3. **agent SOUL/config 迁库**
|
||||
- 新增 `agent_configs` + `agent_config_history` 表
|
||||
- 改写 `setup_agent` / `update_agent` 工具用 repository
|
||||
- 数据迁移脚本:扫 `users/{uid}/agents/` → 入库
|
||||
4. **`extensions_config.json` 拆库**——这条比看起来大,连锁影响展开如下:
|
||||
|
||||
**a. 新表与仓储**
|
||||
- `tenant_skill_state(tenant_id, skill_name, enabled, source, version, updated_at)`
|
||||
- `tenant_mcp_configs(tenant_id, server_name, transport, url, encrypted_config, updated_at)`
|
||||
- 各自配 repository(仿 `persistence/user/` 的 ContextVar AUTO 模式)
|
||||
|
||||
**b. 替换文件 mtime 热重载机制**
|
||||
- 现状:`get_app_config()` + 多处 `_is_cache_stale()`(如 `mcp/cache.py:31`)靠文件 mtime 判失效
|
||||
- 改造:
|
||||
- `extensions_config.py` 保留为"平台默认开关"的载体(公司 ship 的默认值)
|
||||
- 运行时配置全部走 DB;mtime 失效信号改为 `tenant_*_configs.updated_at`(DB 单调递增)
|
||||
- 缓存层从模块级单例 → per-tenant LRU(详见 ADR-006 §2.2 / §2.3)
|
||||
|
||||
**c. Gateway 路由改写**
|
||||
- `app/gateway/routers/skills.py`:`PUT /skills/{name}/enable`、`POST /skills/install` → 全部改为按当前 tenant_id 写 `tenant_skill_state` / 上传 S3
|
||||
- `app/gateway/routers/mcp.py`:`PUT /mcp/servers/{name}` → 改为按当前 tenant_id 写 `tenant_mcp_configs`,写完调 `TenantMCPCache.invalidate(tenant_id)`
|
||||
- 鉴权:admin/owner 才能改(参照 ADR-004 §5.4 `strict=True`)
|
||||
|
||||
**d. AppConfig 反射链不动**
|
||||
- `config.yaml` 里的 model / sandbox / channels 等仍是平台级配置,不动
|
||||
- 只有"per-tenant 可定制的开关"挪到 DB(skill enabled、MCP server、tenant secrets)
|
||||
- 旧的 `extensions_config.json` 在迁移完成后**删文件 + 删读取代码**,CI 加禁字典阻止再被引用
|
||||
|
||||
**e. 数据迁移**
|
||||
- 写一次性脚本:扫现有 `extensions_config.json` → 写入 "legacy_tenant" 的 `tenant_skill_state` / `tenant_mcp_configs`
|
||||
- 校验所有租户 enabled list 与现状一致后,删除文件
|
||||
- 自部署单机用户:first-run upgrade 时自动跑此脚本
|
||||
|
||||
**f. 老用户/单机模式兼容**
|
||||
- 单机部署 = 1 租户("default");体验上不应感知"DB 化"——配置改完仍立即生效
|
||||
- 解决路径:tenant_*_configs 写入后,立即 invalidate 进程内 cache(同一个进程,没问题)
|
||||
- 多 pod 部署:靠 `updated_at` 版本号在请求路径上自动同步,无需广播
|
||||
|
||||
**估工**:原 ADR-005 列了 1 条 bullet 偏乐观;这块包含 c/d/e/f 四子项,**整体约 M 偏 L**(一周量级),不是 S。
|
||||
|
||||
### 第 1 阶段 / 第 2 阶段交界(rollout Stage 2 末 / Stage 3 头)
|
||||
|
||||
5. **`S3ObjectStorage` 实现** — 用 aioboto3,覆盖 protocol 全部方法
|
||||
6. **上传文件改 presigned 直传**
|
||||
- 前端拿 presigned PUT URL → 直传 S3
|
||||
- 后端收 "upload complete" 通知 → 异步触发 markitdown 转换 worker
|
||||
- 沙箱启动时按需 lazy 拉取(不预拉所有上传)
|
||||
7. **agent 产物 outputs 同步**
|
||||
- `present_files` 工具:上传 outputs 到 S3 + 返回 presigned GET URL(带 `Content-Disposition: attachment` 给 HTML/SVG)
|
||||
- sandbox 销毁时本地清理(emptyDir 自然回收)
|
||||
8. **技能包二进制迁 S3**
|
||||
- `POST /api/skills/install`:把 .skill 上传到 S3(带 SHA256 metadata),不再解压到本地全局目录
|
||||
- 启动时按 tenant skill list 从 S3 拉 + 校验 + 解压到 LRU 缓存
|
||||
|
||||
### 第 2 阶段(建议;对应 rollout Stage 3)
|
||||
|
||||
9. **退订 GC**:tenant 标记 deleted 后,30 天定时任务跑 `delete_prefix(f"tenants/{tid}/")` + DB cascade delete
|
||||
10. **跨区域复制 / CDN**:按客户分布加 region replica 或 CloudFront / OSS 加速域名
|
||||
11. **审计接入**:每次 storage 操作(put/delete/presigned)记审计日志,带 (tenant_id, user_id, key, op, request_id)
|
||||
|
||||
---
|
||||
|
||||
## 6. 一致性与失败模式
|
||||
|
||||
| 场景 | 行为 | 备注 |
|
||||
|---|---|---|
|
||||
| Run 中途崩溃,outputs 没上传完 | DB 里 run 标记为 failed;下次重试 / 用户重新发起 | outputs 是"声明产物",丢失中间状态可接受 |
|
||||
| S3 上传成功但 DB 写失败 | 后台 GC 扫"无 DB 引用的 S3 对象"清理 | 标准的 sweep 模式 |
|
||||
| DB 写成功但 S3 上传失败 | 应用层重试;持续失败则把 run 标 failed 并告警 | 不允许 DB 引用一个不存在的 S3 对象 |
|
||||
| 客户上传到 presigned URL 但没通知后端 | 后台 sweeper 扫"上传超时未关联 thread 的对象"清理 | TTL 1h |
|
||||
| Memory 抽取异步任务卡住 | DB 层不会有半成品(事务 commit 才落库) | 保留之前的可观测性 |
|
||||
|
||||
---
|
||||
|
||||
## 7. 风险与推翻条件
|
||||
|
||||
**风险:**
|
||||
- S3 调用延迟(上传/下载几十 MB 文件耗时,影响 sandbox 冷启动)→ 用 region 同区 + 内网 endpoint 缓解;首启延迟拿 LRU 命中率监控
|
||||
- presigned URL 泄露(被截获后任何人可上传/下载)→ TTL 短(≤1h)+ `max_size_bytes` 限制 + content-type 限制
|
||||
- 退订时 `delete_prefix` 误伤 → 强制 prefix 校验 + dry-run 模式 + 二次确认
|
||||
|
||||
**推翻条件:**
|
||||
1. 实测 sandbox 冷启动 P99 > 30s 且无法靠 LRU 缓存解决 → 改用 PVC 存技能包 + 共享只读卷
|
||||
2. 客户合规要求物理隔离(监管类客户) → 切 ADR-001 的 per-tenant DB 方案,对象存储也切 per-tenant bucket(vs prefix)
|
||||
3. 团队规模不足以维护 S3 + Postgres 两套基础设施 → 退回 PVC + 单一存储(接受单副本部署)
|
||||
|
||||
---
|
||||
|
||||
## 8. 默认假设
|
||||
|
||||
如无相反证据,按此推进:
|
||||
|
||||
| 决策项 | 默认值 |
|
||||
|---|---|
|
||||
| 对象存储实现 | S3 兼容接口(生产用 AWS S3 / 阿里 OSS,自部署用 MinIO) |
|
||||
| 加密 | SSE-KMS,每个租户一个 KMS key alias(可选,起步用 SSE-S3) |
|
||||
| Bucket 布局 | 单 bucket + tenant prefix(`tenants/{tid}/...`) |
|
||||
| Presigned URL TTL | 上传 15 min / 下载 1 h |
|
||||
| LRU 缓存大小 | 沙箱 pod 本地 5 GB(按 image size 调整) |
|
||||
| 退订保留期 | 软删除 30 天后 GC |
|
||||
| 技能包大小上限 | 50 MB |
|
||||
| 单上传文件上限 | 100 MB(presigned policy 强制) |
|
||||
@@ -0,0 +1,293 @@
|
||||
# ADR-006 · 运行时与渠道层的租户化
|
||||
|
||||
| 项目 | 内容 |
|
||||
|---|---|
|
||||
| 状态 | 草稿(Draft) · 2026-05-09 据 spike + 审计修订 §1 / §2.1 / §2.2 / §2.5 / §2.6 / §3 / §4 / §6 |
|
||||
| 决策日期 | TBD |
|
||||
| 决策者 | 后端 lead + 架构 + 渠道 owner |
|
||||
| 关联 ADR | ADR-001 数据隔离、ADR-003 LLM Key 与计费、ADR-005 存储拓扑 |
|
||||
| 关联 spike / 审计 | [adr-vs-code-audit](./adr-vs-code-audit.zh-CN.md) · [langgraph-postgres spike](./adr-spike-langgraph-postgres.zh-CN.md) |
|
||||
| 代码命名 | 本 ADR 写 `tenant_id` / `TenantMCPCache` / `tenant-{tenant_id}` namespace,落代码统一读作 `workspace_id` / `WorkspaceMCPCache` / `ws-{workspace_id}`(详 [workspace-schema-design §1](./workspace-schema-design.zh-CN.md#1-命名约定--workspace-vs-tenant)) |
|
||||
|
||||
---
|
||||
|
||||
## 1. 背景
|
||||
|
||||
ADR-001 ~ 005 解决了"数据/沙箱/Key/RBAC/存储"五个面,但 DeerFlow 还有几块**进程级、单例化、跨请求共享**的运行时组件,它们既不在仓储层也不在沙箱层,是 ADR-001 ~ 005 的"夹层",独立讲一遍才不漏:
|
||||
|
||||
| 组件 | 现状 | 现在的 key | 多租户后的问题 |
|
||||
|---|---|---|---|
|
||||
| LangGraph Checkpointer | 内置 `AsyncSqliteSaver` / `AsyncPostgresSaver`(`runtime/checkpointer/async_provider.py:73,117`,走 `from_conn_string`),表结构由 LangGraph 自己定 | `thread_id` | 表里只有 `thread_id`,没有 `tenant_id`;`langgraph-checkpoint-postgres==3.0.5` 不存在 `connection_factory`([spike](./adr-spike-langgraph-postgres.zh-CN.md) §2),`SET LOCAL` 注入路径不通;只能走应用层强校验 |
|
||||
| MCP 工具缓存 | `mcp/cache.py:11` 一个进程级 `_mcp_tools_cache: list[BaseTool]`,按 `extensions_config.json` 的 mtime 失效 | 无(全局) | 每个租户启用的 MCP server 不同;当前缓存命中第一个加载的租户配置 |
|
||||
| MCP OAuth token | `mcp/oauth.py:25-31` `OAuthTokenManager` token 缓存为**进程内存 `dict[str, _OAuthToken]`**,**无任何持久化**——进程重启后重新刷 token | 无(全局) | 多租户后必须按 tenant 隔离 + 持久化,从"无持久化"直接到"KMS 加密 DB"——比"文件挪到 DB"成本高 |
|
||||
| Skills loader | `skills/loader.py` 扫 `skills/public/` + `skills/custom/`,结果走 LRU;MCP 工具拼装在内 | 文件系统路径 | 多租户后 `skills/custom/` 不再是全局共享,要按租户维度拉/解压 |
|
||||
| Sandbox provider 单例 | `LocalSandboxProvider` / `AioSandboxProvider` 在 lifespan 创建一次,所有 thread 共享 | thread_id | ADR-002 切到 K8s 后是 per-tenant Namespace,provider 必须知道当前租户 |
|
||||
| Memory 抽取 LLM 调用 | `MemoryMiddleware` 30s debounce 后发 LLM 抽取事实 | 当前 thread 的 user_id | 这次调用的 token 算平台还是租户?BYO 时用谁的 key? |
|
||||
| Title / Summarization 内部 LLM 调用 | `TitleMiddleware` / `SummarizationMiddleware` 在 thread 上下文里复用主对话 LLM | 同上 | 同上 |
|
||||
| IM 渠道 ↔ 用户绑定 | `app/channels/store.py:36-42` 把 IM `channel:chat[:topic]` 映射到 `{thread_id, user_id}`;落 `${DEER_FLOW_HOME}/channels/store.json`(不是 `channels.yaml`),**当前没有 binding / workspace 概念** | platform user_id | Slack workspace / 飞书租户 / 企微 corp 怎么映射到平台 tenant?webhook 来流量时怎么决定 tenant 上下文?需要新建 `channel_bindings` 表 |
|
||||
|
||||
这一组不解决,ADR-001 的 RLS、ADR-003 的计费都是空中楼阁。
|
||||
|
||||
---
|
||||
|
||||
## 2. 决策(按组件分别给)
|
||||
|
||||
### 2.1 LangGraph Checkpointer
|
||||
|
||||
**决策**:保留 LangGraph 原生 checkpointer(不 fork、不 ALTER 它的表),租户隔离走**应用层强校验 + thread_id 唯一约束兜底**。详见 ADR-001 §4.1.1 两层模型。
|
||||
|
||||
> 历史背景:原稿设想用 `connection_factory` 注入 `SET LOCAL app.tenant_id` + 在 LangGraph 表上挂 RLS。spike 验证([adr-spike-langgraph-postgres](./adr-spike-langgraph-postgres.zh-CN.md))发现 `langgraph-checkpoint-postgres==3.0.5` 不存在 `connection_factory` 参数;备选方案(子类化 `psycopg_pool` / 包装 saver / ALTER 加列)都有不可接受的维护成本。改用应用层强校验。
|
||||
|
||||
具体落地三件事:
|
||||
|
||||
1. **入口路径强校验**:所有调 LangGraph saver 的入口——**`app/gateway/routers/threads.py`** 和 **`app/gateway/routers/thread_runs.py`**——在调用前必须先在 `threads_meta` 上做 `(tenant_id=current, thread_id=requested)` 查询;未命中即 404,命中后才放行。
|
||||
- 创建 thread:先在 `threads_meta` 写入 `(tenant_id, thread_id, user_id)`,依赖 `UNIQUE (tenant_id, thread_id)` 复合索引兜底防重;再调 LangGraph 创建对应 thread
|
||||
- 读/写 thread:先 SELECT `threads_meta`,命中后再放行
|
||||
- **不在 LangGraph 自有表上挂 RLS**——`SET LOCAL app.tenant_id` 只对 DeerFlow 自有表生效(ADR-001 §4.2)
|
||||
2. **CI boundary 测试**:`backend/tests/` 下新增 `test_langgraph_access_boundary.py`——静态扫描禁止任何路径 import LangGraph saver/client 而绕过 `threads.py` / `thread_runs.py`。LangGraph Studio 直连必须走相同入口或显式审批
|
||||
3. **活体测试**:`tests/test_checkpointer_tenant_isolation.py`——建两个租户、各创建一个 thread,互相通过对方 thread_id 调 `/api/threads/{tid}` / `/api/threads/{tid}/runs` 必须 404;同 tid 走自己路径必须正常。这是入口校验是否生效的回归网
|
||||
|
||||
**推翻条件**:
|
||||
- LangGraph 上游加入 `connection_factory` 或等价 hook → 切回"DeerFlow 表 + LangGraph 表统一 RLS"模型
|
||||
- 应用层校验在压测中暴露 perf 瓶颈 → 评估 fork checkpointer 自控 schema
|
||||
|
||||
### 2.2 MCP 工具缓存
|
||||
|
||||
**决策**:模块级单例改为 **per-tenant 多级缓存**。
|
||||
|
||||
```python
|
||||
# packages/harness/deerflow/mcp/cache.py
|
||||
|
||||
class TenantMCPCache:
|
||||
"""Per-tenant MCP 工具缓存。
|
||||
|
||||
第一级 key: tenant_id
|
||||
第二级 key: tenant_mcp_configs.updated_at(DB 版本,替代 mtime)
|
||||
"""
|
||||
_caches: dict[str, _CachedTools] = {}
|
||||
_lock: dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
|
||||
|
||||
async def get(self, tenant_id: str) -> list[BaseTool]:
|
||||
async with self._lock[tenant_id]:
|
||||
cached = self._caches.get(tenant_id)
|
||||
current_version = await mcp_config_repo.version(tenant_id)
|
||||
if cached and cached.version == current_version:
|
||||
return cached.tools
|
||||
# 失效或未初始化:拉 tenant_mcp_configs → 启 MultiServerMCPClient
|
||||
client = await build_mcp_client_for_tenant(tenant_id)
|
||||
tools = await client.get_tools()
|
||||
self._caches[tenant_id] = _CachedTools(tools=tools, version=current_version)
|
||||
return tools
|
||||
|
||||
async def invalidate(self, tenant_id: str) -> None:
|
||||
async with self._lock[tenant_id]:
|
||||
self._caches.pop(tenant_id, None)
|
||||
```
|
||||
|
||||
要点:
|
||||
|
||||
- 现在 `mcp/cache.py:31` 靠 `extensions_config.json` 的 mtime 判 stale;DB 化后用 `tenant_mcp_configs.updated_at`(或 `version` 列单调递增),失效信号走仓储层而不是文件系统。
|
||||
- `MultiServerMCPClient` 实例**也要 per-tenant 持有**,因为它内部缓存了到各 MCP server 的连接 + OAuth token。租户切换不能复用别的租户的连接。
|
||||
- **OAuth token 存储**:当前 `mcp/oauth.py:25-31` 的 token 是**进程内存 `dict[str, _OAuthToken]`,无任何持久化**——进程重启后重新刷 token。多租户化要直接做"租户隔离 + 持久化 + 加密"三步并发:落 `tenant_secrets`(`(tenant_id, key='mcp_oauth:<server_name>')`)+ KMS 加密 + 失败回退到刷新流程。工作量比"文件挪到 DB"高一档,估工 M+。
|
||||
- 进程内存上限:`TenantMCPCache` 加 LRU 上限(默认 1000 租户),超过淘汰最久未访问的;淘汰时关闭它的 MCP client 释放连接。
|
||||
- Gateway PUT mcp 路由(`app/gateway/routers/mcp.py`)改完写 DB 后,调 `TenantMCPCache.invalidate(tenant_id)` 主动失效。
|
||||
|
||||
### 2.3 Skills loader
|
||||
|
||||
**决策**:拆成 **平台 skills(共享只读)+ 租户 skills(隔离可写)** 两套。
|
||||
|
||||
```
|
||||
本地缓存目录 (LRU, 5GB):
|
||||
${DEER_FLOW_SKILLS_CACHE}/platform/{skill_name}-{version}/
|
||||
${DEER_FLOW_SKILLS_CACHE}/tenants/{tenant_id}/{skill_name}-{version}/
|
||||
```
|
||||
|
||||
启动/调用流程:
|
||||
|
||||
1. Sandbox pod 启动时,从 ADR-005 的对象存储拉两类技能包:
|
||||
- 平台启用列表:`platform/skills/{name}/{version}.skill`(公司维护,所有租户可见)
|
||||
- 租户启用列表:`tenants/{tid}/skills/{name}/{version}.skill`(租户私有)
|
||||
2. 解压到本地 LRU 缓存目录,挂到沙箱内的 `/mnt/skills/`(虚拟路径不变,对 agent 透明)
|
||||
3. 工具拼装时 `get_available_tools()` 按 `tenant_skill_state(tenant_id, skill_name, enabled)` 过滤
|
||||
4. 卸载/禁用:直接刷 `tenant_skill_state.enabled = false`,下一次 thread 启动时不挂载(在线 thread 不强制热卸,避免 in-flight 调用失败)
|
||||
|
||||
**LRU 淘汰策略**:按 `(tenant_id, skill_name)` 键淘汰;缓存满时优先淘汰非活跃租户的私有 skill,**永不淘汰 platform skills**(频繁命中)。
|
||||
|
||||
### 2.4 Sandbox provider 单例
|
||||
|
||||
**决策**:`SandboxProvider` 实例本身保持全局单例(每进程一个 K8s client),但 `acquire(thread_id)` 内部按 `tenant_id` 路由到对应 namespace:
|
||||
|
||||
```python
|
||||
class K8sSandboxProvider:
|
||||
async def acquire(self, thread_id: str) -> SandboxHandle:
|
||||
tenant_id = resolve_tenant_id(AUTO, method_name="acquire")
|
||||
namespace = f"tenant-{tenant_id}"
|
||||
# 从 per-tenant prewarm 池借 pod,没有则按 tenant 配额创建
|
||||
pod = await self._pool.borrow(namespace, thread_id)
|
||||
return K8sSandboxHandle(pod=pod, tenant_id=tenant_id, thread_id=thread_id)
|
||||
```
|
||||
|
||||
prewarm 池策略:
|
||||
|
||||
| 租户活跃度 | 池大小 |
|
||||
|---|---|
|
||||
| 7 天内有 thread 创建 | 按 plan 维度(free=0, pro=1, team=3, enterprise=可配)|
|
||||
| 闲置 > 24h | 缩到 0;下次冷启动接受 P95 < 5s |
|
||||
| 7 天内无活动 | 删除 namespace 的 prewarm 资源(保留 NS) |
|
||||
|
||||
`SandboxAuditMiddleware` 已经在 ADR-002 §5.6 加了 tenant_id;这里强调一句:**audit 写入不能复用业务 DB session**,要走独立 audit DB(避免业务回滚把审计也滚掉)。
|
||||
|
||||
### 2.5 Memory / Title / Summarization 的内部 LLM 计费
|
||||
|
||||
ADR-003 只覆盖了"主对话"的 LLM 调用计费,但 DeerFlow 的中间件链里有 3 处会**额外**触发 LLM:
|
||||
|
||||
| 中间件 | 触发时机 | 现状 token 归属 | 决策 |
|
||||
|---|---|---|---|
|
||||
| `MemoryMiddleware` | thread 闲置 30s 后异步抽取 | 主 LLM 配置 | **算入 tenant 用量**:用 tenant 当前 thread 的 model + key |
|
||||
| `TitleMiddleware` | 首轮回复后给 thread 起标题 | 主 LLM 配置 | **算入 tenant 用量**:成本可见,不要藏 |
|
||||
| `SummarizationMiddleware` | token 接近上限时压缩历史 | 主 LLM 配置 | **算入 tenant 用量**:和主对话不可分割 |
|
||||
|
||||
**统一原则**:凡是 tenant 触发的 thread 内部产生的 LLM 调用,**都计入该 tenant 的 quota 和账单**——理由:
|
||||
|
||||
1. 透明:客户能在 usage 报表里看到"主对话 vs 内部任务"分项,不奇怪
|
||||
2. 安全:不会有"租户用 quota 跑完后还能让平台贴钱抽 memory"的 bug
|
||||
3. BYO 一致:BYO 租户用自己的 key,平台不替他付任何 token
|
||||
|
||||
实现侧改造:
|
||||
|
||||
- 这 3 个中间件目前都通过 `create_chat_model()` 拿 LLM;ADR-003 §4.2 把这个函数改成 tenant-aware,自动会带上 tenant key
|
||||
- **`TokenUsageMiddleware` 当前只 log,不持久化**(`agents/middlewares/token_usage_middleware.py:268-275`);新增 `usage_category` 字段(`main` / `memory` / `title` / `summarization`)+ 持久化路径都要从空白起,写入 `tenant_usage_daily(tenant_id, date, usage_category, tokens_in, tokens_out)`。详见 ADR-003 §4.4
|
||||
- usage 报表 UI 区分这四类,让客户对账
|
||||
|
||||
**例外**:平台主动触发的 LLM 调用(比如平台 admin 跑健康检查时调用 LLM)算平台账,不算租户。
|
||||
|
||||
### 2.6 IM 渠道 ↔ 租户映射
|
||||
|
||||
这是产品形态决定的,方案分两档:
|
||||
|
||||
#### 形态 A:单租户独占一个 IM 集成(小客户/SaaS)
|
||||
|
||||
每个 Slack workspace / 飞书企业 / 钉钉 corp 绑到**一个** tenant。
|
||||
|
||||
> 现状澄清:当前 `app/channels/store.py:36-42` 只存 `channel:chat[:topic] → {thread_id, user_id}` 的 JSON 字典(路径 `${DEER_FLOW_HOME}/channels/store.json`),**没有 binding / workspace 概念**。下面的 `channel_bindings` 表是新建,不是扩展现有结构。
|
||||
|
||||
新建 `channel_bindings` 表:
|
||||
|
||||
```sql
|
||||
channel_bindings (
|
||||
id UUID PK,
|
||||
tenant_id UUID FK, -- 新增:每个绑定属于哪个租户
|
||||
platform VARCHAR(32), -- slack / feishu / dingtalk / wecom / telegram / discord / wechat
|
||||
external_workspace_id VARCHAR(128), -- Slack team_id / 飞书 tenant_key / 钉钉 corpId
|
||||
config_encrypted BYTEA, -- bot token / app secret 等
|
||||
status VARCHAR(16),
|
||||
created_by UUID,
|
||||
created_at, updated_at,
|
||||
UNIQUE (platform, external_workspace_id) -- 一个外部 workspace 只能绑一个 tenant
|
||||
)
|
||||
```
|
||||
|
||||
Webhook 路由(`app/channels/manager.py:740` 附近):
|
||||
|
||||
```python
|
||||
async def on_webhook(platform: str, payload: dict):
|
||||
external_id = extract_workspace_id(platform, payload)
|
||||
binding = await binding_repo.get(platform, external_id)
|
||||
if binding is None:
|
||||
return reject_unbound()
|
||||
# 把 tenant 上下文塞进去,再走 lead_agent
|
||||
set_current_tenant(binding.tenant_id, role=None) # IM 渠道无 RBAC,按 tenant 默认权限
|
||||
user = await resolve_or_create_im_user(binding.tenant_id, payload.user)
|
||||
set_current_user(user)
|
||||
...
|
||||
```
|
||||
|
||||
**关键**:IM 来的请求**不走 JWT**,所以 ContextVar 注入要靠 webhook handler 自己写,不能漏。`app/channels/auth_filter.py` 加一道中间层强制每次 webhook 必经 `set_current_tenant`。
|
||||
|
||||
#### 形态 B:多租户共享一个 IM 集成(企业平台)
|
||||
|
||||
平台只装一个 Slack App,多个客户公司装在自己 workspace;通过 Slack `team_id` 自动路由到对应 tenant。逻辑同 A,但 onboarding 流程不一样:客户跳 Slack OAuth → 回调时根据登录态的 `tenant_id` 写 binding。
|
||||
|
||||
**取舍**:起步先做 A(每租户独立 IM 集成)。形态 B 是 enterprise marketplace 上架后才需要,不在 v1 范围。
|
||||
|
||||
#### IM user ↔ platform user 的映射
|
||||
|
||||
```sql
|
||||
channel_user_links (
|
||||
tenant_id UUID FK,
|
||||
platform VARCHAR(32),
|
||||
external_user_id VARCHAR(128),
|
||||
user_id UUID FK, -- platform user
|
||||
linked_at,
|
||||
PRIMARY KEY (tenant_id, platform, external_user_id)
|
||||
)
|
||||
```
|
||||
|
||||
未链接的 IM 用户 → 自动建 ghost user(`tenant_memberships.role = 'member'`),首次发消息时让用户在 IM 里点链接确认 → 落库。
|
||||
|
||||
---
|
||||
|
||||
## 3. 落地改造清单(与 ADR-001/005 不重叠)
|
||||
|
||||
| 模块 | 改动 | 估工 |
|
||||
|---|---|---|
|
||||
| `app/gateway/routers/threads.py` + `thread_runs.py` | 入口注入 `(tenant_id, thread_id)` 校验;写路径先入 `threads_meta` | M |
|
||||
| 新建 `tests/test_checkpointer_tenant_isolation.py` | LangGraph 入口校验活体测试(不再是 RLS 测试) | S |
|
||||
| 新建 `tests/test_langgraph_access_boundary.py` | 静态扫描禁止绕过入口直连 saver | S |
|
||||
| `mcp/cache.py` | 模块级 → `TenantMCPCache` 类 | M |
|
||||
| `mcp/oauth.py` | OAuth token 从**进程内存** → `tenant_secrets`(持久化 + KMS 加密 + 失败回退到刷新) | M+ |
|
||||
| `app/gateway/routers/mcp.py` | PUT 接口改写 DB + 调 invalidate;不再依赖 `extensions_config.json` mtime | S |
|
||||
| `skills/loader.py` | 拆 platform / tenant 两路加载 | M |
|
||||
| `app/gateway/routers/skills.py` | install 路径改为按 tenant 写 DB + S3 | M |
|
||||
| `sandbox/k8s/provider.py`(ADR-002 新建) | `acquire` 注入 namespace = tenant | 已计入 ADR-002 |
|
||||
| Sandbox prewarm pool | 新增 controller,按 tenant 维度管理 | L |
|
||||
| `agents/middlewares/memory_middleware.py` | usage_category=memory | S |
|
||||
| `agents/middlewares/title_middleware.py` / `summarization_middleware.py` | usage_category=title/summarization | S |
|
||||
| `agents/middlewares/token_usage_middleware.py` | **从只 log 升级为持久化**(参 ADR-003 §4.4) + 增加 usage_category 维度 | M |
|
||||
| 新建 `channel_bindings` 表 + 仓储 | IM workspace ↔ tenant 映射(不复用 store.json) | M |
|
||||
| `app/channels/store.py` | 现有 `channel:chat → {thread_id, user_id}` 字典升级为 SQL 表,并按 tenant 分区 | M |
|
||||
| `app/channels/manager.py` | webhook handler 注入 tenant ContextVar | M |
|
||||
| 新建 `app/channels/auth_filter.py` | 强制 tenant 上下文 fail-closed | S |
|
||||
| 新建 `channel_user_links` 仓储 | IM user ↔ platform user | M |
|
||||
|
||||
合计:约 14 人周(2 人 7 周 / 3 人 5 周),不含 ADR-001 ~ 005 各自的改造。
|
||||
|
||||
---
|
||||
|
||||
## 4. 风险与缓解
|
||||
|
||||
| 风险 | 缓解 |
|
||||
|---|---|
|
||||
| **LangGraph 表无 RLS,应用层校验失效则跨租户**(§2.1 trade-off) | `threads_meta` `UNIQUE (tenant_id, thread_id)` 兜底;CI boundary 测试禁止绕过入口路由直连 saver;任何新增 LangGraph 直连路径必须 PR review |
|
||||
| LangGraph 升级换 schema | CI 锁版本;升级前先跑 tenant isolation 测试集 |
|
||||
| `TenantMCPCache` 内存爆 | LRU 上限 + 闲置淘汰;监控租户数 / 进程 |
|
||||
| MCP server 自身泄露租户上下文 | 出网走 ADR-002 egress gateway 白名单;MCP server 在沙箱内 stdio 启动时 env 不带平台 secret |
|
||||
| **MCP OAuth token 持久化是从无到有**,错误处理面更大 | 持久化失败时回退到"无持久化进程内存"模式(不阻塞业务)+ 报警;KMS 不可用时 fail-closed |
|
||||
| Memory 抽取算入租户用量被客户抗议"我没让它跑" | UI 显式开关 + 用量分项展示;默认开启可关 |
|
||||
| IM webhook 没注入 tenant 上下文 → 调用 RLS 全过滤掉空集 | 中间层 fail-closed;监控空集查询率 |
|
||||
| 多 IM 平台 token 在 `channel_bindings.config_encrypted` 泄露 | KMS 加密 + 审计每次 decrypt(参照 ADR-003 §4.6) |
|
||||
| ghost IM user 没绑回 platform user → 数据归到 ghost | 强制首次 IM 交互弹链接卡片 + N 天未确认自动停 |
|
||||
|
||||
---
|
||||
|
||||
## 5. 推翻条件
|
||||
|
||||
- LangGraph 官方推出原生多租户 checkpointer 或上游加入 `connection_factory` 等价 hook → 切回 ADR-001 §4.1.1 "DeerFlow 表 + LangGraph 表统一 RLS" 模型,废弃本 ADR §2.1 的应用层强校验作为唯一防线
|
||||
- 平台决定走"完全集中式 IM"(所有租户共用一个 bot account) → §2.6 切到形态 B 为唯一形态
|
||||
- MCP server 全部跑在 K8s sidecar 而非进程内 → §2.2 缓存策略需要重写,client 实例从进程内挪到 service mesh
|
||||
|
||||
---
|
||||
|
||||
## 6. 默认假设
|
||||
|
||||
| 项 | 默认 |
|
||||
|---|---|
|
||||
| Checkpointer | LangGraph PG saver 原状(不挂 RLS、不 ALTER 表)+ 入口路由强校验 + `threads_meta` `UNIQUE(tenant_id, thread_id)` 兜底 |
|
||||
| MCP cache | per-tenant LRU,上限 1000 租户/进程 |
|
||||
| MCP OAuth token | `tenant_secrets` 持久化 + KMS 加密;持久化失败回退到进程内存 + 报警 |
|
||||
| Skills | platform 公共只读 + tenant 私有;本地 LRU 5GB |
|
||||
| Sandbox prewarm | per-tenant 池,按 plan 配置大小 |
|
||||
| 内部 LLM 调用计费 | 全部记入 tenant,分 4 类 usage_category;`TokenUsageMiddleware` 必须先升级为持久化 |
|
||||
| IM 集成形态 | 形态 A:每租户独立 binding,UNIQUE(platform, external_workspace_id) |
|
||||
| IM ghost user TTL | 7 天未链接自动停 |
|
||||
| 审计 DB | 与业务 DB 物理分离(独立连接池或独立实例) |
|
||||
@@ -0,0 +1,381 @@
|
||||
# ADR-007 · URL 路由与前端租户化
|
||||
|
||||
| 项目 | 内容 |
|
||||
|---|---|
|
||||
| 状态 | 草稿(Draft) · 2026-05-09 据审计修订 §1 / §8 / §10 / §11 / §12(Better Auth 假设作废)|
|
||||
| 决策日期 | TBD |
|
||||
| 决策者 | 前端 lead + 后端 lead + 产品 |
|
||||
| 关联 ADR | ADR-001 数据隔离、ADR-004 RBAC、ADR-006 运行时与渠道 |
|
||||
| 关联审计 | [adr-vs-code-audit](./adr-vs-code-audit.zh-CN.md) |
|
||||
| 代码命名 | 本 ADR 写 `tenant_id` / JWT `tid`,落代码统一读作 `workspace_id` / JWT `wid`(详 [workspace-schema-design §1, §4](./workspace-schema-design.zh-CN.md)) |
|
||||
|
||||
---
|
||||
|
||||
## 1. 背景
|
||||
|
||||
ADR-001 ~ 006 锁定了数据/沙箱/Key/RBAC/存储/运行时——但客户最先看到的是**浏览器地址栏长什么样**。多租户产品形态决定了 URL 形态、cookie scope、登录跳转、auth 改造方式。
|
||||
|
||||
当前 DeerFlow(`frontend/src/`):
|
||||
- nginx 把 `/api/*` → Gateway 8001、`/api/langgraph/*` → 同 Gateway(重写)
|
||||
- 前端**没有 Better Auth**——auth 走后端自签 JWT:Gateway `app/gateway/auth/jwt.py` 签发 → `access_token` cookie(HttpOnly)→ 前端 `core/auth/server.ts:25-26` 读 cookie 调 `/auth/me`
|
||||
- JWT payload 当前结构:`{sub, exp, iat, ver}`(`auth/jwt.py:14-19`),**无 tid/role**
|
||||
- `users.token_version` 列已存在 + JWT `ver` claim 已用作失效(`persistence/user/model.py:49`)
|
||||
- CSRF 双重 cookie 已实现(`csrf_middleware.py` + 前端 `core/api/api-client.ts:20-32`)
|
||||
- 没有租户概念,单 host 单工作区
|
||||
- LangGraph SDK client 在 `core/api/api-client.ts` 单例,所有 thread 操作共用一个 SDK 实例
|
||||
|
||||
多租户后必须回答:
|
||||
|
||||
1. URL 怎么标记 tenant?
|
||||
2. 多 tenant 切换时 SDK 单例怎么处理?
|
||||
3. cookie 怎么 scope(避免跨 tenant session 串)?
|
||||
4. JWT 怎么扩展才能带 tenant_id 和 role?
|
||||
|
||||
> 审计纠正:原稿假设"前端用 Better Auth 走 cookie session"。`frontend/package.json` 实际不依赖 `better-auth`(grep 命中 0 次);前端 auth 是后端自签 JWT + `access_token` cookie 直通——所有"Better Auth 改造"段落都改为"扩展现有 `auth/jwt.py` `TokenPayload`",工作量更小。
|
||||
|
||||
---
|
||||
|
||||
## 2. 决策
|
||||
|
||||
**采用 path-based slug + 顶层 `TenantProvider` + 切换时强制刷新**。
|
||||
|
||||
| 维度 | 决策 |
|
||||
|---|---|
|
||||
| URL 形态 | `/{tenant_slug}/...`(如 `/acme/threads/abc-123`) |
|
||||
| 子域名(如 `acme.deerflow.app`) | 推迟到 v2,通过 `tenants.custom_domain` 列预留 |
|
||||
| Tenant 解析点 | nginx 不解析;后端 `AuthMiddleware` 从 path + JWT 双向交叉校验 |
|
||||
| Cookie scope | `Path=/`、不绑 tenant;通过 JWT 内 `tid` 区分 |
|
||||
| SDK 单例 | 全局单例,但切租户时 `await invalidate()` + 强制 reload |
|
||||
| Auth | 沿用现有 `app/gateway/auth/jwt.py` 自签 JWT;扩 `TokenPayload` 加 `tid` + `role` 字段、bump `ver` 触发旧 token 失效 |
|
||||
|
||||
---
|
||||
|
||||
## 3. 备选方案与拒绝理由
|
||||
|
||||
### A. 子域名(`acme.deerflow.app`)作为默认
|
||||
|
||||
**拒绝(默认)。** 几个硬伤:
|
||||
|
||||
- **本地开发劝退**:每个开发者要起 `*.localtest.me` 之类的通配 DNS,Docker compose 的 nginx 也要改
|
||||
- **TLS 证书**:通配证书或 ACME 动态签证;自部署客户卡这一步
|
||||
- **`access_token` cookie 跨子域**:要走 `Domain=.deerflow.app`,scope 太宽,租户隔离反而变弱
|
||||
- **CSRF 双重 cookie**:当前 csrf_middleware 假定同源;跨子域要改写
|
||||
|
||||
**保留**作为 enterprise plan 的"自定义域名"功能(vanity domain),通过 `tenants.custom_domain` 解析回平台 tenant,但不作为默认。
|
||||
|
||||
### B. Header `X-Tenant-Id`(无 URL 标记)
|
||||
|
||||
**拒绝。** 浏览器分享一个 thread URL 别人打不开(缺 header),UX 灾难;爬虫/SEO 也无法索引租户公开内容。
|
||||
|
||||
### C. URL 不带 tenant,全靠 session
|
||||
|
||||
**拒绝。** 用户多 tenant 切换后,浏览器 history 不可区分;同一个 URL 在不同会话里显示不同内容,BUG 报告噩梦。
|
||||
|
||||
---
|
||||
|
||||
## 4. URL 形态规范
|
||||
|
||||
```
|
||||
公开(不带租户):
|
||||
/ → 营销页
|
||||
/login → 登录页
|
||||
/signup
|
||||
/accept-invite/{token}
|
||||
/pricing
|
||||
|
||||
租户内:
|
||||
/{slug}/ → 租户首页(threads 列表)
|
||||
/{slug}/threads/{tid}
|
||||
/{slug}/skills
|
||||
/{slug}/mcp
|
||||
/{slug}/memory
|
||||
/{slug}/settings → 租户设置(owner/admin)
|
||||
/{slug}/settings/billing
|
||||
|
||||
平台 admin(system_role=platform_admin):
|
||||
/admin/tenants
|
||||
/admin/usage
|
||||
/admin/audit
|
||||
```
|
||||
|
||||
**slug 约束**:
|
||||
|
||||
- `^[a-z0-9](-?[a-z0-9])*$`,3-32 字符
|
||||
- 保留 slug 黑名单:`admin`、`api`、`auth`、`login`、`signup`、`pricing`、`docs`、`status`、`accept-invite`、`platform`
|
||||
- 大小写归一化:DB 存小写
|
||||
- 切换 slug:`tenants` 加 `slug_history` 表,30 天内老 slug 重定向到新 slug,过后 410
|
||||
|
||||
API 路径**不带 slug**:
|
||||
|
||||
```
|
||||
/api/v1/... # 业务 API(tenant 由 JWT / API key 决定;Stage 1 起强制带版本)
|
||||
/api/... # 旧路径,Stage 1 起转发到 /api/v1,Stage 3 sunset
|
||||
/api/langgraph/threads/{tid}/runs/stream # LangGraph 兼容(不带版本,跟随上游 SDK 约定)
|
||||
```
|
||||
|
||||
理由:API 是 SDK 调用的,不需要人类可读 URL;slug 只在浏览器导航/分享时有意义。
|
||||
|
||||
**`/api/v1/` 引入时机与设计**:详见 [headless-api-track §4](../02-rollout/headless-api-track.zh-CN.md#4-核心设计api-版本化) —— Stage 1 切换 mount prefix、保留旧路径转发并加 `X-API-Deprecated` header。
|
||||
|
||||
---
|
||||
|
||||
## 5. 后端:Path slug 与 JWT 的交叉校验
|
||||
|
||||
`AuthMiddleware` 当前从 cookie 读 JWT,注入 `user_id` 到 ContextVar。多租户后改:
|
||||
|
||||
```python
|
||||
async def dispatch(request, call_next):
|
||||
if _is_public(request.url.path):
|
||||
return await call_next(request)
|
||||
|
||||
payload = await verify_jwt_from_cookie(request)
|
||||
jwt_tid: str = payload["tid"]
|
||||
role: str = payload["role"]
|
||||
|
||||
# 1. API 调用:tenant 完全靠 JWT
|
||||
if request.url.path.startswith("/api/"):
|
||||
active_tid = jwt_tid
|
||||
|
||||
# 2. 页面导航:从 path 解析 slug → 反查 tenant_id
|
||||
else:
|
||||
slug = _extract_slug(request.url.path)
|
||||
if slug is None:
|
||||
active_tid = jwt_tid
|
||||
else:
|
||||
tenant = await tenant_repo.get_by_slug(slug)
|
||||
if tenant is None:
|
||||
raise HTTPException(404, "Tenant not found")
|
||||
# JWT tid 与 URL slug 不一致 → 强制重定向到正确 slug 或拒绝
|
||||
if tenant.id != jwt_tid:
|
||||
# 校验 user 是否是该 tenant 的成员
|
||||
membership = await membership_repo.get(tenant.id, payload["sub"])
|
||||
if membership is None:
|
||||
raise HTTPException(403, "Not a member of this tenant")
|
||||
# 是成员但 JWT 没切过来 → 重定向到 /switch-tenant
|
||||
return RedirectResponse(f"/auth/switch-tenant?to={slug}&next={request.url.path}")
|
||||
active_tid = tenant.id
|
||||
|
||||
set_current_user(...)
|
||||
set_current_tenant(active_tid, role)
|
||||
return await call_next(request)
|
||||
```
|
||||
|
||||
**关键**:page 路由用 path slug 校验,API 路由用 JWT tid——两条路只在登录时由 `/auth/switch-tenant` 触发同步。
|
||||
|
||||
---
|
||||
|
||||
## 6. 前端:TenantProvider + SDK 重建
|
||||
|
||||
### 6.1 顶层 Provider
|
||||
|
||||
```tsx
|
||||
// frontend/src/core/tenant/provider.tsx
|
||||
"use client";
|
||||
|
||||
export function TenantProvider({ children, tenantId, slug, role }: Props) {
|
||||
const value = useMemo(() => ({ tenantId, slug, role }), [tenantId, slug, role]);
|
||||
return <TenantContext.Provider value={value}>{children}</TenantContext.Provider>;
|
||||
}
|
||||
|
||||
export function useTenant() {
|
||||
const ctx = useContext(TenantContext);
|
||||
if (!ctx) throw new Error("useTenant() outside TenantProvider");
|
||||
return ctx;
|
||||
}
|
||||
```
|
||||
|
||||
挂载点:`app/(tenant)/[slug]/layout.tsx`:
|
||||
|
||||
```tsx
|
||||
export default async function TenantLayout({ params, children }) {
|
||||
const { slug } = await params;
|
||||
const session = await getSession();
|
||||
const tenant = await fetchTenantBySlug(slug);
|
||||
|
||||
if (!tenant) notFound();
|
||||
if (session.tid !== tenant.id) {
|
||||
// 同上:要么是路径错了,要么是切租户没同步
|
||||
redirect(`/auth/switch-tenant?to=${slug}`);
|
||||
}
|
||||
|
||||
return (
|
||||
<TenantProvider tenantId={tenant.id} slug={slug} role={session.role}>
|
||||
{children}
|
||||
</TenantProvider>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
### 6.2 LangGraph SDK 实例与 tenant 绑定
|
||||
|
||||
当前 `core/api/langgraph-client.ts` 是模块级单例。改造:**SDK 实例**保持单例(HTTP client 不需要重建),但**所有调用 wrapper**强制读 `useTenant()`,把 `tenantId` 作为对话 metadata 传递(实际 tenant 鉴权在后端 JWT,前端传只是为了请求溯源):
|
||||
|
||||
```ts
|
||||
// useThreadStream.ts
|
||||
export function useThreadStream(threadId: string) {
|
||||
const { tenantId } = useTenant();
|
||||
return useStream<ThreadState>(threadId, {
|
||||
apiUrl: "/api/langgraph",
|
||||
metadata: { tenant_id: tenantId }, // 仅用于日志/追踪,不替代鉴权
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
**租户切换时的清理**:
|
||||
- 切换前用 `cancelAllStreams()` 关掉所有打开的 SSE
|
||||
- 调用 `POST /api/auth/switch-tenant`(后端重发 JWT 新 cookie)
|
||||
- 拿到 200 后 `window.location.assign(/{newSlug}/)` 强制硬刷新
|
||||
|
||||
**为什么硬刷新**:
|
||||
- React state 里有大量缓存的 thread / skill / mcp 配置,按租户全洗一遍代码量大
|
||||
- LangGraph SDK 内部维护 EventSource 连接池,强制重建最干净
|
||||
- 一次 nav 1-2 秒可接受,远比 in-memory 切租户的边界 bug 划算
|
||||
|
||||
### 6.3 租户切换 UI
|
||||
|
||||
顶栏组件 `<TenantSwitcher>`:
|
||||
|
||||
- 列出 user 的所有 membership(来自 `/api/auth/me` 返回的 `tenants[]`)
|
||||
- 当前激活租户高亮 + 显著色块(避免误操作)
|
||||
- 点击切换 → 上面 6.2 的硬刷新流程
|
||||
|
||||
### 6.4 路由组与 Server Components
|
||||
|
||||
```
|
||||
frontend/src/app/
|
||||
├── (marketing)/ # 公开:/, /pricing
|
||||
├── (auth)/ # 登录注册:/login, /signup, /accept-invite
|
||||
├── (admin)/ # 平台 admin:/admin/...
|
||||
└── (tenant)/[slug]/ # 租户内:/{slug}/...
|
||||
├── layout.tsx # TenantProvider 注入
|
||||
├── page.tsx # threads 列表
|
||||
├── threads/[tid]/page.tsx
|
||||
├── skills/page.tsx
|
||||
├── settings/page.tsx
|
||||
└── ...
|
||||
```
|
||||
|
||||
`layout.tsx` 内的 `fetchTenantBySlug` 走 Server Component → 直连后端,缓存 60s(用 React `cache()`)。
|
||||
|
||||
---
|
||||
|
||||
## 7. Cookie 与会话
|
||||
|
||||
| 维度 | 决策 |
|
||||
|---|---|
|
||||
| Session cookie name | `access_token`(沿用现状,HttpOnly) |
|
||||
| Path scope | `/`(不绑 tenant slug) |
|
||||
| Domain | 平台主域(不跨子域) |
|
||||
| SameSite | `Lax`(默认) |
|
||||
| HttpOnly | 是 |
|
||||
| Secure | 是(生产) |
|
||||
| 切换 tenant | 后端**重签**新 JWT,覆写同一 cookie;不清旧 cookie |
|
||||
| 跨设备登录 | session 多设备 OK;切 tenant 不强制其他设备退出 |
|
||||
|
||||
**为什么 cookie 不绑 slug**:用户从 `/acme/...` 切到 `/bigco/...` 时如果 cookie path 不同,会出现"两个 cookie 同时存在浏览器但前端选错一个"的边界 case。统一 path=/ + JWT 内 `tid` 单一来源最干净。
|
||||
|
||||
**CSRF**:现有双重 cookie CSRF(`csrf_middleware.py`)保持,CSRF token 不需要按 tenant 区分。
|
||||
|
||||
---
|
||||
|
||||
## 8. Auth 改造(基于现有自签 JWT)
|
||||
|
||||
> 现状:`app/gateway/auth/jwt.py:14-19` `TokenPayload` 当前是 `{sub, exp, iat, ver}`;前端 cookie name 是 `access_token`;`users.token_version` 已存在,bump 该列即让所有旧 JWT 失效。
|
||||
>
|
||||
> **代码字段名以 [workspace-schema-design §4](./workspace-schema-design.zh-CN.md#4-jwt-tokenpayload--一次到位的字段集) 为准**:本 ADR 写 `tid`、落代码写 `wid`(同义)。Stage 0 PR2 已锁定 `wid` 命名 + Stage 0 一次性加齐 `wid` + `role` 两字段,避免 Stage 2 再 bump `token_version` 导致全用户重登。
|
||||
|
||||
多租户化改造:
|
||||
|
||||
1. **扩 `TokenPayload`**(字段名以 workspace-schema-design §4 为准):
|
||||
```python
|
||||
class TokenPayload(BaseModel):
|
||||
sub: str # user_id
|
||||
tid: str # tenant_id(新增;落代码读作 wid / workspace_id)
|
||||
role: str # owner | admin | member(新增)
|
||||
exp: int
|
||||
iat: int
|
||||
ver: int # bump 即让所有旧 token 失效(沿用)
|
||||
```
|
||||
`ver` 字段已经在用——任何 membership 变更(加入/退出/role 调整)都 bump `users.token_version`,下次请求 `access_token` 校验失败强制重新登录。
|
||||
2. **签发流程**:用户登录成功 → 检查 `tenant_memberships` 数量
|
||||
- 0 个:跳到 `/onboarding/create-tenant`(新用户首次登录)
|
||||
- 1 个:直接签 `{tid=该 tenant.id, role=membership.role}`,跳 `/{slug}/`
|
||||
- 多个:跳到 `/select-tenant` 让用户选;选后签对应 JWT,落 `users.default_tenant_id`
|
||||
3. **切换 tenant**:`POST /api/auth/switch-tenant` → 校验 membership → 重签 JWT 覆写 `access_token` cookie → 客户端硬刷新(§6.2)
|
||||
4. **SSO(v2)**:当前自签 JWT 模型可以直接配 SAML / OIDC provider,把外部 IdP 的 user/group 映射到 platform user + membership;不依赖 Better Auth,自由度更高
|
||||
5. **Invitation 流程**:`/accept-invite/{token}` 路径下点击 → 校验 invitation → 自动 attach membership + bump 用户 `token_version` → 跳到 `/{new_slug}/`
|
||||
|
||||
**为什么不引入 Better Auth**:现有 JWT 实现已经有 `token_version` 失效机制 + cookie HttpOnly + CSRF 双重 cookie,扩 2 个字段比引入新 auth 框架的破坏面小得多。引入 Better Auth 反而要重写 `auth_middleware.py` + 前端 `core/auth/` + 所有 server actions 调用——工作量多 1 倍。
|
||||
|
||||
---
|
||||
|
||||
## 9. 自定义域名(v2 预留)
|
||||
|
||||
`tenants` 表加 `custom_domain VARCHAR(253) UNIQUE NULL`。客户配置 CNAME 后:
|
||||
|
||||
1. 客户在 settings 里填域名
|
||||
2. 平台调 ACME 签证(per-domain)+ Caddy/nginx 动态 vhost
|
||||
3. 请求来时 nginx 看 Host 头:
|
||||
- 是平台主域 → 走 path slug 解析
|
||||
- 是 custom_domain → 反查 tenant_id 直接注入
|
||||
|
||||
不在 v1 范围。
|
||||
|
||||
---
|
||||
|
||||
## 10. 落地改造清单
|
||||
|
||||
| 模块 | 改动 | 估工 |
|
||||
|---|---|---|
|
||||
| `tenants.slug` + `slug_history` 表 | DB 迁移 | S |
|
||||
| `AuthMiddleware` path slug 解析 + 交叉校验 | 后端 | M |
|
||||
| `/api/auth/switch-tenant` 路由 | 后端 | S |
|
||||
| `/api/auth/me` 返回 tenant 列表 | 后端 | S |
|
||||
| `auth/jwt.py` `TokenPayload` 扩 `tid/role` 字段 + 签发流程 | 后端 | S |
|
||||
| 前端 `app/(tenant)/[slug]/layout.tsx` + Provider | 前端 | M |
|
||||
| 前端路由全部按 `(tenant)/[slug]/` 重组 | 前端 | L |
|
||||
| `useTenant()` hook + 所有 API 调用接入 | 前端 | M |
|
||||
| `<TenantSwitcher>` 组件 | 前端 | S |
|
||||
| 租户首登 onboarding `/onboarding/create-tenant` | 前端 + 后端 | M |
|
||||
| Tenant picker 页面 `/select-tenant` | 前端 | S |
|
||||
| 平台 admin `/admin/...` 路由 + 鉴权 | 前端 + 后端 | M |
|
||||
| 硬刷新切换流 + cancelAllStreams | 前端 | S |
|
||||
|
||||
合计:约 8 人周(前端 5 + 后端 3)。
|
||||
|
||||
---
|
||||
|
||||
## 11. 风险与缓解
|
||||
|
||||
| 风险 | 缓解 |
|
||||
|---|---|
|
||||
| slug 冲突(保留字 / 已注册) | 注册流程强制校验黑名单;冲突时返显建议 slug |
|
||||
| 浏览器分享 URL 给非成员看 | 后端 403,前端展示"申请加入"按钮 |
|
||||
| 切换 tenant 时 streams 没断干净导致看到上租户的 events | hard reload 兜底;E2E 测试 stream cancellation |
|
||||
| `TokenPayload` 字段升级导致旧 cookie 校验失败 | 加 fallback:旧 4 字段 token 视为"无 tenant 上下文",强制走 `/select-tenant` 重发 |
|
||||
| 自定义域名灰区(DNS / TLS) | v2 才做,v1 不实现 |
|
||||
| `default_tenant_id` 被删除(成员被踢) | 登录时 fallback 到 memberships 第一个;都没了引导建租户 |
|
||||
| SEO 收录租户页 | 默认 `noindex`,租户开关启用公开页 |
|
||||
|
||||
---
|
||||
|
||||
## 12. 推翻条件
|
||||
|
||||
- v2 决定走子域名优先 → §2 决策切到子域名 + 兼容老 path 形态 6 个月
|
||||
- 单页应用改成多页 / SSR 完整迁移 → 前端层重写,路由组结构会变
|
||||
- 决定改用 Better Auth / Auth.js 等成熟框架 → §8 重写为框架接入路径,但当前评估收益不抵迁移成本
|
||||
|
||||
---
|
||||
|
||||
## 13. 默认假设
|
||||
|
||||
| 项 | 默认 |
|
||||
|---|---|
|
||||
| URL 形态 | `/{slug}/...`,slug 3-32 字符小写 |
|
||||
| 自定义域名 | v2 才支持,v1 不开 |
|
||||
| Cookie path | `/` |
|
||||
| Cookie domain | 平台主域,不跨子域 |
|
||||
| 切换 tenant | 硬刷新(`window.location.assign`) |
|
||||
| Tenant picker | 1 个 membership 时跳过 |
|
||||
| API 路径 | 不带 slug(tenant 由 JWT 决定) |
|
||||
| SEO | 默认 `noindex`,可按租户开 |
|
||||
@@ -0,0 +1,170 @@
|
||||
# Spike: langgraph-checkpoint-postgres 的 per-acquire 注入能力验证
|
||||
|
||||
> 验证日期:2026-05-09
|
||||
> 触发原因:ADR-001 §4 / ADR-006 §2.1 假设 `langgraph-checkpoint-postgres>=2.0` 提供 `connection_factory`,可在每次连接 acquire 时注入 `SET LOCAL app.tenant_id`。本 spike 验证此假设是否成立,并给出可落地的备选路径。
|
||||
> 影响范围:ADR-001(数据隔离)、ADR-006(运行时与渠道层)
|
||||
|
||||
---
|
||||
|
||||
## 1. 验证基线
|
||||
|
||||
- **库版本**:`langgraph-checkpoint-postgres==3.0.5`(PyPI 上传时间 2026-03-18)
|
||||
- **当前调用点**:`backend/packages/harness/deerflow/runtime/checkpointer/async_provider.py:73,117`
|
||||
```python
|
||||
async with AsyncPostgresSaver.from_conn_string(conn_string) as saver:
|
||||
await saver.setup()
|
||||
yield saver
|
||||
```
|
||||
- **依赖锁定**:`backend/uv.lock` `>=3.0.5`(postgres extra)
|
||||
|
||||
## 2. 库实际 API 形态
|
||||
|
||||
直接解包 v3.0.5 wheel 检查源码:
|
||||
|
||||
### 2.1 `AsyncPostgresSaver.__init__`
|
||||
|
||||
```python
|
||||
# /tmp/lgcp/langgraph/checkpoint/postgres/aio.py:37-53
|
||||
def __init__(
|
||||
self,
|
||||
conn: _ainternal.Conn,
|
||||
pipe: AsyncPipeline | None = None,
|
||||
serde: SerializerProtocol | None = None,
|
||||
) -> None: ...
|
||||
```
|
||||
|
||||
`Conn` 类型定义(`_ainternal.py:10`):
|
||||
|
||||
```python
|
||||
Conn = AsyncConnection[DictRow] | AsyncConnectionPool[AsyncConnection[DictRow]]
|
||||
```
|
||||
|
||||
→ 可以传一个**自建的 `AsyncConnectionPool`** 进去;这是唯一一个有运行期介入空间的口子。
|
||||
|
||||
### 2.2 `from_conn_string` classmethod
|
||||
|
||||
```python
|
||||
# aio.py:55-80
|
||||
@classmethod
|
||||
@asynccontextmanager
|
||||
async def from_conn_string(
|
||||
cls,
|
||||
conn_string: str,
|
||||
*,
|
||||
pipeline: bool = False,
|
||||
serde: SerializerProtocol | None = None,
|
||||
) -> AsyncIterator[AsyncPostgresSaver]: ...
|
||||
```
|
||||
|
||||
→ **没有 `connection_factory` / `configure` / 任何 callback 参数**。这条路径完全封死。
|
||||
|
||||
### 2.3 `_cursor` 实现
|
||||
|
||||
```python
|
||||
# aio.py:352-392
|
||||
@asynccontextmanager
|
||||
async def _cursor(self, *, pipeline: bool = False):
|
||||
async with self.lock, _ainternal.get_connection(self.conn) as conn:
|
||||
...
|
||||
```
|
||||
|
||||
`get_connection` 行为(`_ainternal.py:13-23`):
|
||||
- `self.conn` 是单 `AsyncConnection`:始终复用同一条
|
||||
- `self.conn` 是 `AsyncConnectionPool`:每次现取(`async with conn.connection() as conn`)
|
||||
|
||||
→ 如果传 pool,**每次 cursor 调用确实独立 acquire**。
|
||||
|
||||
### 2.4 全局搜索
|
||||
|
||||
```bash
|
||||
grep -rn -E "connection_factory|configure_connection|on_acquire" /tmp/lgcp
|
||||
# 命中 0 处
|
||||
```
|
||||
|
||||
→ **库不存在 `connection_factory`**。ADR 写"langgraph-checkpoint-postgres>=2.0 支持 connection_factory"是事实错误。
|
||||
|
||||
## 3. 备选注入路径分析
|
||||
|
||||
### 3.1 Option A:自建 `AsyncConnectionPool` + 自定义 `getconn`
|
||||
|
||||
```python
|
||||
class TenantAwarePool(AsyncConnectionPool):
|
||||
@asynccontextmanager
|
||||
async def connection(self):
|
||||
async with super().connection() as conn:
|
||||
tenant_id = get_current_tenant() # ContextVar
|
||||
await conn.execute(f"SET app.tenant_id = '{tenant_id}'")
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
await conn.execute("RESET app.tenant_id")
|
||||
|
||||
pool = TenantAwarePool(conn_string, ...)
|
||||
saver = AsyncPostgresSaver(conn=pool)
|
||||
```
|
||||
|
||||
**可行性**:可以做。
|
||||
**问题**:
|
||||
- 侵入 `psycopg_pool` 内部,psycopg-pool 升级时要 retest
|
||||
- `psycopg_pool.AsyncConnectionPool.configure` callback 只在**物理连接首次创建**时跑,不能用——拿不到运行期 ContextVar
|
||||
- `SET app.tenant_id`(不带 LOCAL)会在 conn 复用时残留 → 必须在 release 前 `RESET`,否则跨租户泄漏
|
||||
- 每次 acquire 多 2 次 round-trip(SET + RESET),延迟成本不可忽略
|
||||
|
||||
### 3.2 Option B:包一层 `TenantAwarePostgresSaver`
|
||||
|
||||
在 `aput`/`aget`/`alist` 外面起事务 + `SET LOCAL`。
|
||||
|
||||
**问题**:
|
||||
- `AsyncPostgresSaver` 内部已有 `self.lock` + 自己的 cursor 管理(`_cursor` 走 `async with self.lock`),外面套事务和内部锁/事务结构会冲突
|
||||
- `SET LOCAL` 仅在事务内有效,需要确保 langgraph 内部所有 SQL 都跑在同一个事务里——库当前实现并不全是这样(pipeline 模式下行为不一)
|
||||
- 维护成本高,跟随 langgraph 升级风险大
|
||||
|
||||
### 3.3 Option C:放弃 langgraph 表上的 RLS(推荐)
|
||||
|
||||
把租户隔离拆成两层:
|
||||
|
||||
| 表归属 | 隔离机制 |
|
||||
|---|---|
|
||||
| **DeerFlow 自有表**(thread_meta / runs / feedback / users / 新增 tenant_*) | 标准 Postgres RLS + `SET LOCAL app.tenant_id` via SQLAlchemy session(DeerFlow 完全控制 conn pool) |
|
||||
| **langgraph checkpoint 表**(checkpoints / checkpoint_blobs / checkpoint_writes / checkpoint_migrations) | 纯应用层兜底——入口处(`threads.py` + `thread_runs.py`)强制 `(tenant_id, thread_id)` 校验,依赖 thread_meta 表上 `(tenant_id, thread_id)` unique constraint |
|
||||
|
||||
**优点**:
|
||||
- 不依赖 langgraph 库的任何 hook,库升级风险归零
|
||||
- ADR-001 §4 里讨论的"subquery RLS"和"column upgrade path"的纠结都消解了——subquery RLS 只能保护读,写仍然要应用层兜,两条路径在 Option C 下统一为"应用层兜 + 自有表 RLS"
|
||||
- 应用层校验点正好对齐审计报告里修正后的 ADR-001 hook 点(`threads.py` + `thread_runs.py`),不再绕错点
|
||||
- 实现路径短:thread_meta 加 `tenant_id` 列 + unique constraint + 入口校验 = 几十行;不用动 langgraph 表 schema
|
||||
|
||||
**不足**:
|
||||
- langgraph 表本身不是租户感知的,安全模型从"DB 强约束"降级为"应用层强约束"。需要承认这个 trade-off
|
||||
- 如果将来出现绕过 `threads.py` / `thread_runs.py` 的写入路径(例如 LangGraph Studio 直连),租户隔离失效。需要在 CI 里加 boundary 测试禁止此类直连
|
||||
|
||||
### 3.4 不推荐路径:column upgrade(修改 langgraph 表 schema)
|
||||
|
||||
ADR-001 §4 提到的"给 langgraph 表加 tenant_id 列再做 RLS"。
|
||||
|
||||
**风险**:langgraph 用 `MIGRATIONS` 数组管理 schema 升级(`aio.py:82-109`)。我们 ALTER 出来的列会跟库升级冲突——每次升 `langgraph-checkpoint-postgres` 都要 diff 一遍 `MIGRATIONS` 数组。**不接受这个长期维护成本**。
|
||||
|
||||
## 4. 结论与对 ADR 的修订建议
|
||||
|
||||
### 4.1 事实纠正
|
||||
|
||||
- ❌ **错误假设**:"langgraph-checkpoint-postgres>=2.0 支持 `connection_factory`"
|
||||
- ✅ **现实**:v3.0.5 不存在 `connection_factory`;`from_conn_string` 无 hook;唯一能介入的只有 `__init__(conn=AsyncConnectionPool)` 这一个口子,且 pool 自带的 `configure` callback 不是 per-acquire
|
||||
|
||||
### 4.2 推荐落地方案
|
||||
|
||||
**Option C:两层隔离模型**
|
||||
|
||||
- DeerFlow 自有表:RLS + `SET LOCAL` via SQLAlchemy(沿用 ADR-001 思路)
|
||||
- langgraph 表:应用层强校验(`threads.py` + `thread_runs.py`)+ thread_meta 表 unique constraint 兜底
|
||||
|
||||
### 4.3 待修订的 ADR 段落
|
||||
|
||||
- **ADR-001 §4**:删除 "subquery RLS" 和 "column upgrade path" 两段;替换为 "DeerFlow 表 RLS + langgraph 表应用层兜底" 的两层模型描述;hook 点保留审计报告修正后的 `threads.py` + `thread_runs.py`
|
||||
- **ADR-006 §2.1 改造点 B**:删除 "RunManager binding 到 saver 的 conn";明确 RunManager 不持有 langgraph conn pool,租户校验在 router 层完成
|
||||
- **ADR-006 §1 表格**:MCP OAuth token 的描述同步修正(参见审计报告 cross-cutting risk #3)
|
||||
|
||||
### 4.4 后续再确认事项(非阻塞)
|
||||
|
||||
- 如果未来需要 langgraph 表也走 RLS,可以**向上游 PR 加 `connection_factory` 参数**,比 fork 或 hack pool 都干净。当前 v3.0.5 不阻塞 phase-0
|
||||
- psycopg_pool 是否有更优雅的 per-acquire hook(v3.x 有无新增),可以在 phase-1 再 spike——phase-0 用不上
|
||||
@@ -0,0 +1,193 @@
|
||||
# 多租户改造 ADR 与现状代码审计报告
|
||||
|
||||
> 审计日期:2026-05-09
|
||||
> 审计基线:分支 `docs/multi-tenant-redesign` @ `dce5e959`
|
||||
> 目的:在动手写迁移代码之前,逐条核对 7 份 ADR + phase-0 计划对**当前代码**的假设是否成立,避免基于错误前提做架构决策。
|
||||
|
||||
---
|
||||
|
||||
## ADR-001: 数据隔离模型
|
||||
|
||||
### Assumptions about current code
|
||||
|
||||
1. 仓储层使用 `user_id` ContextVar + `AUTO` 哨兵自动注入 — **HOLDS**
|
||||
- Evidence: `backend/packages/harness/deerflow/runtime/user_context.py:135-167`(`AUTO` sentinel + `resolve_user_id`);`backend/packages/harness/deerflow/persistence/thread_meta/sql.py:35-41` 已按此模式调用。
|
||||
2. LangGraph checkpointer 使用 `AsyncPostgresSaver` 自带连接池 — **HOLDS**
|
||||
- Evidence: `backend/packages/harness/deerflow/runtime/checkpointer/async_provider.py:73,117` `AsyncPostgresSaver.from_conn_string(...)`;DeerFlow 不传 `connection_factory`。
|
||||
3. SQLAlchemy 引擎单例 + `AsyncSession` factory 已就位(即"DeerFlow 仓储侧连接池")— **HOLDS**
|
||||
- Evidence: `backend/packages/harness/deerflow/persistence/engine.py:26-27,126`;`init_engine_from_config` 支持 sqlite/postgres/memory。
|
||||
4. `AssistantsCompat` 路由可作为 thread_id↔tenant_id 校验拦截点 — **DOES NOT HOLD**
|
||||
- Evidence: `backend/app/gateway/routers/assistants_compat.py:1-50` 该路由仅服务 `/api/assistants` 的 `assistants.search/get` 静态 stub,**不**触达 thread。Thread 入口在 `backend/app/gateway/routers/threads.py` 与 `thread_runs.py`。
|
||||
- Reality: ADR 把 hook 点写错;实际拦截点应是 `threads.py` + `thread_runs.py`。
|
||||
5. 仓储层 `WHERE user_id` 已是默认行为,加一层 `tenant_id` 是平行扩展 — **PARTIALLY HOLDS**
|
||||
- Evidence: `persistence/thread_meta/sql.py:69-70` 是**应用层 `if row.user_id != resolved_user_id`** 后过滤,而非 SQL `WHERE`;其他仓储多数才走 SQL WHERE。RLS"前导列必须是 tenant_id"的索引前提目前完全不存在。
|
||||
6. 当前是 SQLite 默认,Postgres 是可选 backend — **HOLDS**
|
||||
- Evidence: `persistence/engine.py:80-114` 同时支持 sqlite/postgres/memory,README/CLAUDE 默认演示 sqlite。
|
||||
|
||||
### Highest-risk gaps
|
||||
|
||||
- **`AssistantsCompat` hook 点错位**:迁移时若按 ADR 字面落地,会跳过实际 thread 创建路径 (`threads.py:create`),导致 `(tenant_id, thread_id)` 应用层校验缺位、RLS 兜底成唯一防线。
|
||||
- **SQLite/Postgres 双驱动现状**:ADR §4.4 写"在 dev 同时跑双驱动"——目前 SQLite 是首选 backend,没有 Postgres 测试夹具 / RLS 测试基础设施,迁移启动成本被低估。
|
||||
|
||||
---
|
||||
|
||||
## ADR-002: 沙箱隔离模型
|
||||
|
||||
### Assumptions about current code
|
||||
|
||||
1. 存在两个 sandbox provider:`LocalSandboxProvider`(零隔离)+ `AioSandboxProvider`(Docker) — **HOLDS**
|
||||
- Evidence: `backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py`、`backend/packages/harness/deerflow/community/aio_sandbox/`、`sandbox/security.py:6-7` 把 LocalSandboxProvider 的 host bash 显式禁用。
|
||||
2. SandboxProvider 是进程级单例,`acquire(thread_id)` 在 lifespan 创建一次共享 — **HOLDS**
|
||||
- Evidence: `sandbox/sandbox_provider.py:41-58` `_default_sandbox_provider` + `get_sandbox_provider()` 单例;`sandbox/middleware.py:45-63` 直接调 `provider.acquire(thread_id)`。
|
||||
3. `SandboxAuditMiddleware` 已存在并记录工具调用 — **HOLDS**
|
||||
- Evidence: `backend/packages/harness/deerflow/agents/middlewares/sandbox_audit_middleware.py` 文件存在;CLAUDE.md L164 列在中间件链。
|
||||
4. K8s/Provisioner 模式存在但仅作为 sandbox 选项 — **PARTIALLY HOLDS**
|
||||
- Evidence: `backend/CLAUDE.md` 提到 `provisioner` port 8002 在配置 aio_sandbox+provisioner 时启动;但仓库中无 `K8sSandboxProvider`,没有 namespace/NetworkPolicy/gVisor 任何配套,威胁模型是"未来"而非"现状"。
|
||||
5. Sandbox audit 复用业务 DB session — **UNVERIFIABLE**(未深入审计中间件 DB 写入路径)
|
||||
|
||||
### Highest-risk gaps
|
||||
|
||||
- **`AioSandboxProvider` 出网/资源/cosign 全部缺位**:ADR §1 把它列为"起点不错",但实际上未禁出网、未限 CPU/memory、未 readOnly rootfs、未签名校验——MVP 多租户上线**绝不能直接复用现有 provider**。
|
||||
- **K8s Sandbox 几乎从零开工**:ADR-002 § 5.1 估"新建 K8sSandboxProvider"是单条 bullet,实际是子系统,估工 L+。
|
||||
|
||||
---
|
||||
|
||||
## ADR-003: LLM Key 与计费模型
|
||||
|
||||
### Assumptions about current code
|
||||
|
||||
1. `create_chat_model()` 是进程级模型工厂、配置走 `config.yaml` + 环境变量替换 — **HOLDS**
|
||||
- Evidence: `backend/packages/harness/deerflow/models/factory.py:50` 函数签名 `(name, thinking_enabled, *, app_config, **kwargs)`,无 tenant 参数;`models/factory.py` 通过 `resolve_class` 反射构造 LLM。
|
||||
2. 当前 `TokenUsageMiddleware` 在 `after_model` 一次性提交 token — **HOLDS**
|
||||
- Evidence: `agents/middlewares/token_usage_middleware.py:288-294` `after_model` / `aafter_model` 调 `_apply`;`_apply` 仅 log + 更新 `additional_kwargs`,**不**写 DB(更不区分 usage_category)。
|
||||
3. 存在 `MemoryMiddleware` / `TitleMiddleware` / `SummarizationMiddleware` 三处内部 LLM 调用 — **HOLDS**
|
||||
- Evidence: `agents/middlewares/{memory,title,summarization}_middleware.py` 全部存在(CLAUDE L169-170)。
|
||||
4. tenant_secrets / tenant_quotas / tenant_usage_daily 表已存在 — **DOES NOT HOLD**
|
||||
- Evidence: `persistence/{user,thread_meta,run,feedback}/model.py` 即全部 ORM 模型;无任何 tenant_* 表。
|
||||
5. ADR 描述的 `TokenUsageMiddleware` 已"按 message id 累加 token"写表 — **DOES NOT HOLD**
|
||||
- Evidence: `token_usage_middleware.py:268-275` 仅 logger.info,**未持久化** token;`runs/model.py:35-41` 的 `total_input_tokens` 在 `RunManager.update_run_completion` 时一次写 — 没有按租户/类别维度。
|
||||
|
||||
### Highest-risk gaps
|
||||
|
||||
- **没有任何用量持久化基础**:ADR-003 §4.4 的 `QuotaMiddleware`、`tokens_reserved`、悲观预扣全部要从空白起;现状 `TokenUsageMiddleware` 仅 log,幽灵 token 防御从零开工。
|
||||
- **`create_chat_model` 是同步函数**:ADR §4.2 改造目标签名是 async(要 await secret_vault.get),但当前是 sync——所有调用点(lead agent factory、memory updater 等)要同步改 async 或换 secret 注入路径。
|
||||
|
||||
---
|
||||
|
||||
## ADR-004: 租户 ↔ 用户层级与 RBAC
|
||||
|
||||
### Assumptions about current code
|
||||
|
||||
1. `users.system_role` 存在且仅 `admin`/`user` 两值 — **HOLDS**
|
||||
- Evidence: `persistence/user/model.py:33` `system_role: Mapped[str] = ... default="user"`;`auth/models.py:23` `Literal["admin", "user"]`。
|
||||
2. `users.token_version` 已存在用作 JWT 失效 — **HOLDS**
|
||||
- Evidence: `persistence/user/model.py:49` `token_version: Mapped[int] ... default=0`;`auth/jwt.py:18,36` JWT payload 已带 `ver` claim。
|
||||
3. JWT payload 当前结构是 `{sub, exp, iat, ver}`(无 tid/role)— **HOLDS**
|
||||
- Evidence: `app/gateway/auth/jwt.py:14-19,36`:`TokenPayload` 仅 4 字段,**没有 tid/role**。
|
||||
4. 已有 `@require_permission(resource, action, owner_check=...)` 装饰器,可以扩展 — **HOLDS**
|
||||
- Evidence: `app/gateway/authz.py:197-280`;现状 `owner_check` 是 `bool`,ADR §5.4 想升级为 `"self"|"self_or_admin"|"admin_only"|"owner_only"|strict=True`,需要重构。
|
||||
5. `AuthMiddleware` 在 ContextVar 注入 user — **HOLDS**
|
||||
- Evidence: `app/gateway/auth_middleware.py:122` `set_current_user(user)`;尚无 `set_current_tenant`。
|
||||
6. `tenant_memberships` 表存在 — **DOES NOT HOLD**
|
||||
- Evidence: `persistence/` 目录无 tenants/memberships/invitations 任何表。
|
||||
|
||||
### Highest-risk gaps
|
||||
|
||||
- **`token_version` 只是 column,无 cache 层 / membership 失效 path**:ADR §5.2.3 的 `MembershipCache` 30s LRU + bump 触发机制全部要新建。
|
||||
- **现有 `system_role="admin"` 是平台级管理员且唯一**:ADR §6 计划保留它做 platform_admin,但现状 admin 与"租户内 owner"语义未分离,迁移时首启逻辑("创建第一个 admin")会与新增"创建 default 租户 + 设其为 owner"耦合,需要兼容旧部署。
|
||||
|
||||
---
|
||||
|
||||
## ADR-005: 存储拓扑
|
||||
|
||||
### Assumptions about current code
|
||||
|
||||
1. memory.json 落 `{base_dir}/users/{user_id}/memory.json` 文件 — **HOLDS**
|
||||
- Evidence: `agents/memory/storage.py:84-102`;`config/paths.py:155-157` `user_memory_file()`。
|
||||
2. agent SOUL.md / config.yaml 落 `{base_dir}/users/{user_id}/agents/{name}/` — **HOLDS**
|
||||
- Evidence: `config/paths.py:163-169` user_agent_dir / user_agent_memory_file 系列;CLAUDE backend.md L356-359 描述一致。
|
||||
3. 自定义 skills 走 `skills/custom/` 全局共享、非 per-user — **HOLDS**
|
||||
- Evidence: `skills/storage/local_skill_storage.py:24-32` layout `<root>/{public,custom}/...`;`<root>` 来自 `config.skills.get_skills_path()`,没有 user_id 维度。
|
||||
4. `extensions_config.json` 在仓库根目录、被 mtime 失效驱动 — **HOLDS**
|
||||
- Evidence: `mcp/cache.py:11-53` `_config_mtime` + `_is_cache_stale`;`config/extensions_config.py` 存在;Gateway 路由 `routers/skills.py:321,336` / `routers/mcp.py:142,164` 直接读写文件并 `reload_extensions_config()`。
|
||||
5. 上传走本地 thread 目录 — **HOLDS**
|
||||
- Evidence: `uploads/manager.py:40-48` `get_paths().sandbox_uploads_dir(thread_id, user_id=...)`。
|
||||
6. `ObjectStorage` 抽象 / `LocalObjectStorage` / `S3ObjectStorage` 已存在 — **DOES NOT HOLD**
|
||||
- Evidence: `find ... -name "storage*"` 仅命中 `agents/memory/storage.py` 与 `skills/storage/`;harness 内**没有** `storage/protocol.py`、`storage/s3.py` 任何 ObjectStorage 抽象。
|
||||
7. `agent_configs` / `memory_facts` / `tenant_skill_state` 表存在 — **DOES NOT HOLD**
|
||||
- Evidence: `persistence/` 仅 `user/`/`thread_meta/`/`run/`/`feedback/`;ADR-005 §2.1 列出的 7 张新表全部不存在。
|
||||
|
||||
### Highest-risk gaps
|
||||
|
||||
- **三层拓扑全部要新建**:ObjectStorage 抽象(约 800 行 Protocol+实现)+ 7 张新表 + 4 个迁移脚本——ADR §5 第 1 阶段被列为 4 步实际是 12+ 步子项。
|
||||
- **memory/agent 文件 → DB 迁移会触发 `agents/memory/storage.py` 全面重写**:当前缓存键 `(user_id, agent_name)` + 原子 `temp+rename` 写 + `MemoryUpdateQueue` 30s debounce 全部假设文件系统语义。
|
||||
|
||||
---
|
||||
|
||||
## ADR-006: 运行时与渠道层的租户化
|
||||
|
||||
### Assumptions about current code
|
||||
|
||||
1. `mcp/cache.py:11` 是模块级单例 `_mcp_tools_cache: list[BaseTool]` — **HOLDS**
|
||||
- Evidence: `mcp/cache.py:11-14` 完全字面命中。
|
||||
2. MCP cache 失效靠 `extensions_config.json` mtime — **HOLDS**
|
||||
- Evidence: `mcp/cache.py:31-53` `_is_cache_stale` 比对 `os.path.getmtime`。
|
||||
3. `MultiServerMCPClient` 实例缓存在进程内 + 持有 OAuth token — **PARTIALLY HOLDS**
|
||||
- Evidence: `mcp/oauth.py:25-31` `OAuthTokenManager` token 缓存是 `dict[str, _OAuthToken]` **进程内存**;ADR-006 §2.2 / §1 表格写"OAuth token 落本地文件"——**不正确**,目前**没有持久化**,每次进程重启重新刷 token。
|
||||
- Reality: token 在 memory only,迁移时挪到 `tenant_secrets` 是从 0 起,比"从文件挪到 DB"成本更高(要新加持久化 + 加密)。
|
||||
4. `LocalSandboxProvider` / `AioSandboxProvider` 在 lifespan 创建一次单例 — **HOLDS**
|
||||
- Evidence: `sandbox/sandbox_provider.py:41-58` 全局单例 + `get_sandbox_provider`。
|
||||
5. `MemoryMiddleware` / `TitleMiddleware` / `SummarizationMiddleware` 都通过 `create_chat_model()` 拿 LLM — **HOLDS**(推断)
|
||||
- Evidence: 三个 middleware 文件存在;`models/factory.py:50` 是唯一工厂(CLAUDE 已说明),统一入口意味着 ADR §2.5 改造点统一。
|
||||
6. `app/channels/store.py` 把 IM 用户映射到平台 user_id,落 `~/.deer-flow/channels.yaml` — **PARTIALLY HOLDS**
|
||||
- Evidence: `app/channels/store.py:36-42` 默认路径是 `Paths.base_dir / "channels" / "store.json"`(不是 `channels.yaml`);存的是 `channel:chat → {thread_id, user_id}`。
|
||||
- Reality: 文件名是 `store.json`、且没有 `binding` 概念(IM workspace ↔ platform 映射),ADR §2.6 需新建 `channel_bindings` 表 + 重构 store。
|
||||
7. `RunManager` 创建/恢复 thread 前可以 issue `SET app.tenant_id` — **DOES NOT HOLD**(条件不具备)
|
||||
- Evidence: `runtime/runs/manager.py:41-78` 该类是**纯内存 run 注册表**,不持有 LangGraph 连接池。LangGraph saver 由 `make_checkpointer` 独立 lifespan 管理(`checkpointer/async_provider.py:73,117`),DeerFlow 无法控制每次 acquire;ADR §2.1 改造点 B 假设 RunManager 能 binding 到 saver 的 conn——目前没有这条 binding 通路。
|
||||
|
||||
### Highest-risk gaps
|
||||
|
||||
- **IM channel store 当前没有 binding 概念**(只有 `channel:chat → thread_id` 映射),ADR §2.6 描述的"webhook 来流量时按 binding 注入 tenant"需要先把 `store.json` 升级为 `channel_bindings` 表 + 重构 webhook handler 路径。
|
||||
- **LangGraph `SET LOCAL` 注入路径未验证**:ADR §2.1 说"langgraph-checkpoint-postgres>=2.0 支持 connection_factory"是假设,现状 `from_conn_string` 路径不传 factory;切换前要先验证库版本是否支持。
|
||||
- **MCP OAuth token 不持久化是事实但 ADR 描述错误**:迁移点不是"文件挪到 DB",而是"无持久化 → KMS 加密 DB"——实际工作量更大。
|
||||
|
||||
---
|
||||
|
||||
## ADR-007: 路由与前端租户化
|
||||
|
||||
### Assumptions about current code
|
||||
|
||||
1. nginx 把 `/api/*` → Gateway 8001、`/api/langgraph/*` → 同 Gateway 重写 — **HOLDS**
|
||||
- Evidence: 仓库根 CLAUDE.md "Architecture at a glance" 描述;backend CLAUDE L226 也确认。
|
||||
2. 前端用 Better Auth 走 cookie session — **DOES NOT HOLD**
|
||||
- Evidence: `frontend/package.json` 不依赖 `better-auth`(grep 命中 0 次);`frontend/src/core/auth/proxy-policy.ts:52` cookie name 是 `access_token`、由后端 `app/gateway/auth/jwt.py` 自签 JWT;`frontend/src/core/auth/server.ts:25-26` 直接读 `access_token` cookie 调 Gateway `/auth/me`。
|
||||
- Reality: 前端 auth 是后端自有 JWT + cookie 直通,**不是 Better Auth**;ADR §8 "Better Auth 接入"整段需要重写为"自有 JWT 中间件接入"。
|
||||
3. LangGraph SDK 在 `core/api/` 单例 — **HOLDS**
|
||||
- Evidence: `frontend/src/core/api/api-client.ts` `createCompatibleClient` 内 `new LangGraphClient(...)`;模块导出单一 client。
|
||||
4. 没有租户概念,单 host 单工作区 — **HOLDS**
|
||||
- Evidence: `frontend/src/app` 路由组 `(auth)` / `[lang]` / `workspace` / `blog`,无 `(tenant)/[slug]/`;`grep -rn "tenant"` 在 frontend 命中也基本为零。
|
||||
5. CSRF middleware 已存在 — **HOLDS**
|
||||
- Evidence: `app/gateway/csrf_middleware.py` 文件存在;前端 `core/api/api-client.ts:20-32` `injectCsrfHeader` 从 `csrf_token` cookie 读。
|
||||
6. AuthMiddleware 从 cookie 读 JWT、注入 user_id 到 ContextVar — **HOLDS**
|
||||
- Evidence: `app/gateway/auth_middleware.py:84,112,122`。
|
||||
7. 存在 `/setup` 路径处理首启 — **HOLDS**
|
||||
- Evidence: `frontend/src/app/(auth)/setup/`;`auth/repositories/sqlite.py:118` 数 admin 用户。
|
||||
|
||||
### Highest-risk gaps
|
||||
|
||||
- **Better Auth 不存在**:ADR §8 整段"Better Auth 注入 tid/role/tv"前提作废。要么重写 ADR,要么把现有 `auth/jwt.py:14-19` `TokenPayload` 直接扩字段 + bump `ver`——后者其实更简单,但需要 ADR 显式承认。
|
||||
- **前端路由全部按 `(tenant)/[slug]/` 重组的工作量**:当前 `app/workspace/chats/[thread_id]` / `app/workspace/agents/...` 已是核心路径,整体 L 估工没有问题但会触动几乎所有 Server Components。
|
||||
|
||||
---
|
||||
|
||||
## Cross-cutting risks(跨 ADR)
|
||||
|
||||
1. **整个代码库 0 处 `tenant_id` 字段 / 类型 / 引用**:`grep -rn "tenant" backend/packages/harness/ backend/app/` 命中为空。所有 ADR 假设的 ContextVar (`set_current_tenant`)、JWT claim (`tid`)、表列 (`tenant_id`)、路径 (`/tenants/{tid}/`)、缓存键全部不存在 — 任何"加 tenant 维度"的改造都是从零起,而非"扩展现有"。
|
||||
|
||||
2. **没有 ObjectStorage / 没有 KMS / 没有 Postgres 测试基础设施**:ADR-001 RLS、ADR-003 secret vault、ADR-005 三层存储、ADR-006 OAuth 持久化都共用同一组缺失底座 — 这组底座必须先于任何业务改造落地,否则各 ADR 互为前置条件死锁。
|
||||
|
||||
3. **Better Auth 与 LangGraph connection_factory 两个外部依赖假设错误**:ADR-007 假设有 Better Auth、ADR-001/006 假设 langgraph-checkpoint-postgres 支持 connection_factory;前者**当前不存在**、后者**当前未启用**。两个 ADR 写决策时把"外部库能力"误当现状,是同一类风险。
|
||||
|
||||
4. **`extensions_config.json` 是当前 MCP/skills 状态的唯一真源**:`mcp/cache.py` mtime 失效、`routers/{mcp,skills}.py` 直接读写文件、`tools/tools.py:115-119` 同样依赖;它向 DB 迁移会同时触动 ADR-005 §5.4(拆库)、ADR-006 §2.2/2.3(cache 重构)、ADR-004 §5.4(写敏感操作 strict)三个 ADR。
|
||||
|
||||
5. **当前代码的 user_id 过滤是"应用层后过滤 + 部分 SQL WHERE 混合"**:`thread_meta/sql.py:69-70` 是 `if row.user_id != resolved_user_id` 应用层比对,不是 SQL `WHERE`。RLS 假设"加 tenant_id 是平行扩展"在现状下被打了折扣 — 索引前导列、SQL `WHERE` 形态、应用层过滤路径都需要先标准化才能加 RLS 兜底。
|
||||
@@ -0,0 +1,283 @@
|
||||
# 多租户改造 · 第 0 阶段计划
|
||||
|
||||
> 目标:在写第一行代码前,对齐设计、产出可评审的文档。这一阶段不动代码,时间盒 **两周封顶**。
|
||||
|
||||
---
|
||||
|
||||
## 产出物清单(8 份文档 + 1 份代码盘点)
|
||||
|
||||
| 产出物 | 形式 | 谁批 | 作用 |
|
||||
|---|---|---|---|
|
||||
| [ADR-001 数据隔离模型](./adr-001-data-isolation.zh-CN.md) | ADR(决策记录) | CTO / 架构 | 锁定行级 / schema / 库级;含 LangGraph checkpoint 表的注入路径 |
|
||||
| [ADR-002 沙箱隔离模型](./adr-002-sandbox-isolation.zh-CN.md) | ADR + 威胁模型 | 安全 + 架构 | 锁定 K8s / Firecracker / VM |
|
||||
| [ADR-003 LLM Key 与计费模型](./adr-003-llm-key-billing.zh-CN.md) | ADR | 产品 + CTO | 锁定 BYO / 平台付费 / 混合;含悲观预扣 + 内部 LLM 计费 |
|
||||
| [ADR-004 租户 ↔ 用户层级](./adr-004-tenant-rbac.zh-CN.md) | ADR | 产品 | 锁定二级 RBAC + JWT/cache 一致性策略 |
|
||||
| [ADR-005 存储拓扑](./adr-005-storage-topology.zh-CN.md) | ADR + ObjectStorage 接口签名 | 架构 + SRE | 锁定 DB / 对象存储 / 临时区分层 |
|
||||
| [ADR-006 运行时与渠道租户化](./adr-006-runtime-channel-tenancy.zh-CN.md) | ADR | 后端 lead + 渠道 owner | 锁定 checkpointer / MCP cache / skills loader / 内部 LLM 计费 / IM 渠道 ↔ 租户 |
|
||||
| [ADR-007 路由与前端租户化](./adr-007-routing-frontend.zh-CN.md) | ADR | 前端 lead + 后端 lead | 锁定 URL 形态 / cookie / 自签 JWT 扩字段 / SDK 切换 |
|
||||
| [adr-vs-code-audit](./adr-vs-code-audit.zh-CN.md) | 审计报告 | 架构 | 7 份 ADR 对照现状代码的差异清单(已据其修订 ADR-001/006/007) |
|
||||
| [adr-spike-langgraph-postgres](./adr-spike-langgraph-postgres.zh-CN.md) | spike 报告 | 架构 + 后端 lead | 验证 `langgraph-checkpoint-postgres==3.0.5` 的注入能力,结论改写 ADR-001 §4.1 / ADR-006 §2.1 |
|
||||
| Tenant 数据模型设计 | DB schema 草稿 + ER 图 | 后端 lead | 第 1 阶段直接落地用 |
|
||||
| 多租户改造代码盘点 | 表格 / spreadsheet | 后端 lead | 估工 + 拆 PR 用 |
|
||||
|
||||
每份 ADR 用统一结构:**目标客户画像 → 评估维度 → 选项对比 → 选 X 的理由 → 推翻条件**。
|
||||
|
||||
---
|
||||
|
||||
## 1. 四个决策怎么定(决策框架)
|
||||
|
||||
> 下面只是决策框架的 1 页概览。每份 ADR 的完整正文(背景 / 备选方案 / 落地影响 / 风险 / 推翻条件 / 默认假设)已分别成独立文档:
|
||||
> - [ADR-001 数据隔离模型](./adr-001-data-isolation.zh-CN.md)
|
||||
> - [ADR-002 沙箱隔离模型](./adr-002-sandbox-isolation.zh-CN.md)
|
||||
> - [ADR-003 LLM Key 与计费模型](./adr-003-llm-key-billing.zh-CN.md)
|
||||
> - [ADR-004 租户 ↔ 用户层级](./adr-004-tenant-rbac.zh-CN.md)
|
||||
> - [ADR-005 存储拓扑与持久化策略](./adr-005-storage-topology.zh-CN.md)
|
||||
> - [ADR-006 运行时与渠道租户化](./adr-006-runtime-channel-tenancy.zh-CN.md)
|
||||
> - [ADR-007 路由与前端租户化](./adr-007-routing-frontend.zh-CN.md)
|
||||
|
||||
### ADR-001 数据隔离
|
||||
|
||||
| 维度 | 行级 (tenant_id WHERE) | per-tenant schema | per-tenant DB |
|
||||
|---|---|---|---|
|
||||
| 实现成本 | 低 | 中 | 高 |
|
||||
| 跨租户 bug 爆炸半径 | 高 | 中 | 极低 |
|
||||
| 备份/恢复粒度 | 全量 | 按 schema | 按 DB |
|
||||
| 合规友好度(SOC2/HIPAA) | 一般 | 好 | 最好 |
|
||||
| 跨租户分析查询 | 容易 | 中 | 难 |
|
||||
| 适用客户规模 | <10k 租户 | 10k–100 大客户 | <100 大客户 |
|
||||
|
||||
**90% 团队选 行级 + Postgres RLS(双保险)**。理由:DeerFlow 现在的仓储层已经是 `user_id` 过滤模式,RLS 加上去几乎是平行扩展。
|
||||
|
||||
**推翻条件**:拿到金融/医疗类客户、客户合同里写明"物理数据隔离"——直接跳到 per-tenant DB。
|
||||
|
||||
### ADR-002 沙箱隔离
|
||||
|
||||
威胁模型表(每行一个攻击场景):
|
||||
|
||||
| 攻击 | 共享 Docker | per-tenant Namespace | per-tenant VM |
|
||||
|---|---|---|---|
|
||||
| 容器逃逸 | 全员沦陷 | 单租户沦陷 | 单租户沦陷 |
|
||||
| 侧信道(CPU 缓存等) | 可行 | 可行 | 难 |
|
||||
| 出网到云 metadata | 可行(必须默认禁) | 可行(必须默认禁) | 可行(必须默认禁) |
|
||||
| 资源耗尽(fork bomb) | 影响他人 | 仅影响自己 | 仅影响自己 |
|
||||
| 提权 | 看 K8s 配置 | 看 K8s 配置 | VM 边界更强 |
|
||||
|
||||
**推荐:K8s namespace + gVisor/Kata runtime + NetworkPolicy 默认禁出网**。Firecracker 是更强的方案但运维成本翻倍,留给"premium 租户专属"档位。
|
||||
|
||||
### ADR-003 Key & 计费
|
||||
|
||||
三种模式选一种主线:
|
||||
|
||||
- **BYO key**:客户自己带 OpenAI/Anthropic key。优点:你不背模型成本、不背滥用;缺点:客户体验差,需要 secret vault。
|
||||
- **平台付费**:你统一付,按 token 加价转售给客户。优点:体验顺;缺点:你要做精细的 quota+成本归账,否则会被刷爆。
|
||||
- **混合**(推荐起步):默认平台 key + 限额;高级套餐切 BYO key 不限额。
|
||||
|
||||
**写 ADR 时要把 "成本归因路径" 画清楚**:哪个表记 `tenant_id × model × token`,谁算月度账单,怎么和 Stripe 对账。
|
||||
|
||||
### ADR-004 租户内层级
|
||||
|
||||
最常见的两种:
|
||||
|
||||
- **扁平**:tenant 直接装 user,所有 user 等价。简单,适合自助型 SaaS。
|
||||
- **二级 RBAC**:tenant 有 owner/admin/member,admin 能管 member 的 skill 安装权限和 quota 分配。适合企业销售。
|
||||
|
||||
如果要 SSO(SAML/OIDC),那默认要二级 RBAC——因为客户 IT 部门要能管理"哪些员工进哪些 workspace"。
|
||||
|
||||
### ADR-005 存储拓扑
|
||||
|
||||
详见独立文档:[ADR-005 · 存储拓扑与持久化策略](./adr-005-storage-topology.zh-CN.md)。
|
||||
|
||||
### ADR-006 运行时与渠道租户化
|
||||
|
||||
详见独立文档:[ADR-006 · 运行时与渠道层的租户化](./adr-006-runtime-channel-tenancy.zh-CN.md)。
|
||||
|
||||
要点:
|
||||
|
||||
- **LangGraph checkpointer**:保留原表结构、**不挂 RLS、不 ALTER 表**——`langgraph-checkpoint-postgres==3.0.5` 不存在 `connection_factory`(spike 验证),改用入口路由(`threads.py` + `thread_runs.py`)强校验 + `threads_meta` `UNIQUE(tenant_id, thread_id)` 兜底。详见 ADR-001 §4.1.1。
|
||||
- **MCP 工具缓存**:模块级单例 → per-tenant LRU;OAuth token 当前**进程内存无持久化**,要直接做"持久化 + 加密 + 失败回退"三步并发到 `tenant_secrets`。
|
||||
- **Skills loader**:拆 platform 共享只读 + tenant 私有;按 `tenant_skill_state.enabled` 过滤工具。
|
||||
- **Sandbox provider**:实例单例,`acquire(thread_id)` 内按 tenant 路由到 K8s namespace;prewarm 池按 plan 大小。
|
||||
- **内部 LLM 计费**:Memory / Title / Summarization 调用都算 tenant 用量,分 `usage_category` 报表展示。
|
||||
- **IM 渠道**:每个 binding 加 `tenant_id`,webhook handler 强制注入 tenant ContextVar;ghost user 7 天未链接自动停。
|
||||
|
||||
### ADR-007 路由与前端租户化
|
||||
|
||||
详见独立文档:[ADR-007 · URL 路由与前端租户化](./adr-007-routing-frontend.zh-CN.md)。
|
||||
|
||||
要点:
|
||||
|
||||
- **URL 形态**:`/{slug}/...`,子域名留给 v2 自定义域名。
|
||||
- **Cookie**:`Path=/` + JWT 内 `tid`;切换 tenant 重签 JWT + 硬刷新。
|
||||
- **前端**:`app/(tenant)/[slug]/layout.tsx` 注入 `TenantProvider`;SDK 实例单例但调用读 `useTenant()`;切换时 `cancelAllStreams + window.location.assign`。
|
||||
- **Auth**:扩现有 `app/gateway/auth/jwt.py` `TokenPayload` 加 `{tid, role}` 字段(沿用已有 `ver` 失效机制),不引入 Better Auth;登录后跳 picker / 直进 / onboarding。
|
||||
|
||||
要点:
|
||||
|
||||
- **现状问题**:自定义 skill / agent / memory / 上传 / 产物全部在容器本地文件系统。多副本不一致、容器重建丢数据、无备份。
|
||||
- **决策**:三层拓扑——**结构化进 Postgres**(agent SOUL/config、memory facts、skill enabled、tenant secrets)+ **大对象进对象存储**(上传、产物、技能包)+ **临时区**(沙箱 workspace,不持久化)。
|
||||
- **接口**:`ObjectStorage` Protocol,实现 `LocalObjectStorage`(dev)/ `S3ObjectStorage`(prod)/ `MinIOObjectStorage`(自部署)。
|
||||
- **关键约束**:单 bucket + `tenants/{tid}/` prefix;presigned URL 短 TTL;HTML/SVG 强制 `Content-Disposition: attachment`(保留当前 XSS 防护)。
|
||||
- **拒绝方案**:① 全部进 PVC(小文件读写差、备份难);② 全部进 S3(强一致差、列表慢、无事务)。
|
||||
|
||||
---
|
||||
|
||||
## 2. Tenant 数据模型草稿
|
||||
|
||||
第 0 阶段就把这张表画出来,第 1 阶段直接落地:
|
||||
|
||||
```sql
|
||||
-- 新表
|
||||
tenants (
|
||||
id UUID PK,
|
||||
slug VARCHAR(64) UNIQUE, -- URL 用,/t/{slug}/...
|
||||
display_name VARCHAR(128),
|
||||
plan VARCHAR(32), -- free / pro / enterprise
|
||||
status VARCHAR(16), -- active / suspended / deleted
|
||||
created_at, updated_at
|
||||
)
|
||||
|
||||
tenant_memberships (
|
||||
tenant_id UUID FK,
|
||||
user_id UUID FK,
|
||||
role VARCHAR(16), -- owner / admin / member
|
||||
invited_by UUID,
|
||||
joined_at,
|
||||
PRIMARY KEY (tenant_id, user_id)
|
||||
)
|
||||
|
||||
tenant_secrets (
|
||||
tenant_id UUID FK,
|
||||
key VARCHAR(64), -- e.g. OPENAI_API_KEY
|
||||
encrypted_value BYTEA, -- KMS 加密
|
||||
rotated_at,
|
||||
PRIMARY KEY (tenant_id, key)
|
||||
)
|
||||
|
||||
tenant_quotas (
|
||||
tenant_id UUID FK,
|
||||
metric VARCHAR(32), -- tokens_monthly / runs_concurrent / sandbox_cpu_seconds
|
||||
hard_limit BIGINT,
|
||||
soft_limit BIGINT,
|
||||
PRIMARY KEY (tenant_id, metric)
|
||||
)
|
||||
|
||||
tenant_usage_daily (
|
||||
tenant_id UUID,
|
||||
date DATE,
|
||||
metric VARCHAR(32),
|
||||
value BIGINT,
|
||||
PRIMARY KEY (tenant_id, date, metric)
|
||||
)
|
||||
|
||||
invitations (
|
||||
id UUID PK,
|
||||
tenant_id UUID FK,
|
||||
email VARCHAR(320),
|
||||
role VARCHAR(16),
|
||||
token VARCHAR(64) UNIQUE,
|
||||
expires_at,
|
||||
used_at NULL
|
||||
)
|
||||
|
||||
-- 已有表加列
|
||||
users + default_tenant_id UUID
|
||||
threads_meta + tenant_id UUID + INDEX (tenant_id, user_id, updated_at)
|
||||
runs + tenant_id UUID + INDEX (tenant_id, created_at)
|
||||
run_events + tenant_id UUID
|
||||
feedback + tenant_id UUID
|
||||
|
||||
-- 未来 Tier 2 还要加:
|
||||
skills_state (tenant_id, skill_name, enabled, source)
|
||||
agent_configs (tenant_id, user_id, agent_name, soul_md, config_yaml)
|
||||
mcp_configs (tenant_id, server_name, transport, url, encrypted_config)
|
||||
```
|
||||
|
||||
画完后让 DBA / 后端 lead 评审两件事:**索引覆盖**(每个查询 path 是否走索引)和 **RLS policy 草稿**(每张带 tenant_id 的表写一条 policy)。
|
||||
|
||||
---
|
||||
|
||||
## 3. 多租户改造代码盘点
|
||||
|
||||
第 0 阶段最容易被忽略的是**先量一下工作量**。在 spreadsheet 里把"目前涉及 user_id / 全局状态"的代码点全部列出来:
|
||||
|
||||
```bash
|
||||
grep -rn "user_id\|get_effective_user_id\|DEFAULT_USER_ID" backend/packages/harness/deerflow/ backend/app/
|
||||
grep -rn "users/\|/.deer-flow/" backend/ scripts/
|
||||
grep -rn "extensions_config\|skills/public\|skills/custom" backend/
|
||||
```
|
||||
|
||||
把命中点分成五类:
|
||||
|
||||
| 类别 | 改造动作 | 估计点位 |
|
||||
|---|---|---|
|
||||
| **DB 仓储**(`persistence/*/sql.py`) | 增加 tenant_id 解析与 WHERE | ~10–15 处 |
|
||||
| **文件系统路径**(`ThreadDataMiddleware`、memory storage、agents 存储) | 路径加 tenant 维度 | ~5–8 处 |
|
||||
| **配置/Secret 读取**(`models/factory.py`、MCP client、community tools) | 改成 tenant 上下文取 key | ~8–12 处 |
|
||||
| **路由 handler**(`app/gateway/routers/*.py`) | 加 `@require_permission` + tenant 上下文 | ~14 个 router 文件 |
|
||||
| **全局单例**(沙箱 provider、MCP cache、skills loader) | 缓存 key 加 tenant 维度 | ~5 处 |
|
||||
|
||||
每条点位估一个 S/M/L 工作量。这张表是后面拆 PR、估工期、估钱的依据。
|
||||
|
||||
---
|
||||
|
||||
## 3.5 底座先行(Phase-0 之前 / 并行的基础设施)
|
||||
|
||||
> 来源:[adr-vs-code-audit](./adr-vs-code-audit.zh-CN.md) cross-cutting risk #2 — ADR-001 RLS、ADR-003 secret vault、ADR-005 三层存储、ADR-006 OAuth 持久化共用同一组缺失底座。这组**必须先于任何业务改造落地**,否则各 ADR 互为前置条件死锁。
|
||||
|
||||
| 底座 | 缺失现状 | 为何阻塞 ADR | Phase-0 内必须产出 |
|
||||
|---|---|---|---|
|
||||
| **Postgres 切换 + testcontainers 夹具**(生产 + 测试基础设施一并落) | 仓库当前以 SQLite 为默认后端,`tests/` 下无 Postgres fixture;SQLite 不支持 RLS | ADR-001 / 004 / 005 的所有租户隔离测试都要 Postgres;Stage 0 ALTER 4 张表如果在 SQLite 上做完再切 PG 是纯返工 | **Stage 0 直接切 Postgres 为生产默认**(Stage 0 没有生产数据,迁移阻力最小)+ testcontainers 集成 + 至少 1 个 RLS 冒烟测试模板(policy Stage 2 才启用,但夹具 Stage 0 就位)+ CI 跑通 |
|
||||
| **ObjectStorage Protocol + 实现** | `backend/packages/harness/deerflow/` 内 grep 不到 `ObjectStorage` 类;当前 memory/uploads/artifacts 全走文件系统 | ADR-005 §2 的三层拓扑、ADR-006 §2.2 的 OAuth 持久化都依赖它 | Protocol 接口 + LocalObjectStorage 骨架(可不实现 S3,留接口) |
|
||||
| **KMS / Secret Vault 抽象** | 当前没有 secret 加密层;`mcp/oauth.py` 的 token 是明文进程内存 | ADR-003 §4.6 BYO key、ADR-006 §2.2 MCP OAuth、ADR-007 channel binding token 共用 | 抽象接口(envelope encryption pattern)+ 本地 dev 实现(明文 fallback + 警告日志),生产实现可推迟 |
|
||||
|
||||
**时间盒**:3 项底座**与 ADR 评审并行做**,加 1 周到 Phase-0(总计 3 周封顶)。完成验证标准是这 3 件事**至少有可 CI 验证的最小骨架**——不要求 100% 实现,但接口 + 1 个测试用例必须跑通。
|
||||
|
||||
> 这部分原稿没列。审计后补的。如果跳过这步直接做业务改造,ADR-001/003/005/006 实现时会发现互相依赖、谁都跑不起来。
|
||||
|
||||
---
|
||||
|
||||
## 4. 第 0 阶段的"完成定义" (DoD)
|
||||
|
||||
走完这阶段,团队应该能回答:
|
||||
|
||||
- [ ] 数据存哪、用什么数据库、怎么隔离 → ADR-001 给出
|
||||
- [ ] 客户的 bash/工具跑在哪、能访问什么、爆炸半径多大 → ADR-002 给出
|
||||
- [ ] 客户的 LLM 调用钱谁出、怎么算 → ADR-003 给出
|
||||
- [ ] 客户内部能不能自己加员工、怎么加 → ADR-004 给出
|
||||
- [ ] Skill / agent / 上传 / 产物 / memory 各自存哪、丢失怎么办 → ADR-005 给出
|
||||
- [ ] LangGraph / MCP / 内部 LLM / IM 渠道这些"夹层"怎么按租户隔离 → ADR-006 给出
|
||||
- [ ] 浏览器地址栏长什么样、Cookie 怎么 scope、租户切换怎么走 → ADR-007 给出
|
||||
- [ ] **3 项底座**(Postgres 测试夹具 / ObjectStorage Protocol / KMS 抽象)有可 CI 验证的最小骨架 → §3.5 给出
|
||||
- [ ] 第一阶段 PR 怎么拆、估几人周 → 代码盘点给出
|
||||
- [ ] 第一个内测客户长什么样、什么时候能上 → 项目经理排期
|
||||
|
||||
---
|
||||
|
||||
## 5. 时间盒与节奏
|
||||
|
||||
第 0 阶段 **三周封顶**(原稿两周,加 §3.5 底座先行的 1 周),再长就是过度设计。
|
||||
|
||||
- **第 1 周**:写 ADR-001 ~ 007 草稿,团队读、challenge、收敛
|
||||
- **第 2 周**:定 schema、做代码盘点、估工、定第一阶段范围与 design partner 客户;同时启动 §3.5 底座 spike(Postgres testcontainers / ObjectStorage Protocol / KMS 抽象)
|
||||
- **第 3 周**:底座骨架 PR 合入 + ADR 据实测结果定稿(这一周已经在审计 + spike 中部分提前消耗,参见 [adr-vs-code-audit](./adr-vs-code-audit.zh-CN.md) 与 [adr-spike-langgraph-postgres](./adr-spike-langgraph-postgres.zh-CN.md))
|
||||
|
||||
如果三周后还有 ADR 定不下来,**绝大多数情况是因为缺一个真实客户做参照**——这时候应该先去签一个 design partner(哪怕免费),用他们的合同和合规要求来反推决策。
|
||||
|
||||
---
|
||||
|
||||
## 6. 默认假设(如无特殊情况按此推进)
|
||||
|
||||
为避免决策瘫痪,先写下一个"默认值",所有 ADR 在没有相反证据前按这个走:
|
||||
|
||||
| 决策 | 默认值 | 选它的理由 |
|
||||
|---|---|---|
|
||||
| **数据库** | Stage 0 起直接切 Postgres 为生产默认;SQLite 仅保留为可选 dev 兜底 | Stage 0 没有生产数据,迁移阻力最小;省 Stage 1 重 ALTER 一遍的返工 |
|
||||
| 数据隔离 | 行级 + Postgres RLS(仅 DeerFlow 自有表,policy Stage 2 启用)+ LangGraph 表应用层强校验 | 改造成本低;LangGraph 表无 RLS hook(spike 已验证),应用层兜底 |
|
||||
| 沙箱隔离 | K8s namespace + gVisor + NetworkPolicy 默认禁出网 | 强度足够 + 运维可控 |
|
||||
| LLM Key | 混合:默认平台 key + 限额,premium 切 BYO;悲观预扣防超额 | 体验与成本兼顾 |
|
||||
| 租户层级 | 二级 RBAC(owner/admin/member)+ JWT/cache 双层 | 为 SSO 和企业销售留口 |
|
||||
| 存储拓扑 | Postgres(结构化)+ S3 兼容对象存储(大对象)+ emptyDir(临时) | 沙箱 pod 真正无状态;备份/灾备/横向扩展直接通 |
|
||||
| 运行时夹层 | per-tenant MCP cache + skills 拆双路 + 内部 LLM 计费分类 + IM binding 加 tenant | 关上"非仓储非沙箱"那一组进程级单例的隔离漏洞 |
|
||||
| 前端路由 | `/{slug}/...` 路径 + JWT 内 tid + 硬刷新切换 | UX 简单,与 ADR-001/004 cookie 模型契合 |
|
||||
|
||||
> 这是"中等强度方案",覆盖 90% B2B SaaS。如果客户画像偏极端(大企业 / 强合规 / 自助小客户),再调整。
|
||||
@@ -0,0 +1,306 @@
|
||||
# Workspace Schema 设计 · Stage 0 锁定版
|
||||
|
||||
> 写于 2026-05-10。Stage 0 PR1 动手前的 schema 锁定文档。
|
||||
>
|
||||
> **数据库基线**:Stage 0 的 PR1 必须**先**完成 [phased-rollout Stage 0](../02-rollout/phased-rollout-by-scale.zh-CN.md#stage-0--workspace-模型立起来--postgres-切换--auth-收紧) 的 PR1-PR2(Postgres 接入 + 默认切换),再起 schema PR。下面所有 ALTER 都直接在 Postgres 上跑,**不再走 SQLite → Postgres 二次迁移**。SQLite 仅保留为可选 dev 兜底。
|
||||
>
|
||||
> **范围**:仅 Stage 0 必须落地的 schema —— `workspaces` / `workspace_memberships` 两张新表,`users` / `threads_meta` / `runs` / `feedback` 的 ALTER,`service_accounts` / `api_keys` / `external_users` 的预建(Stage 0 末,schema only),以及 JWT `TokenPayload` 一次到位的字段集。
|
||||
>
|
||||
> **不在范围**:仓储实现细节、ContextVar、路径迁移、路由校验、Stage 1+ 才加的列(`plan` / `allowed_origins` / `custom_domain` 等)。
|
||||
>
|
||||
> **配套阅读**:
|
||||
> - [02-rollout/phased-rollout-by-scale.zh-CN.md](../02-rollout/phased-rollout-by-scale.zh-CN.md) Stage 0 必做项
|
||||
> - [02-rollout/stage-0-code-map.zh-CN.md](../02-rollout/stage-0-code-map.zh-CN.md) 现状代码锚点
|
||||
> - [adr-001-data-isolation.zh-CN.md](./adr-001-data-isolation.zh-CN.md) §4.1 表结构改造
|
||||
> - [adr-004-tenant-rbac.zh-CN.md](./adr-004-tenant-rbac.zh-CN.md) §5.1 / §5.2 RBAC + JWT
|
||||
> - [adr-007-routing-frontend.zh-CN.md](./adr-007-routing-frontend.zh-CN.md) §4 slug 规范、§8 JWT 改造
|
||||
> - [02-rollout/headless-api-track.zh-CN.md](../02-rollout/headless-api-track.zh-CN.md) §2 service account / API key
|
||||
|
||||
---
|
||||
|
||||
## 1. 命名约定 · workspace vs tenant
|
||||
|
||||
| 维度 | 选择 |
|
||||
|---|---|
|
||||
| 数据库列名 | `workspace_id` |
|
||||
| Python 标识符 | `workspace_id` / `WorkspaceRow` / `_current_workspace` |
|
||||
| JWT claim | `wid`(紧凑) |
|
||||
| 用户可见用语 | "Workspace"(团队 workspace 也叫 workspace,不分"个人空间") |
|
||||
|
||||
ADR-001 / 004 / 007 原稿写 `tenant_id`——这些 ADR**不重命名**(成本不抵收益),落代码时统一读作 `workspace_id`。本文档与 02-rollout 系列保持 `workspace` 用语一致。
|
||||
|
||||
> 推翻条件:拿到强企业客户后真的出现"租户内多 workspace"的层级(tenant > workspace > user),那时再分裂概念。Stage 0/1/2 不预留这层。
|
||||
|
||||
---
|
||||
|
||||
## 2. 新增表
|
||||
|
||||
### 2.1 `workspaces`
|
||||
|
||||
| 列 | 类型 | 约束 | 说明 |
|
||||
|---|---|---|---|
|
||||
| `id` | `String(36)` | PK | UUID v4 字符串。与 `users.id` 类型对齐,跨 DB 可移植(SQLite/Postgres 都用 36 字符 CHAR) |
|
||||
| `name` | `String(64)` | NOT NULL | 显示名(用户首次注册时默认 `<email 前缀>'s Workspace`) |
|
||||
| `slug` | `String(32)` | UNIQUE NOT NULL | URL 标识,正则 `^[a-z0-9](-?[a-z0-9])*$`,3-32 字符;DB 存小写 |
|
||||
| `status` | `String(16)` | NOT NULL default `'active'` | `active` / `suspended` / `deleted`(platform admin 暂停/删 workspace) |
|
||||
| `owner_id` | `String(36)` | NOT NULL FK `users.id` | 冗余字段,便于查询;与 `workspace_memberships.role='owner'` 严格一致(事务保证) |
|
||||
| `created_at` | `DateTime(timezone=True)` | NOT NULL | UTC |
|
||||
| `updated_at` | `DateTime(timezone=True)` | NOT NULL | UTC,写入自动更新 |
|
||||
|
||||
**索引**:
|
||||
- PK: `id`
|
||||
- UNIQUE: `slug`
|
||||
- 不加 `(status)` 索引——Stage 0 用户量小,全表扫够用;Stage 1+ 视情况补
|
||||
|
||||
**slug 黑名单**(应用层校验,不写进 DB constraint):
|
||||
|
||||
```
|
||||
admin, api, auth, login, signup, accept-invite, pricing, docs, status,
|
||||
platform, system, health, static, public, favicon.ico, robots.txt,
|
||||
sitemap.xml, _next, .well-known, settings, billing, onboarding, select-workspace
|
||||
```
|
||||
|
||||
> ADR-007 §4 列了一份基础黑名单;本表是落代码版(含 Next.js 保留路径)。
|
||||
|
||||
**Stage 0 不加的列**(决策记录):
|
||||
|
||||
| 列 | 推迟到 | 理由 |
|
||||
|---|---|---|
|
||||
| `plan` | Stage 1(与 `workspace_quotas.plan` 一起加) | Stage 0 没有付费分层 |
|
||||
| `allowed_origins` | Stage 1 末(headless API Pattern B) | 用到时加 ARRAY/JSON 列代价低 |
|
||||
| `custom_domain` | Stage 4(enterprise) | 长尾需求,列加在哪一层都行 |
|
||||
| `billing_email` | Stage 1(Stripe 对接) | 一并加 |
|
||||
| `settings_json` | 永不加 | 扩展点用专门的 `workspace_settings` 表 + 枚举 key,比 JSON dump 易迁移 |
|
||||
|
||||
### 2.2 `workspace_memberships`
|
||||
|
||||
| 列 | 类型 | 约束 | 说明 |
|
||||
|---|---|---|---|
|
||||
| `workspace_id` | `String(36)` | PK / FK `workspaces.id` ON DELETE CASCADE | |
|
||||
| `user_id` | `String(36)` | PK / FK `users.id` ON DELETE CASCADE | |
|
||||
| `role` | `String(16)` | NOT NULL | Stage 0 只允许 `owner`;Stage 2 起 `owner`/`admin`/`member` |
|
||||
| `invited_by` | `String(36)` | NULL FK `users.id` ON DELETE SET NULL | Stage 0 暂不写入;Stage 2 invitation 流程才用 |
|
||||
| `joined_at` | `DateTime(timezone=True)` | NOT NULL | UTC |
|
||||
|
||||
**为什么 `role` 用 `String(16)` 不用 DB enum**:Postgres enum ALTER 加值需要 `ALTER TYPE ... ADD VALUE`,且不可删;string + 应用层校验 = 后续随便加 `viewer` / `auditor` 等角色不动 DB schema。
|
||||
|
||||
**索引**:
|
||||
- PK: `(workspace_id, user_id)` 复合主键
|
||||
- `idx_workspace_memberships_user`: `(user_id, workspace_id)` —— 倒查索引,用于 `/auth/me` 列出当前 user 所有 workspace
|
||||
- `idx_one_owner_per_workspace`: UNIQUE on `(workspace_id)` WHERE `role = 'owner'` —— partial unique index
|
||||
|
||||
**partial unique 兼容性**:
|
||||
- SQLite **支持** `CREATE UNIQUE INDEX ... WHERE ...`(参 `users.idx_users_oauth_identity` 现有用法)
|
||||
- Postgres 同样支持
|
||||
- SQLAlchemy 通过 `Index(..., unique=True, sqlite_where=text(...), postgresql_where=text(...))` 表达;这条索引**保留两套 where 条件等价**
|
||||
|
||||
### 2.3 `service_accounts`(Stage 0 末加,schema only)
|
||||
|
||||
| 列 | 类型 | 约束 | 说明 |
|
||||
|---|---|---|---|
|
||||
| `id` | `String(36)` | PK | |
|
||||
| `workspace_id` | `String(36)` | NOT NULL FK `workspaces.id` ON DELETE CASCADE | 必属于一个 workspace |
|
||||
| `name` | `String(64)` | NOT NULL | 业务系统起的标识名 |
|
||||
| `role` | `String(16)` | NOT NULL default `'member'` | SA 在 workspace 内的 role |
|
||||
| `identity_mode` | `String(16)` | NOT NULL default `'collapsed'` | `collapsed` / `external_passthrough` / `both` |
|
||||
| `status` | `String(16)` | NOT NULL default `'active'` | `active` / `suspended` / `revoked` |
|
||||
| `created_by` | `String(36)` | NOT NULL FK `users.id` ON DELETE RESTRICT | 必须是 workspace owner/admin |
|
||||
| `created_at` | `DateTime(tz)` | NOT NULL | |
|
||||
| `updated_at` | `DateTime(tz)` | NOT NULL | |
|
||||
|
||||
索引:`idx_service_accounts_workspace`: `(workspace_id, status)`
|
||||
|
||||
### 2.4 `api_keys`(Stage 0 末加,schema only)
|
||||
|
||||
| 列 | 类型 | 约束 | 说明 |
|
||||
|---|---|---|---|
|
||||
| `id` | `String(36)` | PK | |
|
||||
| `service_account_id` | `String(36)` | NOT NULL FK `service_accounts.id` ON DELETE CASCADE | |
|
||||
| `key_prefix` | `String(16)` | UNIQUE NOT NULL | 前 16 字符明文(`dfk_live_...`),UI 展示用 |
|
||||
| `key_hash` | `String(128)` | NOT NULL | 完整 key 的 sha256 hex(64 字符)+ 余量 |
|
||||
| `name` | `String(64)` | NOT NULL | "生产环境 key" |
|
||||
| `scopes` | `String(1024)` | NOT NULL default `''` | 逗号分隔字符串。Postgres 已是 Stage 0 默认,但保持 `String` 以兼容 SQLite dev 兜底;如果未来确认完全弃用 SQLite,可平滑迁 `text[]` |
|
||||
| `rate_limit_rpm` | `Integer` | NULL | NULL = 用 workspace plan 默认 |
|
||||
| `expires_at` | `DateTime(tz)` | NULL | |
|
||||
| `last_used_at` | `DateTime(tz)` | NULL | |
|
||||
| `revoked_at` | `DateTime(tz)` | NULL | 软删除标记 |
|
||||
| `created_at` | `DateTime(tz)` | NOT NULL | |
|
||||
|
||||
索引:
|
||||
- `idx_api_keys_sa`: `(service_account_id)`
|
||||
- `idx_api_keys_active`: `(key_prefix)` WHERE `revoked_at IS NULL`(partial)
|
||||
|
||||
### 2.5 `external_users`(Stage 0 末加,schema only)
|
||||
|
||||
| 列 | 类型 | 约束 | 说明 |
|
||||
|---|---|---|---|
|
||||
| `id` | `String(36)` | PK | ghost user id(DeerFlow 内部) |
|
||||
| `workspace_id` | `String(36)` | NOT NULL FK `workspaces.id` ON DELETE CASCADE | |
|
||||
| `service_account_id` | `String(36)` | NOT NULL FK `service_accounts.id` ON DELETE CASCADE | |
|
||||
| `external_id` | `String(128)` | NOT NULL | 业务系统传入 ID,原样存 |
|
||||
| `display_name` | `String(128)` | NULL | |
|
||||
| `metadata_json` | `JSON` | NOT NULL default `{}` | 业务字段 |
|
||||
| `created_at` | `DateTime(tz)` | NOT NULL | |
|
||||
| `last_seen_at` | `DateTime(tz)` | NULL | |
|
||||
|
||||
索引:UNIQUE `(service_account_id, external_id)` —— 同一 SA 下 external_id 唯一
|
||||
|
||||
---
|
||||
|
||||
## 3. ALTER 现有表
|
||||
|
||||
### 3.1 `users`
|
||||
|
||||
```python
|
||||
# 新增列
|
||||
default_workspace_id: Mapped[str | None] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("workspaces.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
comment="登录后默认进入的 workspace;NULL 时强制走 picker(user 多 workspace 场景)"
|
||||
)
|
||||
```
|
||||
|
||||
**为什么不加 `current_workspace_id`**:每次登录时从 `default` 或 `/select-workspace` 决定,写入 JWT 的 `wid` claim;DB 不存"当前激活"状态,避免多设备冲突。
|
||||
|
||||
`system_role` 字段保留——它是**平台级** role(`platform_admin` / `user`),与 workspace role 正交(参 ADR-004 §6)。
|
||||
|
||||
### 3.2 `threads_meta`
|
||||
|
||||
```python
|
||||
# 新增列(Stage 0 PR3 先 nullable,回填后 ALTER 改 NOT NULL)
|
||||
workspace_id: Mapped[str | None] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("workspaces.id", ondelete="CASCADE"),
|
||||
nullable=True, # PR3 中段;回填脚本跑完改 NOT NULL
|
||||
comment="所属 workspace;与 (thread_id) 复合 UNIQUE 防跨 workspace 复用同 ID"
|
||||
)
|
||||
|
||||
# 新增索引(在 __table_args__ 里)
|
||||
Index("idx_threads_meta_workspace_thread", "workspace_id", "thread_id", unique=True),
|
||||
Index("idx_threads_meta_workspace_user_updated", "workspace_id", "user_id", "updated_at"),
|
||||
```
|
||||
|
||||
**关键索引说明**:
|
||||
- `(workspace_id, thread_id)` UNIQUE 是 ADR-001 §4.1.1 修订版的"应用层强约束 + DB 兜底"防线
|
||||
- `(workspace_id, user_id, updated_at)` 覆盖前端 thread list 默认查询模式
|
||||
- 现有 `user_id` 上的非复合索引可以**保留**(删了某些后台 cleanup 脚本会变慢;不阻塞主路径)
|
||||
|
||||
### 3.3 `runs`
|
||||
|
||||
```python
|
||||
workspace_id: Mapped[str | None] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("workspaces.id", ondelete="CASCADE"),
|
||||
nullable=True, # PR3 中段
|
||||
)
|
||||
|
||||
Index("idx_runs_workspace_created", "workspace_id", "created_at"),
|
||||
```
|
||||
|
||||
### 3.4 `feedback`
|
||||
|
||||
```python
|
||||
workspace_id: Mapped[str | None] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("workspaces.id", ondelete="CASCADE"),
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
Index("idx_feedback_workspace_run", "workspace_id", "run_id"),
|
||||
```
|
||||
|
||||
### 3.5 `run_events`(待确认)
|
||||
|
||||
stage-0-code-map §2 标注 `run_events` 是否 DB 持久化"unverified"。PR3 第一步先 grep 确认;若是 DB 表就比照 `runs` 加 `workspace_id`,若是内存队列则跳过。
|
||||
|
||||
---
|
||||
|
||||
## 4. JWT TokenPayload · 一次到位的字段集
|
||||
|
||||
> Stage 0 落 `wid` + `role`,**Stage 2 不再 bump**。理由:每次扩字段都要 bump `token_version` 让所有用户重登,churn 体验差;一次加齐两次的份。
|
||||
|
||||
```python
|
||||
# backend/app/gateway/auth/jwt.py
|
||||
class TokenPayload(BaseModel):
|
||||
sub: str # user_id(沿用)
|
||||
wid: str # workspace_id(Stage 0 新增)
|
||||
role: str # owner / admin / member(Stage 0 新增;1 人 workspace 默认 'owner')
|
||||
exp: datetime # 沿用
|
||||
iat: datetime | None = None # 沿用
|
||||
ver: int = 0 # token_version(沿用;任何 membership 变更 bump)
|
||||
```
|
||||
|
||||
**Stage 0 实际填充值**:
|
||||
- `wid` = 用户 `default_workspace_id`(注册时自动建的 1 人 workspace)
|
||||
- `role` = 总是 `'owner'`(Stage 0 还没有团队 workspace)
|
||||
|
||||
**Stage 2 启用时**:
|
||||
- 加入团队 workspace 后 → `role` 真正区分 admin/member
|
||||
- bump `token_version` 让旧 JWT(仍写 `'owner'`)过期重发
|
||||
|
||||
**兼容性**:
|
||||
- 旧 4 字段 token(`{sub, exp, iat, ver}`)解码失败时**强制走 `/select-workspace`** 重发新 JWT(参 ADR-007 §11)。Stage 0 部署后 7 天(默认 token TTL)内所有老 token 自然轮替完。
|
||||
|
||||
---
|
||||
|
||||
## 5. 不可逆决策清单
|
||||
|
||||
| 决策 | 选择 | 反悔代价 |
|
||||
|---|---|---|
|
||||
| id 类型 | `String(36)` (UUID v4 字符串) | 改 native UUID → 全表 schema rewrite + 所有 FK 重建 + Python `str` ↔ `UUID` 边界改造 |
|
||||
| 命名(workspace_id) | `workspace_id` | 改 `tenant_id` → 全代码改名 + 所有 ADR 文档同步 |
|
||||
| slug 字符集 | `^[a-z0-9](-?[a-z0-9])*$` 3-32 | 改 → 老 URL 全失效(v1 还没暴露 slug 路由前改是免费的) |
|
||||
| memberships PK | `(workspace_id, user_id)` 复合 | 改 surrogate id → migration 脚本要写 dedup 逻辑 |
|
||||
| TokenPayload 字段 | `sub/wid/role/exp/iat/ver` | 加新字段 → 必 bump `token_version`,全用户重登(Stage 0 一次性加 `wid` + `role`,省一次) |
|
||||
| `users.default_workspace_id` 而非 `current_workspace_id` | 默认 + JWT 决定当前 | 改成 `current_*` → 多设备语义混乱 |
|
||||
| FK 删除策略(workspace 删 → memberships/threads CASCADE)| CASCADE | 改 RESTRICT → 平台 admin 删 workspace 时手动级联,运营负担大 |
|
||||
|
||||
> Stage 0 PR1 合入前,上面这 7 项**全部**要在团队 review 中拍板;任何一项改主意都要 revert PR1 重写。
|
||||
|
||||
---
|
||||
|
||||
## 6. PR 拆分(Stage 0 内的 5 个 schema PR)
|
||||
|
||||
> **前置 PR**:本表 PR1 之前必须先完成 [phased-rollout Stage 0 PR1-2](../02-rollout/phased-rollout-by-scale.zh-CN.md#stage-0--workspace-模型立起来--postgres-切换--auth-收紧):Postgres 接入 + testcontainers + 默认 backend 切换。本文档下面的 PR1 = phased-rollout 的 PR3,本文档 PR5 = phased-rollout 的 PR8。
|
||||
>
|
||||
> 在 Postgres 已就绪的基础上,schema 改动按下面 5 个 PR 推:
|
||||
|
||||
| PR | 范围 | 落本文档的哪些章节 | 状态 |
|
||||
|---|---|---|---|
|
||||
| **PR1** | `workspaces` + `workspace_memberships` 表 + 仓储 + 单测 | §2.1 + §2.2 | 设计完,可写 |
|
||||
| **PR2** | 注册/initialize 流程改造(自动建 1 人 workspace)+ JWT 扩 `wid`/`role` + AuthMiddleware ContextVar 注入 | §3.1 (`default_workspace_id`) + §4 | 依赖 PR1 |
|
||||
| **PR3** | 现有 4 表 ALTER 加 `workspace_id`(直接 Postgres,先 nullable)+ 数据回填脚本(legacy_workspace)→ ALTER 改 NOT NULL | §3.2-§3.5 | 依赖 PR1+PR2 |
|
||||
| **PR4** | 路由层 `(workspace_id, thread_id)` 校验 + `Paths` 切 workspace 维度 + 文件系统迁移脚本 | 不在本文档(仓储/路径设计) | 依赖 PR3 |
|
||||
| **PR5** | `service_accounts` / `api_keys` / `external_users` schema(不接路径) | §2.3-§2.5 | 与 PR4 并行 |
|
||||
|
||||
每个 PR 必须独立可上线、可回滚。PR1 单独合入后系统行为不变(新表无人写入)。
|
||||
|
||||
---
|
||||
|
||||
## 7. 测试要点(PR1 范围)
|
||||
|
||||
仿现有 `tests/test_*.py` 模式:
|
||||
|
||||
- `test_workspace_repo.py`
|
||||
- create / get / update / delete workspace
|
||||
- slug 唯一性约束
|
||||
- slug 黑名单校验(应用层)
|
||||
- status 状态机(active → suspended → deleted)
|
||||
- `test_workspace_membership_repo.py`
|
||||
- create membership
|
||||
- 同一 workspace 不能有 2 个 owner(partial unique index)
|
||||
- CASCADE 删(删 workspace 后 memberships 消失)
|
||||
- `list_workspaces_by_user(user_id)` 返回正确顺序
|
||||
- 不写:路由测试(PR4 才有路由)、JWT 测试(PR2 才扩字段)
|
||||
|
||||
---
|
||||
|
||||
## 8. 推翻条件
|
||||
|
||||
整份 schema 设计要重排只在两种情况:
|
||||
|
||||
1. **拿到强企业客户必须自定义角色**:role 列从 String(16) + 应用层校验 → 完整 RBAC engine(`roles` / `permissions` / `role_permissions` 表)。schema **加表**,不改现有列,影响小。
|
||||
2. **决定改用 native UUID 类型**(Postgres 切换时一并):String(36) → `UUID`。需要全表 ALTER + Python 边界改造。建议**不**做,String(36) 在 Postgres 上落地为 `CHAR(36)`,性能差异 < 5%,可接受。
|
||||
|
||||
> 上面 §5 七项不可逆决策不在"推翻条件"覆盖范围——那些一旦发布到生产就只能往前走。
|
||||
@@ -0,0 +1,565 @@
|
||||
# Headless API · 业务系统集成轨道
|
||||
|
||||
> 写于 2026-05-09。承接 [phased-rollout-by-scale.zh-CN.md](./phased-rollout-by-scale.zh-CN.md)。
|
||||
>
|
||||
> **触发**:现已有 1-2 个明确的业务系统集成需求,1-3 个月内要 demo / 调通。集成形态包括 IM channels(已支持)+ 业务系统自研 web 页面。
|
||||
> **商业形态**:与 [phased-rollout-by-scale §0](./phased-rollout-by-scale.zh-CN.md) 一致——**中心化 SaaS 主线**;schema / auth / quota 设计对 on-prem 友好(`workspaces.id` 映射到 self-host 安装),但 on-prem 不作为产品主线,仅按客户合同启用。
|
||||
> **身份模式**:service account 折叠 + external_user_id 透传,两种都支持,按 endpoint 选。
|
||||
> **集成 pattern**:**Pattern A(业务系统 backend 代理)+ Pattern B(浏览器直连 + 短期 JWT)**。不做嵌入式 widget。
|
||||
|
||||
---
|
||||
|
||||
## 0. 现状评估
|
||||
|
||||
DeerFlow 架构已经接近"可被业务系统调用的后端"——证据:`backend/app/channels/` 下的 IM 集成(Slack / 飞书 / 钉钉 / Telegram)就是这种用法的活体范例。它们通过 `langgraph-sdk` 调 Gateway HTTP API + 把响应转给 IM 平台,**根本不经过 `frontend/` 工程**。
|
||||
|
||||
### 已经具备(约 80%)
|
||||
|
||||
| 能力 | 实现位置 |
|
||||
|---|---|
|
||||
| 完整 REST API | `backend/app/gateway/routers/`(threads / runs / messages / events / feedback / models / skills / mcp / memory / uploads / artifacts)|
|
||||
| LangGraph SDK 兼容路径 | `/api/langgraph/*` —— 任何 `langgraph-sdk` 客户端可直接接 |
|
||||
| Streaming(SSE) | `runs/stream` + `messages-tuple` delta + `values` + `custom` |
|
||||
| 嵌入式 Python 客户端 | `packages/harness/deerflow/client.py` `DeerFlowClient` |
|
||||
| 现成的"无 web 前端"调用证明 | `app/channels/manager.py` 完全不依赖 frontend |
|
||||
|
||||
### Stage 0 完成后还差什么
|
||||
|
||||
Stage 0 把 workspace 概念立起来了,但 **auth 模式仍是 cookie + JWT + CSRF**——这是为 web 前端设计的,不适合 server-to-server,更不适合"业务系统自研 web 页面浏览器直连"的场景。要让业务系统调,必须补两条平行 auth 路径 + 一组管理能力,详见 §1。
|
||||
|
||||
### 集成 pattern 速览
|
||||
|
||||
| Pattern | 链路 | 适用 | 优先级 |
|
||||
|---|---|---|---|
|
||||
| **A. Backend 代理**(默认) | browser → 业务系统 backend → DeerFlow API key → DeerFlow | 业务系统已有 backend;不在乎多一跳 | Stage 1 必做 |
|
||||
| **B. Browser 直连**(streaming 友好) | 业务系统 backend 颁短期 JWT → browser 直连 DeerFlow `/api/v1/*`(含 SSE) | chat / agent 类、对 token 流延迟敏感、自研 web 页面 | Stage 1 末必做 |
|
||||
| ~~C. 嵌入式 widget / iframe~~ | DeerFlow 托管 chat widget URL,业务系统 embed | 集成方零前端开发 | **不做**(产品决定) |
|
||||
|
||||
Pattern A 是"server-to-server",Pattern B 是"browser-to-server"。两者共用同一组 service account / API key 数据模型,**只是 auth 路径不同**:
|
||||
- Pattern A:`Authorization: Bearer dfk_live_<long-lived API key>`
|
||||
- Pattern B:`Authorization: Bearer eyJ<short-lived JWT>`
|
||||
|
||||
---
|
||||
|
||||
## 1. 改造清单(按 MVP 优先级)
|
||||
|
||||
| # | 能力 | 是否 MVP | 缺它会怎么样 | 工作量 |
|
||||
|---|---|---|---|---|
|
||||
| 1 | **API Key 认证(Pattern A)** | **必须** | 业务系统 backend 没法调,跨域 + CSRF 灾难 | M |
|
||||
| 2 | **Service Account 概念** | **必须** | API key 必须挂在某个"账户"上做归属、计费、quota | M |
|
||||
| 3 | **CSRF bypass on bearer** | **必须** | bearer 路径走 CSRF middleware 直接 403 | XS |
|
||||
| 4 | **External User ID 透传** | **必须** | 业务系统的"小明"在 DeerFlow 内不能体现,memory / 个性化失效 | M |
|
||||
| 5 | **API 版本化 `/api/v1/`** | **必须**(早做便宜)| 演进时业务系统要全量改 | S(早做)/ L(晚做)|
|
||||
| 6 | **Rate limit per API key** | **必须**(基础版) | 业务系统 bug 把 DeerFlow 打爆 | S(基础)/ M(分层)|
|
||||
| 7 | **Token Exchange(Pattern B)** | **必须** | 浏览器只能走 backend 代理,自研 web 页面延迟差 | S |
|
||||
| 8 | **CORS 中间件 + per-workspace allowed_origins** | **必须**(与 Pattern B 配套)| 浏览器请求被同源策略挡 | M |
|
||||
| 9 | **短期 JWT 验证路径**(AuthMiddleware 第三条) | **必须**(与 Pattern B 配套)| 短期 JWT 没法验 | S |
|
||||
| 10 | **Idempotency keys** | 推荐 | 业务系统重试时重复建 thread/run | S |
|
||||
| 11 | **Webhook outbound** | 推迟 | 业务系统轮询事件,多调几次 SSE 而已 | M(推到 Stage 2)|
|
||||
|
||||
**MVP 包**(前 9 项)= **4-5 周**;放进 Stage 1 并行做。Pattern B(7-9)依赖 1-3 完成,建议 Stage 1 末(最后 1-2 周)。
|
||||
|
||||
---
|
||||
|
||||
## 2. 核心设计:API Key + Service Account
|
||||
|
||||
### 数据模型
|
||||
|
||||
```sql
|
||||
service_accounts (
|
||||
id UUID PK,
|
||||
workspace_id UUID FK NOT NULL, -- 必属于一个 workspace
|
||||
name VARCHAR(64), -- "X 业务系统集成"
|
||||
role VARCHAR(16), -- 在 workspace 内的 role:member / admin
|
||||
identity_mode VARCHAR(16), -- collapsed | external_passthrough | both
|
||||
status VARCHAR(16), -- active / suspended / revoked
|
||||
created_by UUID, -- 哪个 user 创建的(必须是 workspace owner/admin)
|
||||
created_at, updated_at
|
||||
)
|
||||
|
||||
api_keys (
|
||||
id UUID PK,
|
||||
service_account_id UUID FK NOT NULL,
|
||||
key_prefix VARCHAR(16) UNIQUE, -- 前 16 字符明文(dfk_live_abc123...)UI 可显示
|
||||
key_hash BYTEA NOT NULL, -- 完整 key 的 sha256,比对用
|
||||
name VARCHAR(64), -- "生产环境 key" / "灰度 key"
|
||||
scopes TEXT[], -- ["threads:read", "runs:create", "uploads:write"...]
|
||||
rate_limit_rpm INT NULL, -- 每分钟请求数;NULL=用 workspace plan 默认
|
||||
expires_at TIMESTAMP NULL, -- 可选过期时间
|
||||
last_used_at TIMESTAMP NULL,
|
||||
revoked_at TIMESTAMP NULL,
|
||||
created_at
|
||||
)
|
||||
|
||||
external_users ( -- ghost user,按需建(identity_mode=external_passthrough 时)
|
||||
id UUID PK,
|
||||
workspace_id UUID FK NOT NULL,
|
||||
service_account_id UUID FK NOT NULL,
|
||||
external_id VARCHAR(128) NOT NULL, -- 业务系统传过来的 ID,原样存
|
||||
display_name VARCHAR(128) NULL,
|
||||
metadata JSONB, -- 可选业务字段
|
||||
created_at, last_active_at,
|
||||
UNIQUE (workspace_id, service_account_id, external_id)
|
||||
)
|
||||
```
|
||||
|
||||
### Key 格式约定
|
||||
|
||||
```
|
||||
dfk_live_<24 字符随机> # 生产 key
|
||||
dfk_test_<24 字符随机> # 测试 key
|
||||
```
|
||||
|
||||
- 前缀 `dfk_live_` / `dfk_test_` 让一眼区分环境(防止把测试 key 投进生产)
|
||||
- 写入 DB 时只存 `sha256(key)`,明文创建后只能在 UI 显示一次
|
||||
- `key_prefix` 列存前 16 字符(`dfk_live_abc12345`)—— UI 列表 + 审计日志可识别但不能用
|
||||
|
||||
### 认证流程
|
||||
|
||||
```
|
||||
请求 → AuthMiddleware → 检测 Authorization header
|
||||
│
|
||||
├── "Bearer dfk_..."
|
||||
│ ↓
|
||||
│ APIKeyAuthBackend.authenticate
|
||||
│ ↓
|
||||
│ SELECT api_keys WHERE key_hash = sha256(token)
|
||||
│ ↓
|
||||
│ load service_account + workspace
|
||||
│ ↓
|
||||
│ set_current_workspace(workspace_id)
|
||||
│ set_current_service_account(account)
|
||||
│ set_current_user(None) # 没有真人 user
|
||||
│ ↓
|
||||
│ CSRFMiddleware skip(bearer 路径不要 CSRF)
|
||||
│ ↓
|
||||
│ 如有 X-External-User-Id header 且 identity_mode 允许:
|
||||
│ → upsert external_users → set_current_external_user
|
||||
│
|
||||
└── "Cookie: access_token=..." → 走现有 web 前端流程
|
||||
```
|
||||
|
||||
### `@require_permission` 装饰器升级
|
||||
|
||||
```python
|
||||
# 现有签名(cookie 模式)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
|
||||
# 改为同时支持 service account 路径
|
||||
@require_permission(
|
||||
resource="threads",
|
||||
action="read",
|
||||
scopes=["threads:read"], # API key 必须有此 scope
|
||||
owner_check="workspace_or_user", # SA: 校验 workspace 归属;user: 校验 user_id
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. 核心设计:两种身份模式
|
||||
|
||||
### 模式 A:service account 折叠(identity_mode=`collapsed`)
|
||||
|
||||
业务系统调一切操作都归到该 service account 名下。**适合工具类集成**(CRM 自动总结、安全面板分析)。
|
||||
|
||||
```
|
||||
POST /api/v1/threads
|
||||
Authorization: Bearer dfk_live_xxx
|
||||
|
||||
→ 创建 thread,所有归属都是 service_account_id
|
||||
threads_meta.user_id = NULL
|
||||
threads_meta.service_account_id = <SA.id>
|
||||
threads_meta.workspace_id = <SA.workspace_id>
|
||||
```
|
||||
|
||||
memory 也是 service account 共享的(`workspaces/{wid}/service_accounts/{sa_id}/memory.json`)。
|
||||
|
||||
### 模式 B:external_user_id 透传(identity_mode=`external_passthrough`)
|
||||
|
||||
业务系统的"小明"在 DeerFlow 内独立。**适合 chatbot / 助手类集成**。
|
||||
|
||||
```
|
||||
POST /api/v1/threads
|
||||
Authorization: Bearer dfk_live_xxx
|
||||
X-External-User-Id: bizsys_user_42 # 业务系统的用户 ID
|
||||
|
||||
→ AuthMiddleware:
|
||||
1. 解 API key → load SA
|
||||
2. 看 SA.identity_mode 允许 external_passthrough
|
||||
3. SELECT external_users WHERE (workspace_id, service_account_id, external_id="bizsys_user_42")
|
||||
4. 没找到 → 自动建 ghost external_user
|
||||
5. set_current_external_user(...)
|
||||
|
||||
→ 创建 thread:
|
||||
threads_meta.workspace_id = <SA.workspace_id>
|
||||
threads_meta.service_account_id = <SA.id>
|
||||
threads_meta.external_user_id = <external_users.id>
|
||||
```
|
||||
|
||||
memory 是 per external_user 的(`workspaces/{wid}/service_accounts/{sa_id}/external_users/{eu_id}/memory.json`)。
|
||||
|
||||
### 模式选择策略
|
||||
|
||||
`SA.identity_mode` 三态:
|
||||
- `collapsed`:忽略所有 `X-External-User-Id` header
|
||||
- `external_passthrough`:必须传 `X-External-User-Id`,缺失时 400
|
||||
- `both`:传了走透传、不传走折叠(最灵活但最复杂;建议默认不开放)
|
||||
|
||||
业务系统接入时由 workspace owner 创建 SA 时选定。
|
||||
|
||||
### 这影响哪些 endpoints
|
||||
|
||||
| endpoint | service account 折叠 | external_user_id 透传 |
|
||||
|---|---|---|
|
||||
| `POST /threads` | thread 归 SA | thread 归 external_user |
|
||||
| `GET /threads` | 列出 SA 的所有 thread | 仅列出该 external_user 的 thread |
|
||||
| memory 注入 | SA 共享 memory | 该 external_user 的 memory |
|
||||
| `usage_daily` 写入 | `(workspace_id, SA_id, "main", ...)` | 同上 + `external_user_id` 维度 |
|
||||
| feedback | feedback.user_id 留空,标 SA | feedback.external_user_id 标人 |
|
||||
|
||||
---
|
||||
|
||||
## 3.5 核心设计:Pattern B(Browser 直连 + 短期 JWT 交换)
|
||||
|
||||
### 为什么需要
|
||||
|
||||
业务系统自研的 web 页面如果走 Pattern A(Backend 代理),他们 backend 必须实现 SSE 流式转发——这是个不小的工程量,且每个 token 多一跳延迟。**Pattern B 把 streaming 直接交给浏览器**,业务系统 backend 只做一次性 token 颁发。
|
||||
|
||||
### 数据模型扩展
|
||||
|
||||
```sql
|
||||
-- workspaces 表加列
|
||||
workspaces.allowed_origins TEXT[] -- ["https://app.partner.com", "https://staging.partner.com"]
|
||||
|
||||
-- 不需要新表;短期 JWT 不持久化(足够短就不需要 revoke list)
|
||||
```
|
||||
|
||||
### Token Exchange 流程
|
||||
|
||||
```
|
||||
[业务系统 backend] [DeerFlow] [浏览器]
|
||||
│ │ │
|
||||
│ 1. 用户在业务系统登录 │ │
|
||||
│ ◄────────────────────────────────│────────────────────────────────│
|
||||
│ │ │
|
||||
│ 2. 业务 backend 鉴权后调 │ │
|
||||
│ POST /api/v1/auth/exchange-token │
|
||||
│ Authorization: Bearer dfk_live_xxx │
|
||||
│ { external_user_id: "ming_42", expires_in: 600 } │
|
||||
│ ────────────────────────────────►│ │
|
||||
│ │ │
|
||||
│ │ 3. 校验 API key + SA 状态 │
|
||||
│ │ upsert external_users 行 │
|
||||
│ │ 签短期 JWT │
|
||||
│ │ │
|
||||
│ { access_token: "eyJ...", │ │
|
||||
│ expires_at: "..." } │ │
|
||||
│ ◄────────────────────────────────│ │
|
||||
│ │ │
|
||||
│ 4. 把 access_token 发给浏览器 │ │
|
||||
│ ─────────────────────────────────────────────────────────────────►│
|
||||
│ │ │
|
||||
│ │ 5. browser 直连 DeerFlow │
|
||||
│ │ GET /api/v1/threads/.../events│
|
||||
│ │ Authorization: Bearer eyJ...│
|
||||
│ │ Origin: https://app.partner.com│
|
||||
│ │ ◄──────────────────────────────│
|
||||
│ │ │
|
||||
│ │ 6. CORS preflight 通过 │
|
||||
│ │ SSE stream 200 │
|
||||
│ │ ──────────────────────────────►│
|
||||
```
|
||||
|
||||
### 短期 JWT 设计
|
||||
|
||||
```python
|
||||
# 不复用现有 cookie JWT 的 TokenPayload,新增 ServiceTokenPayload
|
||||
class ServiceTokenPayload(BaseModel):
|
||||
sub: str # external_user_id(DeerFlow 内部 id,不是业务方原始 id)
|
||||
sa: str # service_account_id
|
||||
wid: str # workspace_id
|
||||
eid: str # external_id(业务方原始 id,传给 audit log)
|
||||
scopes: list[str] # 从 api_key.scopes 继承(不能放大)
|
||||
exp: int # 5-15 min(默认 10 min)
|
||||
iat: int
|
||||
iss: "deerflow" # 区分自签 vs 业务方签
|
||||
typ: "service" # 区分 cookie JWT(typ=user)
|
||||
```
|
||||
|
||||
### Endpoint 规格
|
||||
|
||||
```
|
||||
POST /api/v1/auth/exchange-token
|
||||
Authorization: Bearer dfk_live_xxx (API key)
|
||||
Content-Type: application/json
|
||||
|
||||
Request:
|
||||
{
|
||||
"external_user_id": "ming_42", // 必填(identity_mode=external_passthrough)
|
||||
"expires_in": 600, // 可选,默认 600s,最大 3600s
|
||||
"scopes": ["threads:write", "runs:read"] // 可选;省略则继承 API key 全部 scopes
|
||||
}
|
||||
|
||||
Response 200:
|
||||
{
|
||||
"access_token": "eyJ...",
|
||||
"token_type": "Bearer",
|
||||
"expires_at": "2026-05-09T15:20:00Z"
|
||||
}
|
||||
|
||||
Errors:
|
||||
- 401 invalid API key
|
||||
- 403 SA suspended / API key revoked
|
||||
- 400 identity_mode 不允许 external_user_id(collapsed 模式)
|
||||
- 400 scopes 超出 API key 授权
|
||||
```
|
||||
|
||||
### AuthMiddleware 第三条路径
|
||||
|
||||
```
|
||||
请求 → AuthMiddleware → 检测 Authorization
|
||||
│
|
||||
├── "Bearer dfk_..." → APIKeyAuthBackend (Pattern A)
|
||||
├── "Bearer eyJ...typ=service" → ServiceTokenAuthBackend (Pattern B) ← 新增
|
||||
└── "Cookie: access_token=..." → CookieAuthBackend (web 前端)
|
||||
```
|
||||
|
||||
`ServiceTokenAuthBackend.authenticate`:
|
||||
1. 验 JWT 签名(DeerFlow 自签私钥;HS256 即可,不需要 RSA)
|
||||
2. 检 `iss=deerflow, typ=service`(防止把 cookie JWT 误用)
|
||||
3. 校 `sa` service account 状态(可能在签发后被 suspend)
|
||||
4. 校 scope 子集合法(不能超过 SA 当前 scopes)
|
||||
5. `set_current_workspace(wid)` + `set_current_service_account(sa)` + `set_current_external_user(sub)`
|
||||
|
||||
### CORS 设计
|
||||
|
||||
```python
|
||||
# CORSMiddleware 在 AuthMiddleware 之前加载(FastAPI 中间件顺序)
|
||||
class WorkspaceAwareCORSMiddleware:
|
||||
async def dispatch(self, request, call_next):
|
||||
origin = request.headers.get("origin")
|
||||
if not origin:
|
||||
return await call_next(request) # 非浏览器请求
|
||||
|
||||
# 解析当前 workspace(从 token 或 query string)
|
||||
# CORS preflight (OPTIONS) 没有 token,要从其他维度推
|
||||
# 简化方案:所有 /api/v1/* 路径在 OPTIONS 时回 wildcard,但不带 credentials;
|
||||
# 真请求时按 token 上的 wid 查 workspaces.allowed_origins 做精确匹配
|
||||
...
|
||||
```
|
||||
|
||||
**关键安全点**:
|
||||
- `Access-Control-Allow-Credentials: false`(短期 JWT 不依赖 cookie,不需要 credentials;防止意外打开 cookie 跨域)
|
||||
- `Access-Control-Allow-Origin` 精确匹配,不用通配
|
||||
- `Access-Control-Max-Age` 短一点(5 分钟),方便切换 origin 时不被缓存卡
|
||||
|
||||
### 与现有 CSRF 的关系
|
||||
|
||||
CSRFMiddleware 检测到 `Authorization: Bearer ...` 直接 skip——Pattern A 和 Pattern B 都走同一条 skip 逻辑。CSRF 仅对 cookie 路径生效。
|
||||
|
||||
### Token revoke 策略
|
||||
|
||||
短期 JWT 默认**不显式 revoke**(5-15 min 过期,自然失效)。但有三个例外:
|
||||
1. SA `status=suspended` → ServiceTokenAuthBackend 在 step 3 检测时直接拒
|
||||
2. API key 被 `revoked_at` → 同上(短期 JWT 验证时关联回 `sa.api_key`)
|
||||
3. 如果客户提需要立即 revoke 用户访问 → 业务系统调 DeerFlow `POST /api/v1/external-users/{id}/revoke-tokens` 把 `external_users.token_version` bump(`exchange-token` 签发时把它写进 JWT,验证时比对)
|
||||
|
||||
> 第 3 条是 nice-to-have;MVP 不做,等业务方明确提需求再加。
|
||||
|
||||
### Pattern B 不做的事(重要)
|
||||
|
||||
- **不**让浏览器直接持有 API key(`dfk_live_*`)——再短的 TTL 也不行;API key 必须留在业务方 backend
|
||||
- **不**支持 OAuth 2.0 完整 flow(authorize / consent / refresh)——太重,业务方 backend 自己做完用户认证再换 token 即可
|
||||
- **不**做客户端 SDK——给一份 OpenAPI + 浏览器原生 fetch / EventSource 例子,业务方自己接
|
||||
|
||||
---
|
||||
|
||||
## 4. 核心设计:API 版本化
|
||||
|
||||
**早做的成本**:仅是 `app/gateway/routers/__init__.py` 里 mount prefix 改 `/api/v1`。
|
||||
**晚做的成本**:所有业务系统 client 全量改地址。
|
||||
|
||||
### 设计
|
||||
|
||||
```python
|
||||
# 现状
|
||||
app.include_router(threads_router, prefix="/api/threads")
|
||||
|
||||
# Stage 1 改造
|
||||
app.include_router(threads_router, prefix="/api/v1/threads")
|
||||
# 同时保留 /api/threads → 转发到 v1(前端用),sunset 在 Stage 3
|
||||
```
|
||||
|
||||
**deprecation 策略**:
|
||||
- `/api/*`(无版本)路径在 response 加 `X-API-Deprecated: 2027-01-01` header
|
||||
- frontend 同步迁到 `/api/v1`
|
||||
- LangGraph SDK 兼容路径 `/api/langgraph/` 不带版本(跟随上游 SDK 约定)
|
||||
|
||||
**版本演进规则**:
|
||||
- 加字段:v1 内做,不升 v2
|
||||
- 改语义、删字段、改默认值:v2
|
||||
- 整体 endpoints 重组:v2
|
||||
|
||||
---
|
||||
|
||||
## 5. 核心设计:Rate Limit + Idempotency
|
||||
|
||||
### Rate Limit(基础版)
|
||||
|
||||
```
|
||||
api_keys.rate_limit_rpm 设了值:用 key 自己的
|
||||
没设:用 workspace plan 默认(free=60, pro=600, team=3000,可配)
|
||||
|
||||
实现:进程内 sliding window(依赖 Postgres 即可,不需要 Redis):
|
||||
rate_limit_log(api_key_id, minute_bucket, count) 单独小表
|
||||
```
|
||||
|
||||
进阶(Stage 2):分维度(per endpoint、per LLM、per sandbox quota)的复合 rate limit;引入 Redis。
|
||||
|
||||
### Idempotency Keys
|
||||
|
||||
```
|
||||
请求带 Idempotency-Key: <client-generated-uuid>
|
||||
↓
|
||||
SELECT idempotency_records WHERE (api_key_id, key=...) AND created_at > NOW() - 24h
|
||||
↓
|
||||
命中 → 直接返回上次 response(200 + body 原样)
|
||||
未命中 → 处理请求 → 写 idempotency_records + 返回
|
||||
```
|
||||
|
||||
只在写操作(POST / PUT / PATCH / DELETE)支持;GET 不需要。
|
||||
|
||||
---
|
||||
|
||||
## 6. 与 Stage 1 的整合
|
||||
|
||||
把 headless API MVP 包并入 Stage 1,**时间盒 8-13 周**(Postgres 切换已前移到 Stage 0,原"6-10 周 + 4-5 周 headless = 10-15"减去 Postgres 的 ~2 周)。
|
||||
|
||||
### 修订后 Stage 1 必做项
|
||||
|
||||
| 改动 | 类型 | 估工 |
|
||||
|---|---|---|
|
||||
| Quota 系统 + TokenUsage 持久化 | 原 Stage 1(Postgres 已就绪) | M+ |
|
||||
| Stripe 基础订阅 | 原 Stage 1 | M |
|
||||
| AioSandbox 出网/资源收紧 | 原 Stage 1 | M |
|
||||
| **API Key + Service Account 数据模型 + 仓储** | **新增(Pattern A)** | M |
|
||||
| **APIKeyAuthBackend + AuthMiddleware 双路径** | **新增(Pattern A)** | M |
|
||||
| **CSRF skip on bearer** | **新增** | XS |
|
||||
| **External User ID 透传 + ghost user** | **新增** | M |
|
||||
| **`/api/v1/` 版本化** | **新增**(早做便宜)| S |
|
||||
| **Per-API-key rate limit(基础版)** | **新增** | S |
|
||||
| **`@require_permission` 装饰器升级支持 SA 路径** | **新增** | S |
|
||||
| **API key 管理 UI(workspace settings 内)** | **新增**(最低限:CLI 也行)| S(CLI)/ M(UI)|
|
||||
| **`POST /api/v1/auth/exchange-token` endpoint** | **新增(Pattern B)** | S |
|
||||
| **`ServiceTokenAuthBackend`(短期 JWT 验证)** | **新增(Pattern B)** | S |
|
||||
| **`workspaces.allowed_origins` + CORS 中间件** | **新增(Pattern B)** | M |
|
||||
| **Idempotency keys** | **可选**(推荐) | S |
|
||||
|
||||
合计原 Stage 1 不含 Postgres(M+M+M=3M)+ 新增 Pattern A(M+M+XS+M+S+S+S+S=4M)+ 新增 Pattern B(S+S+M=2M)= 约 9M ≈ 8-13 周。Pattern B 依赖 Pattern A 完成,建议放 Stage 1 末。
|
||||
|
||||
### 修订后 Stage 1 PR 顺序
|
||||
|
||||
**轨道一:付费 SaaS 基础**(Postgres 已在 Stage 0 切完,本轨道直接从 quota 起)
|
||||
1. `workspace_quotas` / `workspace_usage_daily` 表 + 仓储
|
||||
2. `TokenUsageMiddleware` 升级为持久化
|
||||
3. `QuotaMiddleware` 加入中间件链
|
||||
4. Stripe webhook + 订阅状态同步
|
||||
5. AioSandbox 收紧
|
||||
6. 基础监控
|
||||
|
||||
**轨道二:Headless API Pattern A**(与轨道一并行;无前置依赖)
|
||||
1. `service_accounts` + `api_keys` + `external_users` 仓储(**先于业务路径**)
|
||||
2. `APIKeyAuthBackend` + `AuthMiddleware` 双路径(cookie + bearer)
|
||||
3. CSRF middleware skip on bearer
|
||||
4. `/api/v1/` mount prefix 切换 + 旧路径兼容转发
|
||||
5. `external_user_id` 透传机制(依赖 1-4)
|
||||
6. `service_accounts.identity_mode` 三态行为分支
|
||||
7. `@require_permission` 升级 + scope 校验
|
||||
8. 基础 rate limit
|
||||
9. API key 管理 CLI + UI
|
||||
10. Idempotency keys(可选)
|
||||
|
||||
**轨道三:Headless API Pattern B**(依赖轨道二的 1-7 完成;建议 Stage 1 最后 1-2 周)
|
||||
1. `workspaces.allowed_origins` 列 + workspace settings UI 的 origin 管理
|
||||
2. `WorkspaceAwareCORSMiddleware`(在 AuthMiddleware 之前)
|
||||
3. `POST /api/v1/auth/exchange-token` endpoint + `ServiceTokenPayload` 设计
|
||||
4. `ServiceTokenAuthBackend`(AuthMiddleware 第三条路径)
|
||||
5. SSE 在 CORS 跨域下的 streaming 验证(写一个 fixture web 页面测)
|
||||
6. 业务方文档:从换 token 到浏览器直连的端到端示例(HTML + JS)
|
||||
|
||||
---
|
||||
|
||||
## 7. SaaS vs on-prem 差异(SaaS 是主线)
|
||||
|
||||
> **口径**:phased-rollout §0 已明确"中心化 SaaS 主线,不做 self-host 主线"。本节列出**如果**未来按客户合同启用 on-prem 时的差异点——目的是让 Stage 0/1 的 schema 与 auth 设计**不阻塞** on-prem,而不是把 on-prem 当作并行产品线投入资源。
|
||||
|
||||
| 能力 | SaaS 形态 | on-prem 形态(按合同启用) |
|
||||
|---|---|---|
|
||||
| API key 管理 | workspace settings UI + CLI | CLI 必须;UI 可选;env var 注入预置 key 也合理 |
|
||||
| 配额 / 计费 | 按 plan,Stripe 同步 | 配额作为容量管理(防内部失控),不接 Stripe |
|
||||
| KMS | AWS / 阿里云 KMS(Stage 2 后落) | 客户自带 KMS(HashiCorp Vault / 客户自有);env var 兜底(明文)|
|
||||
| 监控 | 你们运维的 Grafana | 客户自有;DeerFlow 暴露 `/metrics` Prometheus 端点(Stage 2 加)|
|
||||
| Webhook | 平台默认 | 客户内网回调;要求 url 可配置 |
|
||||
| Rate limit | 平台分档强制 | 客户自定义;默认放宽 |
|
||||
| 升级路径 | 你们灰度发布 | docker image tag + 升级文档;schema migration 跑通 |
|
||||
|
||||
**重要**:Stage 0 的 schema 设计已经兼容两者(workspace 是 self-hosted 时也 1 对 1 对应一个安装)。**不需要为 on-prem 单独建分支**。
|
||||
|
||||
### on-prem 专属工作(Stage 1+ 一次性)
|
||||
|
||||
- docker compose 模板(已部分有,加 service_accounts seed 流程)
|
||||
- "首次安装预置 1 个 workspace + 1 个 admin + 1 个 platform key" 的 init script
|
||||
- 升级脚本(schema migrations 自动跑)
|
||||
- 部署文档(一份就够)
|
||||
|
||||
工作量约 1 人周,可以在 Stage 1 末或 Stage 2 头做。
|
||||
|
||||
---
|
||||
|
||||
## 8. 不可逆决策(动手前想清楚)
|
||||
|
||||
| 决策 | 难回头的原因 |
|
||||
|---|---|
|
||||
| API Key 格式(前缀 `dfk_` / 长度 / 加密路径)| 业务系统接入后改格式所有 key 全失效 |
|
||||
| `service_accounts` 表是否能跨 workspace(暂定不允许) | 改了所有 quota / 计费归属逻辑 |
|
||||
| `X-External-User-Id` header 名 | 业务系统集成后改 header 名要全部联调 |
|
||||
| `identity_mode` 三态语义(collapsed / external / both) | 改语义要联调所有业务系统 |
|
||||
| `/api/v1/` 是不是从 Stage 1 第 5 步开始;具体 deprecation 时间 | 业务系统接了之后再要求换 prefix 很不友好 |
|
||||
| ghost user 自动创建策略(X-External-User-Id 缺失时拒绝还是 fallback collapsed) | 改了业务系统接入测试要重跑 |
|
||||
| memory 隔离粒度(SA 共享 vs per external_user) | 一旦客户用上,迁移 memory 数据极麻烦 |
|
||||
| **短期 JWT TTL 默认值**(5 / 10 / 15 min)和 max | 业务系统集成后调短会断当前会话;调长会留更大被偷窃的窗口 |
|
||||
| **`ServiceTokenPayload` 字段集**(claim 名 / 是否带 `eid`) | 改 claim 名所有签名校验失败;早一点把审计需要的字段都设计进去 |
|
||||
| **`workspaces.allowed_origins` 是 workspace 级还是 SA 级** | 暂定 workspace 级(更简单);改成 SA 级要拆数据 |
|
||||
|
||||
---
|
||||
|
||||
## 9. 推荐推进路径
|
||||
|
||||
**第 0-1 周**:
|
||||
- 找现有需要集成的业务系统聊一下,把"两种身份模式哪个适合 / 是否有 webhook 需求 / 是否有 rate limit 偏好"问清楚
|
||||
- 写一份 API key 创建 + 第一次 hello-world 调用的快速指南草稿(不写代码,纯设计验证)
|
||||
|
||||
**第 2-3 周(Stage 0 末)**:
|
||||
- Stage 0 收尾时把 `service_accounts` / `api_keys` / `external_users` schema 也加上(schema 加上,路径不接)
|
||||
|
||||
**第 4-13 周(Stage 1)**:
|
||||
- 按上面的 13 步 PR 顺序推进
|
||||
- 第 6-8 周可以拉一个业务系统做内测集成(用真实 API key 调真实 endpoint)
|
||||
- 第 12-13 周收尾,给业务系统正式 go-live
|
||||
|
||||
**与 Stage 1 业务客户上线的关系**:
|
||||
- Stage 1 一头是付费 SaaS 客户、一头是业务系统集成客户。两条产品线**共用同一套** workspace + auth + quota,**只是认证模式不同**(cookie vs bearer)
|
||||
- 因此架构上没有冲突,团队可以并行推
|
||||
|
||||
---
|
||||
|
||||
## 10. 与现有 ADR / rollout 的关系
|
||||
|
||||
| 引用 | 关系 |
|
||||
|---|---|
|
||||
| [adr-007 §6 路由](../01-redesign/adr-007-routing-frontend.zh-CN.md) | 路由层假设 cookie + JWT;本文档加了 bearer + API key 平行路径 |
|
||||
| [adr-007 §8 Auth](../01-redesign/adr-007-routing-frontend.zh-CN.md) | TokenPayload 扩 `tid/role` 已对齐;本文档新增 API key 路径不复用 JWT |
|
||||
| [adr-004 RBAC](../01-redesign/adr-004-tenant-rbac.zh-CN.md) | service_accounts.role 复用 owner/admin/member;不另建 role 体系 |
|
||||
| [adr-003 §4.4 quota](../01-redesign/adr-003-llm-key-billing.zh-CN.md) | quota 写入按 SA 归属时只看 workspace_id;按 external_user 归属时多一维 |
|
||||
| [phased-rollout-by-scale Stage 1](./phased-rollout-by-scale.zh-CN.md) | 本文档 §6 给出 Stage 1 修订后必做项与 PR 顺序 |
|
||||
| [stage-0-code-map](./stage-0-code-map.zh-CN.md) | Stage 0 schema 时把 `service_accounts` 也加上不阻塞 |
|
||||
@@ -0,0 +1,331 @@
|
||||
# 多租户改造 · 按规模分期落地方案
|
||||
|
||||
> 写于 2026-05-09。基于已收敛的目标客户画像 + ADR 决策。
|
||||
>
|
||||
> **目标客户画像**:
|
||||
> - 用户群体:以个人用户为主,少量小团队
|
||||
> - workspace 模型:**统一只有 workspace 概念**,个人用户 = 1 人 workspace;团队 = 多成员 workspace(无单独"个人空间"概念)
|
||||
> - 部署形态:**中心化 SaaS**(DeerFlow 团队运维,不做 self-host 主线)
|
||||
> - 商业化:**Freemium**——免费用户 + 付费分层,DeerFlow 付 LLM 账单
|
||||
>
|
||||
> 这三个画像决定了分期取舍:**成本/隔离要早做、企业级特性可以一直推迟**。
|
||||
|
||||
---
|
||||
|
||||
## 总览
|
||||
|
||||
| Stage | 触发条件(业务事实) | 主旋律 | 时间盒 |
|
||||
|---|---|---|---|
|
||||
| **0** | 现在 → 第一个付费客户准备 | workspace 模型立起来;**Postgres 切换**;现有 auth 收紧;不做真隔离 | 4–5 周 |
|
||||
| **1** | 第一批付费客户(10–50 付费 / 500–2000 free) + **1-2 业务系统集成(含自研 web 页面)** | **Quota + Headless API(Pattern A backend 代理 + Pattern B browser 直连)必落**;workspace 全链路 + 入口强校验;AioSandbox 收紧 | 8–13 周 |
|
||||
| **2** | 增长期(100–500 付费 / 5k–20k 用户) | DeerFlow 表 RLS、KMS、ObjectStorage S3、内部 LLM 计费分类、付费分层、Webhook outbound | 10–16 周 |
|
||||
| **3** | 成熟期(1k+ 付费 / 50k+ 用户)**或** 出现安全/成本事故 | K8s sandbox + namespace、BYO key(付费档福利)、audit DB 拆分、prewarm 池 | 16–26 周 |
|
||||
| **4** | 单客户合同驱动(合规 / 企业销售) | SSO、custom domain、per-tenant DB(仅强合规) | 按需,单客户 4–8 周 |
|
||||
|
||||
> **并行轨道**:Stage 1 同时承载"第一批付费 SaaS 客户"和"业务系统集成"两条产品线,共用 workspace + auth 基座;详见 [headless-api-track.zh-CN.md](./headless-api-track.zh-CN.md)。On-prem 部署形态在 Stage 0 schema 设计层面就已兼容,Stage 1 末加部署文档即可。
|
||||
|
||||
**核心原则**:每期只做下一档规模真正逼出来的事;做了就不回头的"不可逆决策"集中在 Stage 0/1,避免后期重写。
|
||||
|
||||
---
|
||||
|
||||
## Stage 0 — workspace 模型立起来 + Postgres 切换 + auth 收紧
|
||||
|
||||
**触发**:你现在所在的位置——刚把 ADR 收敛完,准备开第一个付费客户。
|
||||
**退出**:能给一个外部用户开账号,他登进来看到自己的 workspace、能创建 thread、隔离干净;生产已跑在 Postgres 上。
|
||||
**时间盒**:4–5 周(原 3–4 周;Postgres 切换 + testcontainers + 部署/onboarding 调整加 1 周)
|
||||
|
||||
### 必做
|
||||
|
||||
| 改动 | 说明 |
|
||||
|---|---|
|
||||
| **Postgres 切换**(dev + 生产)| **不可逆决策**——Stage 0 没有生产数据,迁移阻力最小;现在切完省掉 Stage 1 重 ALTER 一遍的返工。`init_engine_from_config` 已支持双驱动,docker-compose 加 PG service、`make setup`/`make doctor`/CI 切默认。**这是 §3.5 底座先行的成果落地**,不是单独 spike。 |
|
||||
| **Postgres testcontainers + RLS 测试夹具骨架** | phase-0 §3.5 底座之一;CI 跑通至少 1 个 RLS 冒烟测试模板(Stage 0 还没用 RLS,但夹具就位) |
|
||||
| **`workspaces` 表 + 自动建 1 人 workspace** | 每个新注册用户自动获得 1 个 workspace;用户 = workspace owner。这是后面所有租户改造的底座。 |
|
||||
| **`workspace_id` 列加到现有表**(直接在 Postgres 上加)| `threads_meta` / `runs` / `feedback` / `users` 加 `workspace_id`。**不可逆决策**——直接在 Postgres 上 ALTER 一次,不再走 SQLite → Postgres 二次迁移。 |
|
||||
| **`workspace_memberships` 表** | 即使个人用户也是"1 个 owner 成员",团队功能未启用但模型先就位。`role` 字段先只有 `owner`。 |
|
||||
| **`service_accounts` / `api_keys` / `external_users` schema** | Stage 1 才接路径,但 schema 在 Stage 0 末加上不阻塞——避免 Stage 1 临时改表。详见 [headless-api-track §2](./headless-api-track.zh-CN.md)。 |
|
||||
| **JWT 扩 `wid` 字段** | 沿用现有 `app/gateway/auth/jwt.py` `TokenPayload`(参 ADR-007 §8 修订版),加 `wid`(workspace_id),不引入 Better Auth。 |
|
||||
| **入口路由 `(workspace_id, thread_id)` 校验** | `threads.py` + `thread_runs.py` 入口处必校验(参 ADR-001 §4.1.2)。Stage 0 就上,避免 Stage 1 临时补。 |
|
||||
| **CLI / admin UI 的"workspace 管理"基础** | platform admin 能看 workspace 列表、暂停/删除某个 workspace(防止滥用第一时间反应)。 |
|
||||
| **现有 auth 完善** | setup flow 能创建第一个 admin、邀请用户走基本流程(不必 invitation token,可手动建账号)。`token_version` 已存在,复用。 |
|
||||
|
||||
### 不做(推迟到 Stage 1+)
|
||||
|
||||
- ❌ RLS policy 启用 — Stage 2(夹具 Stage 0 就位,但 policy 不上)
|
||||
- ❌ Quota 系统 — Stage 1(早一点也行,但有了第一个付费客户再做反应快)
|
||||
- ❌ K8s sandbox — Stage 3
|
||||
- ❌ KMS / ObjectStorage S3 — Stage 2
|
||||
- ❌ 团队 invitation 流程 — Stage 2(先不做,反正还没真团队用户)
|
||||
- ❌ Stripe 对接 — Stage 1
|
||||
|
||||
### 关键 PR 顺序(避免半截不可运行)
|
||||
|
||||
1. **Postgres 接入 + testcontainers**:docker-compose 加 PG、`make doctor` 兼容、CI 跑通;现有 SQLite 数据导入(如有 dev 数据)
|
||||
2. **将默认 backend 切到 Postgres**:`make setup` / `make dev` / `.env.example` 默认指向 PG;SQLite 保留为可选 dev 兜底
|
||||
3. `workspaces` + `workspace_memberships` 表 + 仓储
|
||||
4. 注册流程改造(自动建 1 人 workspace)+ JWT 扩 `wid`
|
||||
5. 现有表 ALTER 加 `workspace_id` 列(直接在 Postgres 上加,先 nullable)+ 数据回填脚本("legacy_workspace")→ ALTER 改 NOT NULL
|
||||
6. `threads.py` / `thread_runs.py` 入口校验 + 迁移所有现有 thread 到对应 workspace
|
||||
7. CI boundary 测试:禁止任何路径绕过入口直连 LangGraph saver
|
||||
8. `service_accounts` / `api_keys` / `external_users` schema only(Stage 0 末,为 Stage 1 准备)
|
||||
|
||||
### Go/No-Go 进入 Stage 1
|
||||
|
||||
- 第一个付费意向客户出现
|
||||
- Stage 0 已部署到生产 ≥ 2 周,无 workspace 隔离 bug 报告
|
||||
- 生产已稳定运行在 Postgres 上 ≥ 2 周,无 schema / 性能 regression
|
||||
|
||||
---
|
||||
|
||||
## Stage 1 — 第一批付费客户 + 业务系统集成
|
||||
|
||||
**触发**:Stage 0 跑稳 + 拿到第一批付费用户(10–50 付费 / 500–2000 free)+ 1-2 个业务系统集成需求。
|
||||
**退出**:① 能放心让媒体/产品社区曝光,不会被白嫖跑偏;② 业务系统能用 API key 调通核心 endpoint,go-live。
|
||||
**时间盒**:8–13 周(原 10-15 周;Postgres 切换已在 Stage 0 完成,省 2 周)
|
||||
|
||||
> Stage 1 是**双轨并行**:付费 SaaS(cookie auth + Stripe + quota)和 Headless API(bearer auth + service account + `/api/v1/`)。两者共用 workspace + auth + quota 基座(Stage 0 已落地)。详细 headless API 设计见 [headless-api-track.zh-CN.md](./headless-api-track.zh-CN.md)。
|
||||
|
||||
### 必做(付费 SaaS 轨道)
|
||||
|
||||
| 改动 | 说明 | 关联 ADR |
|
||||
|---|---|---|
|
||||
| **Quota 系统 v1**(强制) | `workspace_quotas` + `workspace_usage_daily` 表;`QuotaMiddleware` 在 lead_agent 链最前;硬限到达拒调用。**Freemium 不上 quota = 信用卡递给攻击者**。 | ADR-003 §4.4 |
|
||||
| **`TokenUsageMiddleware` 持久化** | 当前只 log(参 audit ADR-003);要写入 `workspace_usage_daily(workspace_id, date, model, tokens_in, tokens_out)`,按 SA / external_user 维度同时支持。 | ADR-003 §4.3 |
|
||||
| **悲观预扣**(轻量版) | 按 `model_max_input_tokens` 估上限;幽灵 token 防御。 | ADR-003 §4.4.1 |
|
||||
| **Stripe 对接(基础订阅)** | 单档付费先;webhook 同步到 `workspace_quotas.plan` 字段。 | — |
|
||||
| **AioSandbox 出网收紧** | egress 白名单(默认禁出网,按需放行)+ cgroup CPU/memory 限额。**不上 K8s**——AioSandbox 加这两个补丁就能撑到 Stage 3。 | ADR-002(轻量版) |
|
||||
| **Sandbox 资源 quota** | 每 workspace 的"沙箱 CPU 秒/月"也进 quota(防止白嫖跑挖矿)。 | ADR-003 §4.3 |
|
||||
| **基础监控** | per-workspace token 用量曲线、quota 命中率、异常用量告警。 | — |
|
||||
|
||||
### 必做(Headless API 轨道 - Pattern A:业务 backend 代理)
|
||||
|
||||
| 改动 | 说明 | 关联文档 |
|
||||
|---|---|---|
|
||||
| **API Key + Service Account 仓储** | `service_accounts` / `api_keys` 表(schema Stage 0 已加)+ 仓储 + 哈希存储。 | headless-api §2 |
|
||||
| **APIKeyAuthBackend + AuthMiddleware 双路径** | bearer 走 SA 路径、cookie 走 user 路径;CSRF middleware 在 bearer 路径 skip。 | headless-api §2 |
|
||||
| **External User ID 透传 + ghost user** | `external_users` 表(schema Stage 0 已加)+ `X-External-User-Id` header 解析;`identity_mode` 三态语义。 | headless-api §3 |
|
||||
| **`/api/v1/` 版本化** | mount prefix 切换;旧 `/api/*` 兼容转发并加 deprecation header。**早做便宜**。 | headless-api §4 |
|
||||
| **`@require_permission` 装饰器升级** | 同时支持 cookie user 路径和 SA + scope 校验;`owner_check` 扩为 enum(`workspace_or_user`)。 | headless-api §2 |
|
||||
| **Per-API-key rate limit(基础版)** | sliding window,存 Postgres;分档默认配 free/pro/team。 | headless-api §5 |
|
||||
| **API key 管理(CLI 优先 + UI 跟进)** | workspace owner / admin 创建 SA + key + 选 identity_mode;CLI 必有,UI 在前端 workspace settings 跟。 | headless-api §2 |
|
||||
| **Idempotency keys**(推荐) | 业务系统重试不重复建 thread/run。 | headless-api §5 |
|
||||
|
||||
### 必做(Headless API 轨道 - Pattern B:自研 web 浏览器直连)
|
||||
|
||||
| 改动 | 说明 | 关联文档 |
|
||||
|---|---|---|
|
||||
| **`POST /api/v1/auth/exchange-token` endpoint** | 业务系统 backend 用 API key + `external_user_id` 换 5-15 min 短期 JWT。 | headless-api §3.5 |
|
||||
| **`ServiceTokenAuthBackend`(AuthMiddleware 第三条路径)** | 验短期 JWT 签名 + `iss=deerflow,typ=service` + SA 当前状态 + scope 子集合法。 | headless-api §3.5 |
|
||||
| **`workspaces.allowed_origins` 列 + WorkspaceAwareCORSMiddleware** | per-workspace 配置允许的 origin;浏览器请求过 CORS preflight。 | headless-api §3.5 |
|
||||
| **SSE 在 CORS 跨域下的 streaming 验证** | 写一份业务方对接示例(HTML + 原生 EventSource) | headless-api §3.5 |
|
||||
|
||||
### 不做(推迟到 Stage 2+)
|
||||
|
||||
- ❌ RLS — Stage 2(应用层 + 入口校验先撑着)
|
||||
- ❌ KMS — Stage 2(先用环境变量管理 key + 平台 key 散列入 DB)
|
||||
- ❌ ObjectStorage S3 — Stage 2(先继续本地文件 + 备份脚本)
|
||||
- ❌ K8s sandbox — Stage 3
|
||||
- ❌ BYO key — Stage 3(先全部用平台 key + quota)
|
||||
- ❌ 团队 invitation 流程 — Stage 2(除非有团队客户先到)
|
||||
- ❌ 多档付费 — Stage 2
|
||||
- ❌ Webhook outbound — Stage 2(先轮询)
|
||||
- ❌ 分维度 / 分档 rate limit — Stage 2
|
||||
|
||||
### 关键 PR 顺序(双轨)
|
||||
|
||||
**轨道 A:付费 SaaS**(Postgres 已在 Stage 0 切完,本轨道直接从 quota 起)
|
||||
1. `workspace_quotas` / `workspace_usage_daily` 表 + 仓储
|
||||
2. `TokenUsageMiddleware` 升级为持久化(参 audit + ADR-003 §4.3)
|
||||
3. `QuotaMiddleware` 加入中间件链
|
||||
4. Stripe webhook + 订阅状态同步到 `workspace_quotas.plan`
|
||||
5. AioSandbox egress 白名单 + 资源限额
|
||||
6. 监控/告警接入
|
||||
|
||||
**轨道 B:Headless API Pattern A**(与 A 并行;无前置依赖)
|
||||
1. `service_accounts` / `api_keys` / `external_users` 仓储(schema 已在 Stage 0 加上)
|
||||
2. `APIKeyAuthBackend` + `AuthMiddleware` 双路径(cookie + bearer)
|
||||
3. CSRF middleware skip on bearer
|
||||
4. `/api/v1/` mount prefix 切换 + 旧路径兼容转发
|
||||
5. `external_user_id` 透传机制
|
||||
6. `service_accounts.identity_mode` 三态行为分支
|
||||
7. `@require_permission` 升级 + scope 校验(依赖轨道 A 的 quota 完成)
|
||||
8. 基础 rate limit
|
||||
9. API key 管理 CLI + UI
|
||||
10. Idempotency keys(可选)
|
||||
|
||||
**轨道 C:Headless API Pattern B**(依赖轨道 B 的 1-7 完成;建议 Stage 1 末 1-2 周)
|
||||
1. `workspaces.allowed_origins` 列 + workspace settings UI 的 origin 管理
|
||||
2. `WorkspaceAwareCORSMiddleware`(在 AuthMiddleware 之前)
|
||||
3. `POST /api/v1/auth/exchange-token` endpoint + `ServiceTokenPayload` 设计
|
||||
4. `ServiceTokenAuthBackend`(AuthMiddleware 第三条路径)
|
||||
5. SSE 跨域 streaming 验证 + 业务方对接示例(HTML + JS)
|
||||
|
||||
### Go/No-Go 进入 Stage 2
|
||||
|
||||
- 月活付费用户 ≥ 50 **或** 月活免费用户 ≥ 1000
|
||||
- 1-2 个业务系统集成完成 go-live 并稳定运行 ≥ 1 个月
|
||||
- 出现一次"差点超额"事件(quota 在悲观预扣下还是漏了一次)
|
||||
- 文件存储或 secret 管理出现一次手忙脚乱(备份遗漏 / key 误提交等)
|
||||
- 业务系统开始要求 webhook 推送(不再满足于轮询)
|
||||
|
||||
---
|
||||
|
||||
## Stage 2 — 增长期,安全与隔离深化
|
||||
|
||||
**触发**:用户量级跳到下一档(100–500 付费 / 5k–20k 用户)。
|
||||
**退出**:架构能撑住"用户翻 10 倍而不爆炸",团队功能上线。
|
||||
**时间盒**:10–16 周
|
||||
|
||||
### 必做
|
||||
|
||||
| 改动 | 说明 | 关联 ADR |
|
||||
|---|---|---|
|
||||
| **DeerFlow 表 RLS** | 仅 DeerFlow 自有表(threads_meta / runs / feedback / workspace_*)启用 RLS;testcontainers 必须先有。LangGraph 表继续走应用层强校验。 | ADR-001 §4.1.1(修订版)+ §4.2 |
|
||||
| **Postgres testcontainers + RLS smoke 测试** | phase-0 §3.5 底座之一;CI 里跑 RLS 冒烟。 | phase-0 §3.5 |
|
||||
| **ObjectStorage Protocol + Local + S3 实现** | 用户上传 / 产物 / 技能包迁到对象存储;按 `workspaces/{wid}/` prefix。 | ADR-005 §4 + §5 |
|
||||
| **KMS 抽象 + AWS/阿里云 KMS 接入** | secret 加密落地;envelope encryption。dev 仍可明文 fallback。 | ADR-003 §4.1 |
|
||||
| **多档付费分层** | Free / Pro / Team 三档;quota 按档次配;Stripe 多 product。 | — |
|
||||
| **团队 workspace invitation 流程** | `invitations` 表 + 邀请链接 + 邮件;新成员 join workspace 后 bump 用户 `token_version`。 | ADR-004 §5 |
|
||||
| **per-workspace MCP cache** | `mcp/cache.py` 模块单例 → `WorkspaceMCPCache` 类;OAuth token 落 KMS 加密 DB。 | ADR-006 §2.2 |
|
||||
| **per-user skill 覆盖** | 在 workspace 级 `tenant_skill_state` 之上加 `user_skill_overrides(workspace_id, user_id, skill_name, enabled)`。解析时 `final_enabled = user_override ?? workspace_default`。UI 仅在团队 workspace 显示"个人偏好"开关;1 人 workspace 隐藏。 | ADR-005 §5.4 扩展 |
|
||||
| **per-user skill config** | 新建 `user_skill_configs(workspace_id, user_id, skill_name, config_encrypted)`,KMS 加密。承载 skill 私有配置(API key、个人偏好等)——这部分不能共享。 | ADR-005 §5.4 + ADR-003 §4.1 |
|
||||
| **skill 上传权限收口** | workspace owner / admin 才能上传 skill 包;其他成员只能 enable/disable + 填自己的 config。 | ADR-004 §5.4 |
|
||||
| **内部 LLM 计费分类** | Memory/Title/Summarization 三类 LLM 调用都计入 workspace 用量,区分 `usage_category`。 | ADR-006 §2.5 |
|
||||
| **role 扩到 owner/admin/member** | 团队 workspace 出现 → RBAC 真正发挥作用;`@require_permission` 装饰器升级。 | ADR-004 §5.4 |
|
||||
| **基础 audit log** | 写入业务 DB(暂不拆分),关键操作(quota 改、role 改、删 workspace、API key 创建/吊销)记录。 | — |
|
||||
| **Webhook outbound** | `webhook_subscriptions` 表 + 重试机制;业务系统订阅 thread 完成 / run 失败 / quota 触底。Stage 1 推迟来的,此时业务系统已经开始要。 | headless-api §1 |
|
||||
| **API key 分维度 rate limit** | per-endpoint / per-LLM / per-sandbox 复合限速;引入 Redis。 | headless-api §5 |
|
||||
|
||||
### 不做(推迟到 Stage 3+)
|
||||
|
||||
- ❌ K8s sandbox — Stage 3
|
||||
- ❌ BYO key — Stage 3
|
||||
- ❌ audit DB 拆分 — Stage 3
|
||||
- ❌ prewarm pool — Stage 3
|
||||
- ❌ SSO / custom domain — Stage 4
|
||||
|
||||
### 关键 PR 顺序
|
||||
|
||||
1. testcontainers + Postgres CI 跑通
|
||||
2. ObjectStorage Protocol + Local 实现 + 端到端打通(用户上传走对象存储)
|
||||
3. KMS 抽象 + AWS KMS 实现 + 已有 secret 灰度迁移
|
||||
4. DeerFlow 表 RLS(testcontainers 验证后上生产)
|
||||
5. S3 实现 + 上传/产物迁移
|
||||
6. 多档付费 + invitation 流程 + role 扩展(这块可并行)
|
||||
7. WorkspaceMCPCache + OAuth token 加密
|
||||
8. **skill per-user 覆盖 + config 加密**(依赖 6 的 RBAC + 3 的 KMS)
|
||||
9. 内部 LLM 计费分类
|
||||
|
||||
### Go/No-Go 进入 Stage 3
|
||||
|
||||
- 月活付费 ≥ 500 **或** 月活总用户 ≥ 20k
|
||||
- 单一 sandbox 进程出现资源争用(一个 workspace 卡死影响其他)
|
||||
- 出现一次跨 workspace 数据访问尝试(哪怕只是日志里看到)
|
||||
- 客户开始问"我能不能用我自己的 OpenAI key"
|
||||
|
||||
---
|
||||
|
||||
## Stage 3 — 成熟期,K8s 隔离 + BYO
|
||||
|
||||
**触发**:Stage 2 出口条件中任意一条。
|
||||
**退出**:架构能撑到 enterprise 销售前夕。
|
||||
**时间盒**:16–26 周
|
||||
|
||||
### 必做
|
||||
|
||||
| 改动 | 说明 | 关联 ADR |
|
||||
|---|---|---|
|
||||
| **K8s namespace + NetworkPolicy** | per-workspace namespace + 默认禁出网;`K8sSandboxProvider` 全新建。 | ADR-002 §1 + §5 |
|
||||
| **prewarm pool** | per-workspace 池,按 plan 大小(Free=0、Pro=1、Team=3)。 | ADR-006 §2.4 |
|
||||
| **BYO LLM key(付费档福利)** | Pro/Team 用户可填自己的 OpenAI/Anthropic key;BYO 时不走 quota。 | ADR-003 §4.6 |
|
||||
| **S3 实现已就位**(Stage 2 已建)→ 加 presigned URL + lifecycle | — | ADR-005 §5 |
|
||||
| **audit DB 拆分** | 独立连接池或独立实例;关键操作 + 沙箱审计独立写。 | ADR-006 §2.4(脚注) |
|
||||
| **跨 region 备份 / DR** | 至少 1 个备份 region + RTO ≤ 4h。 | — |
|
||||
| **完整监控栈** | Grafana + Prometheus + APM;per-workspace SLA 跟踪。 | — |
|
||||
|
||||
### 不做(推迟到 Stage 4)
|
||||
|
||||
- ❌ gVisor / Kata(K8s + NetworkPolicy 已经足够,gVisor 是 nice-to-have)
|
||||
- ❌ SSO
|
||||
- ❌ custom domain
|
||||
- ❌ per-tenant DB
|
||||
|
||||
### Go/No-Go 进入 Stage 4
|
||||
|
||||
- 拿到第一个 enterprise 合同(合同写明 SSO / 合规要求 / 自定义域名)
|
||||
- 拿到金融/医疗/政府类客户
|
||||
|
||||
---
|
||||
|
||||
## Stage 4 — 企业化,按需开启
|
||||
|
||||
**触发**:单客户合同驱动,不做"为了万一"的提前准备。
|
||||
**退出**:— 长期持续。
|
||||
**时间盒**:每个 enterprise 客户 4–8 周
|
||||
|
||||
### 按客户需求选做
|
||||
|
||||
| 客户要 | 你做 | 关联 ADR |
|
||||
|---|---|---|
|
||||
| SSO(SAML/OIDC) | 接入 Auth.js / 自实现 SAML provider;`workspace_memberships` 与外部 group 映射 | ADR-007 §8 SSO 段 |
|
||||
| 自定义域名 | `workspaces.custom_domain` 列;ACME 动态签证;nginx vhost 路由 | ADR-007 §9 |
|
||||
| 物理数据隔离(合规) | per-tenant DB(仅该客户)+ 独立连接池 | ADR-001 §6 推翻条件 |
|
||||
| 私有部署(self-host)| 打 enterprise tier docker 镜像 + 部署文档;放弃中心化运维优势 | — |
|
||||
| gVisor / Kata 运行时 | K8s 切 gVisor RuntimeClass | ADR-002 §5 |
|
||||
| 合规审计报告(SOC2/ISO) | 强化 audit log 留存 + 评估机构对接 | — |
|
||||
|
||||
---
|
||||
|
||||
## 不可逆决策一览(重点关注)
|
||||
|
||||
| 决策 | 在哪 Stage 做 | 做错了的代价 |
|
||||
|---|---|---|
|
||||
| **Postgres 切换**(dev + 生产)| Stage 0 | Stage 0 选这个时机:没有生产数据,迁移阻力最小;切完不回头 |
|
||||
| **`workspace_id` 列加到所有业务表**(直接 Postgres)| Stage 0 | 漏了某张表 → Stage 1 还在补;不再走 SQLite → Postgres 二次迁移 |
|
||||
| **`workspaces` 表设计**(slug、plan、status 字段) | Stage 0 | 后期改 schema 要写迁移;用户 URL 全变 |
|
||||
| **JWT payload 字段** | Stage 0 一次加齐 `wid+role` | 加字段时旧 cookie 失效;workspace-schema-design §4 锁定一次到位,避免 Stage 2 再 bump |
|
||||
| **ObjectStorage prefix 形态**(`workspaces/{wid}/...`) | Stage 2 | 改了所有用户产物 URL 失效 |
|
||||
| **K8s namespace 命名规则**(`ws-{wid}` 还是 `tenant-{wid}`) | Stage 3 | 改了所有 NetworkPolicy / RBAC |
|
||||
|
||||
**建议**:Stage 0 的 `workspaces` 表 schema、URL slug 形态、JWT 字段集 这三件事**多设计一周也不亏**——后面不可逆,错了重写代价大。
|
||||
|
||||
---
|
||||
|
||||
## 与 ADR 的对应关系
|
||||
|
||||
每个 Stage 对应 ADR 子集,**不是全部 ADR 一起做**:
|
||||
|
||||
| Stage | 启用的 ADR 章节 |
|
||||
|---|---|
|
||||
| 0 | ADR-001 §4.1.2(入口校验,SQLite 版)+ ADR-004 §1-§3(owner-only 简化版)+ ADR-007 §1-§5(无 slug 路由可暂缓) |
|
||||
| 1 | ADR-001 §4.1(应用层校验完整版,**不含 RLS**)+ ADR-002 §3(AioSandbox 补丁版)+ ADR-003 §4.3-§4.4(quota + 持久化)+ ADR-007 §6-§8(slug 路由 + JWT 扩字段) |
|
||||
| 2 | ADR-001 §4.2(DeerFlow 表 RLS)+ ADR-003 §4.1-§4.2(KMS + create_chat_model 改造)+ ADR-004 §5(完整 RBAC)+ ADR-005 §1-§5(ObjectStorage 完整)+ ADR-006 §2.2 + §2.5 + §2.6 |
|
||||
| 3 | ADR-002 §1-§5(K8s 完整)+ ADR-003 §4.6(BYO)+ ADR-006 §2.4(prewarm 池)+ ADR-007 §9(custom domain 预留) |
|
||||
| 4 | ADR-001 §6(per-tenant DB)+ ADR-002 §5(gVisor)+ ADR-007 §8 SSO 段 + §9(custom domain 落地) |
|
||||
|
||||
---
|
||||
|
||||
## 工时预估汇总
|
||||
|
||||
| Stage | 触发 | 时间盒 | 累计 |
|
||||
|---|---|---|---|
|
||||
| 0 | 现在 | 4–5 周(含 Postgres 切换 + testcontainers) | 1.0–1.3 个月 |
|
||||
| 1 | 首批付费 + 业务系统集成(含自研 web 直连) | 8–13 周(headless API Pattern A+B 并行 4-5 周;Postgres 切换已前移到 Stage 0) | 3–4 个月 |
|
||||
| 2 | 增长期 | 10–16 周 | 7–8 个月 |
|
||||
| 3 | 成熟期 | 16–26 周 | 13–14 个月 |
|
||||
| 4 | 企业客户 | 单客户 4–8 周 | + 按需 |
|
||||
|
||||
**全功能落地**:~13-14 个月(Stage 0–3 累计),不含 Stage 4 enterprise 特性。
|
||||
**最小可付费 + 业务系统集成**(Stage 0 + 1):~3-4 个月。
|
||||
**风险可控的增长**(Stage 0 + 1 + 2):~7-8 个月。
|
||||
|
||||
---
|
||||
|
||||
## 推翻条件
|
||||
|
||||
整个分期方案在以下情况下要重排:
|
||||
|
||||
- **目标客户画像突变**:比如拿到一个 enterprise 合同要求 SSO + custom domain → Stage 4 部分提前到 Stage 2
|
||||
- **出现安全事故**:跨 workspace 泄露 / sandbox 逃逸 → 立即跳过未启动的 Stage,直接做 Stage 3 的 K8s + RLS
|
||||
- **付费转化远不及预期**:Stage 1 上线 6 个月付费用户 < 10 → 重新评估 freemium 模型,可能不需要走完 Stage 2/3
|
||||
- **LLM 价格大跌 / 自部署模型成熟**:成本控制优先级下降 → quota 与 BYO 可以简化
|
||||
@@ -0,0 +1,374 @@
|
||||
# Stage 0 · 当前代码地图
|
||||
|
||||
> 目的:把 Stage 0 必做项落到具体文件 + 行号上,让你在动手前能快速找到要改的位置。
|
||||
>
|
||||
> **范围**:仅 Stage 0 涉及的 7 个子系统。Stage 1+ 的代码(quota / KMS / RLS / sandbox 等)不在本文档。
|
||||
>
|
||||
> **配套阅读**:
|
||||
> - [02-rollout/phased-rollout-by-scale.zh-CN.md](./phased-rollout-by-scale.zh-CN.md):Stage 0 必做项清单
|
||||
> - [01-redesign/adr-vs-code-audit.zh-CN.md](../01-redesign/adr-vs-code-audit.zh-CN.md):原始审计来源(部分行号引用此处)
|
||||
|
||||
---
|
||||
|
||||
## 0. 整体链路速览
|
||||
|
||||
```
|
||||
Browser ─→ nginx :2026 ─→ Gateway :8001
|
||||
│
|
||||
│ 请求
|
||||
▼
|
||||
┌──────────────────────┐
|
||||
│ CSRFMiddleware │ 写操作校验 X-CSRF-Token
|
||||
│ AuthMiddleware │ cookie → JWT → ContextVar
|
||||
└──────────┬───────────┘
|
||||
│ ContextVar 注入 _current_user
|
||||
▼
|
||||
┌──────────────────────┐
|
||||
│ Routers │ threads.py / thread_runs.py / auth.py / ...
|
||||
│ @require_permission │ owner_check via threads_meta
|
||||
└──────────┬───────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────┐
|
||||
│ Repositories │ resolve_user_id(AUTO) → 从 ContextVar 读
|
||||
│ ThreadMetaRepository │ 仓储自动填充 user_id 列
|
||||
│ RunRepository │
|
||||
│ ... │
|
||||
└──────────┬───────────┘
|
||||
│
|
||||
▼ SQLAlchemy AsyncSession
|
||||
┌──────────────────────┐
|
||||
│ Postgres / SQLite │ threads_meta / runs / feedback / users
|
||||
└──────────────────────┘
|
||||
|
||||
┌──────────────────────┐
|
||||
│ Agent 运行时 │ ThreadDataMiddleware → Paths.thread_dir(user_id, thread_id)
|
||||
│ Sandbox 工具 │ replace_virtual_path 把 /mnt/... 翻译到物理路径
|
||||
└──────────────────────┘
|
||||
```
|
||||
|
||||
**Stage 0 的工作就是**:在这条链路上每个层级都补一个 `workspace_id` 维度,并在入口路由层强制 `(workspace_id, thread_id)` 校验。
|
||||
|
||||
---
|
||||
|
||||
## 1. Auth 体系
|
||||
|
||||
### 关键文件
|
||||
|
||||
| 文件 | 行号 | 作用 |
|
||||
|---|---|---|
|
||||
| `backend/app/gateway/auth/jwt.py` | `12-19` | `TokenPayload` 定义:`{sub, exp, iat, ver}` |
|
||||
| `backend/app/gateway/auth/jwt.py` | `21-37` | `create_access_token()` 签发 |
|
||||
| `backend/app/gateway/auth/jwt.py` | `40-55` | `decode_token()` 验证 |
|
||||
| `backend/app/gateway/auth_middleware.py` | `52-127` | `AuthMiddleware`:cookie → JWT → ContextVar 注入 |
|
||||
| `backend/app/gateway/auth_middleware.py` | `112,122,124-126` | 注入/重置 ContextVar 的关键三行 |
|
||||
| `backend/app/gateway/csrf_middleware.py` | `169-216` | `CSRFMiddleware` 双重 cookie |
|
||||
| `backend/app/gateway/authz.py` | `197-301` | `@require_permission(resource, action, owner_check)` 装饰器 |
|
||||
| `backend/packages/harness/deerflow/runtime/user_context.py` | `52-167` | `CurrentUser` Protocol + ContextVar + AUTO sentinel + `resolve_user_id` |
|
||||
|
||||
### 关键函数
|
||||
|
||||
| 名称 | 位置 | 现状签名 |
|
||||
|---|---|---|
|
||||
| `TokenPayload` | `auth/jwt.py:12` | `BaseModel` 字段:`sub:str, exp:datetime, iat:datetime|None, ver:int` |
|
||||
| `set_current_user(user)` | `user_context.py:55-62` | 注入 ContextVar,返回 reset token |
|
||||
| `resolve_user_id(value, *, method_name)` | `user_context.py:138-167` | 三态:`AUTO`/explicit `str`/`None` |
|
||||
| `AuthMiddleware.dispatch` | `auth_middleware.py:75-126` | cookie 检查 → JWT 解码 → user 入 ContextVar + `request.state.user` |
|
||||
|
||||
### 当前数据流
|
||||
|
||||
1. 登录:`POST /auth/login/local` 验 password → 签 JWT 含 `sub=user.id, ver=user.token_version` → 写 `access_token` cookie(HttpOnly)
|
||||
2. 后续请求:`AuthMiddleware` 读 cookie → `decode_token` → 查库验 user 存在且 `ver` 匹配 → `set_current_user(user)` 注入 ContextVar → `request.state.user = user`
|
||||
3. 路由:`@require_permission` 取 `request.state.user`,对 thread 资源调 `ThreadMetaStore.check_access(thread_id, user.id)`
|
||||
|
||||
### Stage 0 改动锚点
|
||||
|
||||
- `TokenPayload`:在 `jwt.py:12-19` 加 `wid: str` 字段(参 ADR-007 §8);`create_access_token` 签发处一并加
|
||||
- `AuthMiddleware.dispatch`:在 `set_current_user` 之后再调一个新的 `set_current_workspace(workspace_id)`(要新建 `workspace_context.py`,仿照 `user_context.py`)
|
||||
- `decode_token`:兼容旧 4 字段 token,缺 `wid` 时强制走 `/select-workspace` 重发(参 ADR-007 §11)
|
||||
|
||||
---
|
||||
|
||||
## 2. User 数据模型 + 现有业务表
|
||||
|
||||
### 关键文件
|
||||
|
||||
| 文件 | 行号 | 作用 |
|
||||
|---|---|---|
|
||||
| `backend/packages/harness/deerflow/persistence/user/model.py` | `22-59` | `UserRow` ORM |
|
||||
| `backend/app/gateway/auth/models.py` | `15-42` | `User` Pydantic(API 接口)|
|
||||
| `backend/packages/harness/deerflow/persistence/engine.py` | `26-27,126` | engine 单例 + `AsyncSession` factory |
|
||||
|
||||
### 现有 ORM 模型清单(**Stage 0 全部要加 `workspace_id` 列**)
|
||||
|
||||
| 表 | 模型文件 | 现有 user_id 列 | 备注 |
|
||||
|---|---|---|---|
|
||||
| `users` | `persistence/user/model.py:22-59` | — (它就是 user 本身) | 加 `default_workspace_id`(用户登录后默认进哪个 workspace)|
|
||||
| `threads_meta` | `persistence/thread_meta/model.py:18` | `user_id String(64) index` | 加 `workspace_id` + `UNIQUE(workspace_id, thread_id)` 复合索引 |
|
||||
| `runs` | `persistence/run/model.py:19` | `user_id String(64) index` | 加 `workspace_id` |
|
||||
| `feedback` | `persistence/feedback/model.py:21` | `user_id String(64) index` | 加 `workspace_id` |
|
||||
| `run_events` | `persistence/run/model.py` (推测同目录) | (unverified — 部分版本是 DB,部分是内存)| 如果是 DB 持久化则加 `workspace_id` |
|
||||
|
||||
### `UserRow` 关键列
|
||||
|
||||
| 列 | 行号 | 现状 | Stage 0 |
|
||||
|---|---|---|---|
|
||||
| `id` | `model.py:26` | `String(36)` PK | 不变 |
|
||||
| `email` | `model.py` | unique | 不变 |
|
||||
| `password_hash` | `model.py` | nullable | 不变 |
|
||||
| `system_role` | `model.py:33` | default `"user"`,可为 `"admin"` | 保留为**平台级 role**(platform_admin),不和 workspace role 混 |
|
||||
| `token_version` | `model.py:49` | default 0;改密时 bump | 沿用——加入/退出 workspace 时 bump |
|
||||
| `needs_setup` | `model.py:48` | default False | 不变 |
|
||||
| **`default_workspace_id`** | — | (新增) | 登录后默认进哪个 workspace;可为 NULL(用户多 workspace 时强制走 picker)|
|
||||
|
||||
### Stage 0 改动锚点
|
||||
|
||||
- 新建 `persistence/workspace/model.py` + `persistence/workspace/sql.py`(仓储)
|
||||
- 新建 `persistence/workspace_membership/model.py` + 仓储
|
||||
- 现有 4 张表 ALTER 加 `workspace_id` 列(先 nullable,回填后改 NOT NULL)
|
||||
- 写迁移脚本:所有现有 `users` 自动建 1 个 workspace(owner=自己),把 `threads_meta`/`runs`/`feedback` 的现有行 `workspace_id` 回填为对应 user 的 default_workspace_id
|
||||
|
||||
---
|
||||
|
||||
## 3. Thread 入口路由(Stage 0 强校验落点)
|
||||
|
||||
### 关键文件
|
||||
|
||||
| 文件 | 行数 | 作用 |
|
||||
|---|---|---|
|
||||
| `backend/app/gateway/routers/threads.py` | 622 行 | thread CRUD |
|
||||
| `backend/app/gateway/routers/thread_runs.py` | 377 行 | run 创建/恢复/事件流 |
|
||||
|
||||
### threads.py 关键 endpoint
|
||||
|
||||
| 路由 | 行号 | 现状 |
|
||||
|---|---|---|
|
||||
| `POST /api/threads` | `224-286` | 调 `ThreadMetaRepository.create(thread_id, user_id=AUTO)` + 初始 checkpoint |
|
||||
| `DELETE /api/threads/{thread_id}` | `190-221` | `_delete_thread_data` + 移 checkpoint + 删 thread_meta |
|
||||
| `POST /api/threads/search` | `289+` | 委托 `ThreadMetaStore` |
|
||||
| `GET /api/threads/{thread_id}` | (其他) | 状态查询 |
|
||||
| `PATCH /api/threads/{thread_id}` | (其他) | 更新 metadata |
|
||||
|
||||
### thread_runs.py 关键 endpoint
|
||||
|
||||
| 路由 | 行号 | 现状 |
|
||||
|---|---|---|
|
||||
| `POST /api/threads/{tid}/runs` | `95-100` | `start_run()` 后台 |
|
||||
| `POST /api/threads/{tid}/runs/stream` | (其他) | run + SSE |
|
||||
| `POST /api/threads/{tid}/runs/wait` | (其他) | run + 阻塞 |
|
||||
| `GET /.../runs/{rid}/messages` | (其他) | 分页消息 |
|
||||
| `GET /.../runs/{rid}/events` | (其他) | 完整事件流 |
|
||||
|
||||
### 当前权限模型
|
||||
|
||||
所有 thread 端点都用 `@require_permission("threads", "<action>", owner_check=True)` 装饰;`owner_check=True` 触发 `ThreadMetaStore.check_access(thread_id, user.id)`,只比对 `threads_meta.user_id == current_user.id`。
|
||||
|
||||
### Stage 0 改动锚点
|
||||
|
||||
- **`@require_permission` 装饰器升级**:`owner_check` 当前是 bool,要扩成支持 `(workspace_id, thread_id)` 复合校验(参 ADR-001 §4.1.2 + ADR-004 §5.4)
|
||||
- **创建路径**:`threads.py:224-286` 写 thread_meta 时也要写 `workspace_id`(从 ContextVar 读);依赖 thread_meta 上的 `UNIQUE(workspace_id, thread_id)` 复合索引兜底
|
||||
- **读/写路径**:先用 `(current_workspace_id, requested_thread_id)` SELECT thread_meta,未命中即 404
|
||||
- 这两个文件就是 ADR-001 §4.1.2 修订后的"第一道防线"落点
|
||||
|
||||
---
|
||||
|
||||
## 4. ThreadDataMiddleware + 路径系统
|
||||
|
||||
### 关键文件
|
||||
|
||||
| 文件 | 行号 | 作用 |
|
||||
|---|---|---|
|
||||
| `backend/packages/harness/deerflow/agents/middlewares/thread_data_middleware.py` | `24-79` | 中间件创建 thread 目录树 |
|
||||
| `backend/packages/harness/deerflow/config/paths.py` | `1-250` | `Paths` 类(虚拟路径系统)|
|
||||
| `backend/packages/harness/deerflow/sandbox/middleware.py` | `45-63` | `SandboxMiddleware`(紧随 ThreadDataMiddleware 之后)|
|
||||
|
||||
### Paths 关键方法
|
||||
|
||||
| 方法 | 位置 | 现状返回 |
|
||||
|---|---|---|
|
||||
| `Paths.thread_dir(thread_id, user_id)` | `paths.py:171-189` | `{base_dir}/users/{user_id}/threads/{thread_id}/` |
|
||||
| `Paths.sandbox_work_dir` | `paths.py:191-197` | `{thread_dir}/user-data/workspace/` |
|
||||
| `Paths.sandbox_uploads_dir` | `paths.py` | `{thread_dir}/user-data/uploads/` |
|
||||
| `Paths.sandbox_outputs_dir` | `paths.py` | `{thread_dir}/user-data/outputs/` |
|
||||
| `Paths.user_memory_file(user_id)` | `paths.py:155-157` | `{base_dir}/users/{user_id}/memory.json` |
|
||||
| `Paths.user_agent_dir(user_id, agent_name)` | `paths.py:163-169` | `{base_dir}/users/{user_id}/agents/{name}/` |
|
||||
|
||||
### ThreadDataMiddleware 关键函数
|
||||
|
||||
| 方法 | 位置 | 现状 |
|
||||
|---|---|---|
|
||||
| `_get_thread_paths(thread_id, user_id)` | `thread_data_middleware.py:52-66` | 算 workspace_path / uploads_path / outputs_path |
|
||||
| `_create_thread_directories(thread_id, user_id)` | `thread_data_middleware.py:68-79` | 调 `Paths.ensure_thread_dirs` |
|
||||
|
||||
### 当前数据流
|
||||
|
||||
agent 启动 → ThreadDataMiddleware 在 `before_model` 调 `_create_thread_directories(thread_id, user_id=get_effective_user_id())` → 沙箱工具(bash 等)通过 `replace_virtual_path` 把 `/mnt/user-data/workspace/foo.txt` 翻译到 `{base_dir}/users/{user_id}/threads/{thread_id}/user-data/workspace/foo.txt`。`user_id` 没认证时 fallback `"default"`。
|
||||
|
||||
### Stage 0 改动锚点
|
||||
|
||||
| 文件 | 改什么 |
|
||||
|---|---|
|
||||
| `paths.py:171-189` `thread_dir()` | 路径加 workspace 维度:`{base_dir}/workspaces/{wid}/threads/{thread_id}/`(**注意**:原 `{base_dir}/users/{user_id}/threads/...` 是不可逆的迁移点,要写迁移脚本把现有目录搬到新形态)|
|
||||
| `paths.py:155-157` `user_memory_file` | 同上加 `/workspaces/{wid}/users/{uid}/memory.json`(memory 仍 per-user,但放 workspace 下)|
|
||||
| `paths.py:163-169` `user_agent_dir` | 同上 |
|
||||
| `thread_data_middleware.py:52-79` | 取 workspace_id 也从 ContextVar(新建 `get_effective_workspace_id`),传给 Paths |
|
||||
|
||||
> **重要**:路径迁移是 Stage 0 的不可逆点之一。建议 Stage 0 PR 顺序里把"加路径维度"放最后一步,前面所有 PR 跑通后再切。
|
||||
|
||||
---
|
||||
|
||||
## 5. 仓储层访问模式
|
||||
|
||||
### 当前模式(以 `ThreadMetaRepository.create` 为例)
|
||||
|
||||
```python
|
||||
# persistence/thread_meta/sql.py:30-56
|
||||
async def create(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO, # 哨兵默认值
|
||||
...
|
||||
) -> dict:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="create")
|
||||
row = ThreadMetaRow(
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
...
|
||||
)
|
||||
```
|
||||
|
||||
### 关键点
|
||||
|
||||
- 所有仓储遵循相同模式:参数默认 `AUTO`、入口调 `resolve_user_id` 一次、WHERE/INSERT 使用 resolved 值
|
||||
- 跨 ContextVar 边界(后台任务、线程)调仓储要显式传 `user_id=...` 或 `user_id=None`(绕过隔离)
|
||||
|
||||
### `check_access` 用法(owner_check 逻辑)
|
||||
|
||||
```python
|
||||
# persistence/thread_meta/sql.py:74-101
|
||||
async def check_access(self, thread_id: str, user_id: str, *, require_existing: bool = False):
|
||||
# SELECT 行,比 row.user_id == user_id;不匹配 raise 403
|
||||
```
|
||||
|
||||
### Stage 0 改动锚点
|
||||
|
||||
- 在 `runtime/` 下新建 `workspace_context.py`,仿 `user_context.py` 提供 `_current_workspace` ContextVar + `set_current_workspace` + `resolve_workspace_id` + `_AutoSentinel`
|
||||
- 所有仓储方法加 `workspace_id: str | None | _AutoSentinel = AUTO` 参数(仿 user_id 模式)
|
||||
- WHERE 子句加 `workspace_id = :wid`(必须放索引前导列,复合索引重新设计)
|
||||
- `check_access` 升级为 `check_access(thread_id, user_id, workspace_id, *, require_existing)`
|
||||
|
||||
---
|
||||
|
||||
## 6. Setup / 注册流程
|
||||
|
||||
### 后端流程
|
||||
|
||||
| 步骤 | 路由 / 文件 | 行号 | 现状 |
|
||||
|---|---|---|---|
|
||||
| 1. 检查首启 | `GET /auth/setup-status` | `auth.py:390-417` | 数 `admin_count == 0` |
|
||||
| 2. 建首个 admin | `POST /auth/initialize` | `auth.py:429-458` | `system_role="admin", needs_setup=False` + auto-login |
|
||||
| 3. 普通注册 | `POST /auth/register` | `auth.py:304-323` | `system_role="user"` + auto-login |
|
||||
| 4. 改密码/finish setup | `POST /auth/change-password` | `auth.py:328-375` | bump `token_version`,重签 JWT |
|
||||
| 5. 当前用户 | `GET /auth/me` | `auth.py:378-382` | 返回 `User` Pydantic |
|
||||
|
||||
### 前端
|
||||
|
||||
- `frontend/src/app/(auth)/setup/page.tsx`(位置 unverified,需要 ls 确认)
|
||||
- `frontend/src/core/auth/server.ts:25-26` 读 `access_token` cookie 调 `/auth/me`
|
||||
- `frontend/src/core/auth/proxy-policy.ts:52` cookie name `access_token`
|
||||
|
||||
### Stage 0 改动锚点
|
||||
|
||||
- `POST /auth/initialize` (`auth.py:429-458`):建完 admin 后**同步**建 1 个默认 workspace(owner=该 admin),写 `users.default_workspace_id`
|
||||
- `POST /auth/register` (`auth.py:304-323`):每次注册新用户也建 1 个 1 人 workspace(owner=新用户)
|
||||
- `POST /auth/change-password` (`auth.py:328-375`):不变
|
||||
- `GET /auth/me` (`auth.py:378-382`):返回值加 `workspaces: [{id, name, role}]`,前端 picker 用
|
||||
- 前端 setup flow:登录成功后如果用户只有 1 个 workspace,直接进;多个走 picker(Stage 1+ 的事,但 Stage 0 数据结构要支持)
|
||||
|
||||
---
|
||||
|
||||
## 7. CSRF + ContextVar 注入(一并)
|
||||
|
||||
### CSRFMiddleware 行为
|
||||
|
||||
- `csrf_middleware.py:169-216`:对 POST/PUT/DELETE/PATCH 校验
|
||||
- Auth 端点(login/register/initialize):仅校 Origin(跨源保护)
|
||||
- 其他端点:要求 `X-CSRF-Token` header 与 `csrf_token` cookie 完全相等
|
||||
|
||||
### AuthMiddleware ContextVar 流程
|
||||
|
||||
| 行号 | 动作 |
|
||||
|---|---|
|
||||
| `auth_middleware.py:112` | `user = await get_current_user_from_request(request)` (JWT 验 + 库查) |
|
||||
| `auth_middleware.py:120` | `request.state.user = user` |
|
||||
| `auth_middleware.py:122` | `token = set_current_user(user)` |
|
||||
| `auth_middleware.py:124-126` | `try / finally: reset_current_user(token)` |
|
||||
|
||||
### 当前数据流
|
||||
|
||||
每个 FastAPI 请求是独立 task → 独立 ContextVar context → user 注入即可让所有下游 await 链都能取到。`asyncio.create_task` 子任务自动继承父 context;`threading.Timer` 不继承(memory queue 已经显式捕获 user_id 解决)。
|
||||
|
||||
### Stage 0 改动锚点
|
||||
|
||||
- 在 `auth_middleware.py:122` 之后再调 `workspace_token = set_current_workspace(user.default_workspace_id_or_resolved_from_jwt)`
|
||||
- `finally` 块同时 reset 两个
|
||||
- 所有 `threading.Timer` / 后台 task 都要显式捕获 workspace_id(参考现有 memory queue 已对 user_id 做的)
|
||||
|
||||
---
|
||||
|
||||
## Stage 0 改动影响面总览
|
||||
|
||||
把 Stage 0 必做项(来自 02-rollout/phased-rollout-by-scale.zh-CN.md)映射到代码位置:
|
||||
|
||||
| Stage 0 必做项 | 主要文件 | 涉及 §节 |
|
||||
|---|---|---|
|
||||
| `workspaces` 表 + 自动建 1 人 workspace | 新建 `persistence/workspace/{model,sql}.py`;改 `auth.py:429-458` `/auth/initialize` 与 `auth.py:304-323` `/auth/register` | §2 + §6 |
|
||||
| `workspace_id` 列加到现有表 | `thread_meta/model.py:18`、`run/model.py:19`、`feedback/model.py:21`、`user/model.py`(加 `default_workspace_id`)+ 迁移脚本 | §2 |
|
||||
| `workspace_memberships` 表 | 新建 `persistence/workspace_membership/{model,sql}.py` | §2 |
|
||||
| JWT 扩 `wid` 字段 | `auth/jwt.py:12-19,21-37,40-55` | §1 |
|
||||
| 入口路由 `(workspace_id, thread_id)` 校验 | `routers/threads.py:224-286,190-221`、`routers/thread_runs.py:95-100`;`authz.py:197-301` 装饰器升级 | §3 |
|
||||
| ThreadDataMiddleware 加 workspace 维度 | `thread_data_middleware.py:52-79`、`config/paths.py:171-197,155-169` + 文件系统迁移脚本 | §4 |
|
||||
| 仓储层加 workspace_id 哨兵 | 新建 `runtime/workspace_context.py`;所有 `persistence/*/sql.py` 加参数 | §5 |
|
||||
| AuthMiddleware 注入 workspace ContextVar | `auth_middleware.py:122,124-126` | §1 + §7 |
|
||||
| Setup flow 自动建 workspace | `auth.py:429-458,304-323` | §6 |
|
||||
|
||||
---
|
||||
|
||||
## 不可逆决策落点(动手前想清楚)
|
||||
|
||||
| 决策 | 一旦合入难回头的原因 |
|
||||
|---|---|
|
||||
| `workspaces` 表 schema 字段(slug / plan / status / 自定义域名预留列) | 改 schema 要写迁移;如果 URL 用了 slug,所有用户书签失效 |
|
||||
| `workspace_id` 列类型(UUID v7 vs string vs int) | 一致性;和 `users.id` 类型对齐 |
|
||||
| JWT `TokenPayload` 字段集(一次性想清楚 `wid + role + plan` 都加上还是分次加)| 每次加字段 bump `ver` 让所有用户重新登录;少一次 churn 用户体验好 |
|
||||
| 文件系统路径形态(`workspaces/{wid}/threads/{tid}/` vs `tenants/{tid}/...`) | 改了所有用户产物 URL 失效;迁移脚本要重写 |
|
||||
| ContextVar 命名(`_current_workspace` 还是 `_current_tenant`) | 影响 import 全链路;和 ADR 用语保持一致(建议用 `workspace`,因为产品概念用了它)|
|
||||
|
||||
---
|
||||
|
||||
## 推荐阅读顺序(最快上手)
|
||||
|
||||
按依赖关系从底层到上层读,每读完一份能对下一份有更准确的预期:
|
||||
|
||||
1. **`runtime/user_context.py:52-167`** — AUTO sentinel 模式是整个改造的核心模式,先吃透
|
||||
2. **`persistence/user/model.py:22-59`** — 看清 UserRow 现有字段
|
||||
3. **`persistence/thread_meta/{model.py,sql.py}`** — 选一个仓储看完整 CRUD pattern
|
||||
4. **`auth/jwt.py:12-55`** — TokenPayload + sign/decode(最小、5 分钟读完)
|
||||
5. **`app/gateway/auth_middleware.py:75-126`** — 看 ContextVar 怎么注入
|
||||
6. **`app/gateway/authz.py:197-301`** — `@require_permission` 装饰器实现
|
||||
7. **`app/gateway/routers/threads.py:190-286`** — Stage 0 强校验的落点 1
|
||||
8. **`app/gateway/routers/thread_runs.py:95-100`** — Stage 0 强校验的落点 2
|
||||
9. **`app/gateway/routers/auth.py:304-458`** — register / initialize / change-password
|
||||
10. **`agents/middlewares/thread_data_middleware.py` + `config/paths.py:155-197`** — 文件系统迁移用得到
|
||||
11. **`app/gateway/csrf_middleware.py:169-216`**(可选,CSRF 现成不太需要改)
|
||||
|
||||
预计阅读时间:1.5–2.5 小时(粗读)/ 半天(精读 + 跑一遍 dev 调用链)
|
||||
|
||||
---
|
||||
|
||||
## 出现疑问时
|
||||
|
||||
- ADR 现状假设错了 → 检查 [adr-vs-code-audit.zh-CN.md](../01-redesign/adr-vs-code-audit.zh-CN.md)
|
||||
- LangGraph saver 注入相关 → [adr-spike-langgraph-postgres.zh-CN.md](../01-redesign/adr-spike-langgraph-postgres.zh-CN.md)
|
||||
- 改造方向 → [adr-001](../01-redesign/adr-001-data-isolation.zh-CN.md) §4.1.2 + [adr-007](../01-redesign/adr-007-routing-frontend.zh-CN.md) §8
|
||||
@@ -0,0 +1,94 @@
|
||||
# Stage 0 进度面板
|
||||
|
||||
> **每完成 1 个 PR 后必更新**。本文是 Stage 0 唯一的"现在到哪了"权威来源——其它文件(plan、ADR、各 PR impl note)都是静态的,不反映执行进度。
|
||||
>
|
||||
> 上次更新:2026-05-13,PR6 merge 进 docs branch 后
|
||||
|
||||
## 一句话状态
|
||||
|
||||
PR1 + PR2 + PR3 + PR4 + PR5 + **PR6** 已 merge。**PR6 (2026-05-13)** 落地:4 个业务仓储 30+ 方法的 `workspace_id` 哨兵 + WHERE;`check_access` 升级三参数 (`thread_id, user_id, workspace_id`);`@require_permission` 装饰器接入 `get_effective_workspace_id()`,跨 workspace **404 not 403**;`Paths` 切 workspace 维度(`{base}/workspaces/{wid}/threads/{tid}/...` + per-user state 嵌套);`ThreadDataMiddleware` 切 workspace;`scripts/migrate_paths_to_workspace.py` 文件迁移(带 dry-run + 冲突分流);lifespan 探测残留 `users/` 时 WARNING 引导跑 `make migrate-paths`;**T5.11 ORM `nullable=False` 一并翻**(PR5 推迟项就位)。**3214 passed + 30 skipped + 17 caplog flake**(PR5 末 3150 + 30 + 16;+64 测试,+1 flake——新 flake `test_path_migration_pending_warning::test_warns`,solo 跑 PASS)。**下一个:PR7(CI boundary 静态扫描)**。
|
||||
|
||||
## 8 PR 状态表
|
||||
|
||||
| PR | 状态 | Commits | 分支 / 落点 | impl note |
|
||||
|---|---|---|---|---|
|
||||
| **PR0** | ✅ merged | 1 | `a74b88a4` on docs branch | — |
|
||||
| **PR1** | ✅ merged | 8 (T1.1-T1.10) | merged into docs branch (`fab85b14..85a14f4c`) | [pr1-postgres-setup.md](./pr1-postgres-setup.md) |
|
||||
| **PR2** | ✅ merged | 8 (T2.1-T2.10) | merged into docs branch (`404135a1..1112a197`) | [pr2-postgres-default.md](./pr2-postgres-default.md) |
|
||||
| **PR3** | ✅ merged | 7 (T3.1-T3.10) | merged into docs branch (`f63089ae..dda82640`) | [pr3-workspaces.md](./pr3-workspaces.md) |
|
||||
| **PR4** | ✅ merged | 14 (T4.1-T4.14) | merged into docs branch (`d98498b7..5c7753c0`) | [pr4-auth-workspace.md](./pr4-auth-workspace.md) |
|
||||
| **PR5** | ✅ merged | 11 (T5.1-T5.10 + T5.12) | merged into docs branch (`a7326978..30f2bd00`) | [pr5-business-workspace-id.md](./pr5-business-workspace-id.md) |
|
||||
| **PR6** | ✅ merged | 13 (T5.11 + T6.1-T6.15) | merged into docs branch (`361e653d..87ea715c`) | [pr6-routes-paths-workspace.md](./pr6-routes-paths-workspace.md) |
|
||||
| **PR7** | 🟡 pending | 0 | — | — |
|
||||
| **PR8** | 🟡 pending | 0 | — | — |
|
||||
|
||||
**测试基线**:**PR6 末 3214 passed + 30 skipped**(PR5 末 3150 + 30;+64 PR6 新测试,覆盖 thread_meta workspace_id 过滤、Run/Feedback/RunEvent 同款、require_permission probes、跨 workspace 404 e2e、Paths workspace 形态、ThreadDataMiddleware workspace、文件迁移脚本、lifespan warning)。PR4 末 3136 + 26;PR3 末 3134 + 25;PR2 末 3087。**17 个 caplog 排序 flake 持续存在**(16 个 pre-existing + 1 新增 `test_path_migration_pending_warning::test_warns`)→ isolate 跑全 PASS,与 stage 无关;集中清理仍推迟到 follow-up。
|
||||
|
||||
## 用户必须跟进的事(live verification / 决策)
|
||||
|
||||
下列任务**只能用户做**,agent 没权限或没环境:
|
||||
|
||||
| 项 | 状态 | 谁做 | 怎么做 |
|
||||
|---|---|---|---|
|
||||
| 启动 Docker daemon 后实跑 PG smoke 测试(testcontainers 路径)| ⏳ | 用户 | `docker compose -f docker/docker-compose-dev.yaml up -d postgres && cd backend && PYTHONPATH=. uv run pytest -m postgres -v`。注:现在 RDS 已 live 验证(`make dev` 起 gateway + 9 张表已建),但 testcontainers ephemeral 路径仍未实跑过 |
|
||||
| ~~远程 RDS 大版本对齐 testcontainers 镜像~~ | ✅ done 2026-05-11 | — | RDS = PostgreSQL 17.9(`make doctor` 确认),fixture 已调到 `postgres:17-alpine` |
|
||||
| ~~Push docs branch 到 origin 跑 CI(含新 `backend-postgres-tests` workflow)~~ | ✅ done 2026-05-12 | — | 38 commits pushed(dce5e959..a592319e),SSH-over-443 绕代理;CI 用户确认绿 |
|
||||
| ~~7 项 schema 不可逆 LOCK 决策团队 review~~ | ✅ done 2026-05-12 | — | 全 7 项 ✅ sign-off:id=String(36) / 命名=workspace_id+wid / slug `^[a-z0-9](-?[a-z0-9])*$` 3-32 / memberships 复合 PK / JWT 一次到位 / default_workspace_id / FK CASCADE。详见 [workspace-schema-design §5](../01-redesign/workspace-schema-design.zh-CN.md#5-不可逆决策清单)。PR4 可开工 |
|
||||
| 远程 RDS 密码轮换 | ⏳ | 用户 | 之前在聊天里给过明文密码——建议事后轮换 |
|
||||
|
||||
## 跳过 / 推迟的子任务(agent 当时主动跳的,需用户认可或后续补)
|
||||
|
||||
| 来源 | 跳过项 | 原因 | 建议 |
|
||||
|---|---|---|---|
|
||||
| PR1 T1.10 | 本地实跑 PG smoke 测试 | docker daemon 未起 | 用户跟进表第 1 项 |
|
||||
| PR2 T2.7 | 写 setup_wizard 推荐 PG 的代码 | 已在 PR1 T1.8 完整实现(empty commit `745a33e0` 仅做 task tracking) | 无需跟进 |
|
||||
| PR2 T2.8 | sqlite→pg 数据迁移工具 (`scripts/migrate_sqlite_to_postgres.py`) | plan 标 optional + Stage 0 没生产数据 | 如果出现"dev 用 SQLite 跑过一段、想保留数据迁 PG"的需求再补 |
|
||||
| PR2 T2.9 | `backend/CLAUDE.md` Database 段更新 | README 已覆盖 80% 价值 | 写 PR3 时顺手补一句(agent 自己能做,不阻塞) |
|
||||
| PR4 T4.14 | 真机 `make dev` smoke 注册流程 | agent 无法实际起 gateway daemon | 用户跟进;命令清单见 [pr4-auth-workspace.md "Live smoke 命令"](./pr4-auth-workspace.md#live-smoke-命令用户跟进) |
|
||||
| PR4 follow-up | Regular user pre-PR4 backfill 脚本 | login 路径已 lazy backfill 覆盖;如果生产有大量预存 regular user,可补 batch 脚本 | 等真出现这个场景再写 |
|
||||
| PR4 follow-up | 17 个 pre-existing caplog flake 集中清理 | 跨多个 test 文件的 propagation 问题,与 PR4/5/6 无关 | 单独 follow-up 处理 |
|
||||
| ~~PR5 T5.11~~ | ~~ORM model.py `nullable=False` 翻转~~ | **PR6 已落** (commit `87ea715c`) | — |
|
||||
| PR5 T5.12 真机 PG smoke | `alembic 0002 → backfill → 0003` 端到端 | agent 不能起 RDS 操作 | 用户跟进;命令清单见 [pr5-business-workspace-id.md "Live smoke 命令"](./pr5-business-workspace-id.md#live-smoke-命令用户跟进) |
|
||||
| PR6 T6.15 真机迁移 smoke | `make migrate-paths --dry-run` → 真迁移 → lifespan warning 消失 → 双账户互访 404 | agent 起不了 dev 服务 | 用户跟进;命令清单见 [pr6-routes-paths-workspace.md "Live smoke 命令"](./pr6-routes-paths-workspace.md#live-smoke-命令用户跟进) |
|
||||
|
||||
## 即将遇到的开放问题(plan 末尾列的,下个 session 处理)
|
||||
|
||||
详见 plan [关键开放问题](../../superpowers/plans/2026-05-10-stage-0-multi-tenant-foundation.md#关键开放问题执行-session-第一件事处理):
|
||||
|
||||
1. ~~**PR4 起会真正用到 alembic**——首个 revision 之前要不要加 baseline?~~ **✅ T4.1 (2026-05-12) 已验证**:`versions/` 空 + `alembic heads`/`history` 都空输出 → 0001 直接当首个 revision、`down_revision = None`,**不需要 baseline**。`alembic_version` 表首次 `upgrade head` 时自动建;现有 create_all() 已建好的 schema 不冲突(0001 只 ADD COLUMN)。`doctor.py` 不需要加自动检测
|
||||
2. ~~**`_ensure_admin_user(app)` 现状的孤立 thread 迁移逻辑**~~ **✅ T4 准备阶段 (2026-05-12) 已 grep**:`app.py:52` 当前只做两件事——(a) admin_count==0 时仅日志提示去 `/setup`,(b) admin 已存在时跑 LangGraph store 孤立 thread 迁移。**不自建 admin**。所以 T4.13 真实任务范围 = "admin 已存在但无 workspace"的 idempotent backfill 分支(plan 顶部"风险与缓解"段写的才对,task 措辞"建完 admin 顺带建"是误导,实际归 T4.8)
|
||||
3. **PG 大版本对齐**(同上"用户必须跟进"#2)
|
||||
|
||||
## 下一步建议
|
||||
|
||||
**PR7(CI boundary 静态扫描)**。Stage 0 收尾的最后一项;plan 描述:"静态扫描 ban `deerflow.* → app.*` 反向 import"。PR1-PR6 的代码已经维持这条边界,PR7 是把单测 `tests/test_harness_boundary.py` 上的检查升级为 grep 级 / CI workflow 级扫描,加更细粒度的禁止规则(如禁止 `app.*` 反向再 import 回 `deerflow.runtime.*` 等不应有的间接环)。**PR8 (service_accounts / api_keys / external_users schema) 可并行**,依赖只到 PR3 的 workspaces 表。
|
||||
|
||||
PR6 经验回顾:plan 推荐 Inline 模式是对的,路由 + 仓储 + Paths 强耦合每一步都依赖前一步的接口形态。如果走 subagent 派单会反复阻塞在跨 task 的 signature 协调上。
|
||||
|
||||
历史模式回顾:
|
||||
|
||||
| | Inline(PR1/PR2 模式) | Subagent-Driven |
|
||||
|---|---|---|
|
||||
| 速度 | 主 agent 推全流 | 主 agent 派单到 subagent,等结果 |
|
||||
| 上下文消耗 | 多 | 少(任务上下文不污染主 agent) |
|
||||
| 调试 | 错了主 agent 直接看 | 错了要找 subagent log |
|
||||
| 适用 | PR1/PR2 这种"一个 PR 内有强耦合 reasoning"的 | PR3 这种"10 个机械任务,每个独立"的 |
|
||||
|
||||
## 维护规则
|
||||
|
||||
**完成一个 PR 后**(merge 进 docs branch 那刻)必更新本文件:
|
||||
|
||||
1. 把 PR 的状态行从 🟡 pending 改 ✅ merged
|
||||
2. 填 commits 数 + commit hash 范围 + impl note 链接
|
||||
3. 把 PR 跳过/推迟的子任务移到"跳过 / 推迟的子任务"表
|
||||
4. 把 PR 引入的开放问题加到"即将遇到的开放问题"
|
||||
5. 更新"上次更新"时间 + "一句话状态"
|
||||
|
||||
**进入新 session 第一件事**:读本文件 + 读"即将遇到的开放问题"段 + 验证文件提到的代码锚点是否还在(防 plan 与代码漂移)。
|
||||
|
||||
## 阅读路径
|
||||
|
||||
- **新加入项目想立即了解状态** → 本文件
|
||||
- **写代码前要看 plan** → [Stage 0 master plan](../../superpowers/plans/2026-05-10-stage-0-multi-tenant-foundation.md)
|
||||
- **理解某个具体 PR 怎么落的** → 03-impl/prN-*.md
|
||||
- **需要 Stage 0 之外的全局理解** → [README.zh-CN.md](../README.zh-CN.md)(多租户改造汇总索引)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user