deliverables: add onspace-ai, manus-client, firecrawl-fastapi stacks

deliverables/firecrawl-fastapi/.dockerignore

@@ -0,0 +1,8 @@
+.venv
+__pycache__
+*.pyc
+.env
+.git
+logs
+tests
+.pytest_cache

deliverables/firecrawl-fastapi/.gitignore

@@ -0,0 +1,6 @@
+.venv/
+__pycache__/
+*.pyc
+.env
+logs/*.log
+.pytest_cache/

deliverables/firecrawl-fastapi/Dockerfile

@@ -0,0 +1,31 @@
+# syntax=docker/dockerfile:1.7
+# Multi-stage: build deps → runtime (API worker แยกใน compose)
+FROM python:3.13-slim AS base
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+WORKDIR /app
+
+# ---------- stage: deps ----------
+FROM base AS deps
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# ---------- stage: runtime ----------
+FROM base AS runtime
+COPY --from=deps /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages
+COPY --from=deps /usr/local/bin /usr/local/bin
+
+COPY . .
+
+# non-root + security
+RUN useradd -m -u 1000 appuser
+USER appuser
+
+EXPOSE 8000
+HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
+  CMD python -c "import urllib.request,sys; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3)"
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

deliverables/firecrawl-fastapi/README.md

@@ -0,0 +1,72 @@
+# FireCrawl + FastAPI
+
+โปรเจกต์รวบรวมข้อมูลเว็บระดับโปรดักชัน: **scrape** หน้าเว็บ + **crawl** ลิงก์ลึก ผ่าน REST API
+พร้อมคิวงานเบื้องหลัง (Celery), แคช Redis, rate-limit, webhook แบบ HMAC และ log โครงสร้าง
+
+## สแตก
+
+| ชิ้นส่วน | บทบาท |
+|----------|--------|
+| FastAPI | เว็บเฟรมเวิร์ก async |
+| firecrawl-py **v1.x** | SDK (pin `<2.0.0` — surface `scrape_url`/`crawl_url`) |
+| Redis | แคช + สถานะงาน + rate-limit |
+| Celery | คิวงาน background (งานยาว) |
+| structlog | log JSON |
+| Docker | API + Worker + Redis |
+
+## โครงสร้าง
+
+```
+app/
+├── config/settings.py       # env + คอนฟิก (pydantic-settings)
+├── core/
+│   ├── firecrawl.py         # wrapper SDK + แคช + retry
+│   ├── redis.py             # client fail-open
+│   └── logging.py           # log JSON/console
+├── api/v1/endpoints/firecrawl.py
+├── schemas/firecrawl.py     # Pydantic
+├── services/webhook.py      # HMAC-SHA256
+├── middleware/rate_limit.py
+└── workers/{celery,tasks}.py
+```
+
+## เริ่มใช้งาน
+
+```bash
+# local (dev)
+python -m venv .venv && . .venv/bin/activate
+pip install -r requirements.txt
+cp .env.example .env   # ใส่ FIRECRAWL_API_KEY
+uvicorn main:app --reload
+
+# หรือ Docker
+docker compose up -d --build
+```
+
+## API
+
+- `POST /api/v1/firecrawl/scrape` — ดึงหน้า (markdown/metadata), แคช Redis, `cache: HIT|MISS`
+- `POST /api/v1/firecrawl/crawl` — ส่งงาน → `202 {task_id}`
+- `GET /api/v1/firecrawl/tasks/{id}` — สถานะ/ผลงาน
+- `GET /health` — สุขภาพระบบ
+
+## คุณสมบัติ production
+
+- **Fail-open**: ถ้า Redis/FireCrawl ล่ม → cache miss, rate-limiter ปล่อยผ่าน, task lookup 404 — แอปไม่ crash
+- **Retry อัตโนมัติ** 3 ครั้ง (scrape) / `task_acks_late` + retry สำหรับ Celery
+- **Cache key** จาก URL + options (SHA-256) → Redis TTL ตามประเภท
+- **Webhook** ลงนาม HMAC-SHA256 ทุกครั้ง (`X-Signature`)
+- **Rate limit** ต่อ IP: `/scrape` 120/นาที, `/crawl` 60/นาที
+- **Log JSON** ผ่าน structlog
+
+## Test
+
+```bash
+pip install pytest pytest-asyncio httpx
+pytest tests/ -q   # ไม่ต้องใช้ service จริง (mock + fail-open path)
+```
+
+## หมายเหตุสำคัญ (SDK)
+
+`firecrawl-py` มี 2 major version ที่ API ไม่เข้ากัน — โปรเจกต์ pin `<2.0.0` (v1.x) เพราะใช้
+`scrape_url`/`crawl_url`/`check_crawl_status` อย่า upgrade ข้าม major โดยไม่ตรวจสอบ surface

deliverables/firecrawl-fastapi/app/api/v1/endpoints/firecrawl.py

@@ -0,0 +1,74 @@
+"""API endpoints: /scrape, /crawl, /tasks/{id} — ล้วน async + fail-open"""
+from __future__ import annotations
+
+from typing import Optional
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from app.config.settings import Settings, get_settings
+from app.core.redis import get_cache_client, get_state_client
+from app.core.firecrawl import FireCrawlService
+from app.schemas.firecrawl import (
+    CrawlRequest,
+    ScrapeRequest,
+    ScrapeResponse,
+    TaskResponse,
+    TaskResult,
+)
+from app.workers import tasks
+
+router = APIRouter(prefix="/firecrawl", tags=["firecrawl"])
+
+
+def _service(settings: Settings = Depends(get_settings)) -> FireCrawlService:
+    cache = get_cache_client(settings)
+    return FireCrawlService(settings, cache=cache)
+
+
+@router.post("/scrape", response_model=ScrapeResponse, status_code=200)
+async def scrape(req: ScrapeRequest, svc: FireCrawlService = Depends(_service)):
+    url = str(req.url)
+    options = {
+        "formats": req.formats,
+        "onlyMainContent": req.only_main_content,
+    }
+    if req.wait_for:
+        options["waitFor"] = req.wait_for
+    if req.timeout:
+        options["timeout"] = req.timeout
+
+    result = await svc.scrape(url, options)
+    if not result.get("success") and result.get("error"):
+        # fail-open: คืน 200 พร้อม error (ไม่ crash) ตาม blueprint
+        pass
+    return ScrapeResponse(
+        success=result.get("success", False),
+        url=url,
+        markdown=result.get("markdown"),
+        metadata=result.get("metadata"),
+        cache=result.get("cache", "MISS"),
+        error=result.get("error"),
+    )
+
+
+@router.post("/crawl", response_model=TaskResponse, status_code=202)
+async def crawl(req: CrawlRequest, settings: Settings = Depends(get_settings)):
+    url = str(req.url)
+    options = {
+        "max_pages": req.max_pages,
+        "ignore_sitemap": req.ignore_sitemap,
+    }
+    if req.webhook_url:
+        options["webhook"] = str(req.webhook_url)
+
+    task_id = await tasks.crawl_url(url, options)
+    return TaskResponse(task_id=task_id, status="queued", url=url, max_pages=req.max_pages)
+
+
+@router.get("/tasks/{task_id}", response_model=TaskResult)
+async def task_status(task_id: str, settings: Settings = Depends(get_settings)):
+    cache = get_state_client(settings)
+    result = await cache.get_json(f"fc:task:{task_id}")
+    if result is None:
+        return TaskResult(task_id=task_id, status="not_found", data=None)
+    return TaskResult(task_id=task_id, status=result.get("status", "unknown"), data=result.get("data"), error=result.get("error"))

deliverables/firecrawl-fastapi/app/config/settings.py

@@ -0,0 +1,45 @@
+"""การตั้งค่าแอป — โหลดจาก .env ด้วย pydantic-settings"""
+from functools import lru_cache
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
+
+    app_name: str = "FireCrawl FastAPI"
+    app_env: str = "development"
+    debug: bool = False
+
+    # FireCrawl
+    firecrawl_api_key: str = ""
+    firecrawl_base_url: str = "https://api.firecrawl.dev"
+    firecrawl_timeout: int = 30
+    firecrawl_retries: int = 3
+
+    # Redis
+    redis_url: str = "redis://localhost:6379/0"
+    redis_cache_url: str = "redis://localhost:6379/1"
+
+    # Cache TTL (วินาที)
+    cache_ttl_scrape: int = 600
+    cache_ttl_crawl: int = 1800
+
+    # Rate limit (คำขอ/นาที)
+    rate_limit_scrape: int = 120
+    rate_limit_crawl: int = 60
+
+    # Webhook
+    webhook_secret: str = ""
+
+    # Celery
+    celery_broker_url: str = "redis://localhost:6379/2"
+    celery_result_backend: str = "redis://localhost:6379/3"
+
+    # Logging
+    log_format: str = "json"  # json | console
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

deliverables/firecrawl-fastapi/app/core/firecrawl.py

@@ -0,0 +1,115 @@
+"""ห่อ firecrawl SDK v1.x — แคช Redis + retry อัตโนมัติ + fail-open"""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from typing import Any, Optional
+
+from app.config.settings import Settings
+
+logger = logging.getLogger("firecrawl")
+
+try:
+    from firecrawl import FirecrawlApp
+
+    _HAS_FIRECRAWL = True
+except Exception:  # pragma: no cover
+    _HAS_FIRECRAWL = False
+
+
+def _cache_key(kind: str, payload: dict) -> str:
+    canonical = json.dumps(payload, sort_keys=True, ensure_ascii=False)
+    digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+    return f"fc:{kind}:{digest}"
+
+
+class FireCrawlService:
+    """Service ครอบ SDK — ถ้า SDK/คีย์หาย หรือ FireCrawl ล่ม จะ fail-open (ไม่ crash)"""
+
+    def __init__(self, settings: Settings, cache=None):
+        self._settings = settings
+        self._cache = cache  # RedisClient หรือ None
+        self._client = None
+        if _HAS_FIRECRAWL and settings.firecrawl_api_key:
+            try:
+                self._client = FirecrawlApp(
+                    api_key=settings.firecrawl_api_key,
+                    api_url=settings.firecrawl_base_url,
+                )
+            except Exception as exc:  # pragma: no cover
+                logger.warning("Firecrawl client init failed: %s", exc)
+
+    # ---------- scrape ----------
+    async def scrape(self, url: str, options: Optional[dict] = None) -> dict:
+        options = options or {}
+        payload = {"url": url, **options}
+        cache_key = _cache_key("scrape", payload)
+
+        # 1) แคช
+        if self._cache is not None:
+            cached = await self._cache.get_json(cache_key)
+            if cached is not None:
+                return {**cached, "cache": "HIT"}
+
+        result = await self._scrape_remote(url, options)
+
+        # 2) เก็บแคช
+        if self._cache is not None and result.get("success"):
+            await self._cache.set_json(cache_key, result, self._settings.cache_ttl_scrape)
+
+        return {**result, "cache": "MISS"}
+
+    async def _scrape_remote(self, url: str, options: dict) -> dict:
+        if self._client is None:
+            # fail-open: คืนค่า degraded แทน crash
+            return {"success": False, "error": "FireCrawl ไม่ได้ตั้งค่า (ไม่มี key)", "markdown": None, "metadata": None}
+        last_err: Optional[Exception] = None
+        for attempt in range(1, self._settings.firecrawl_retries + 1):
+            try:
+                res = self._client.scrape_url(url, params=options)
+                if isinstance(res, dict) and res.get("success") is False:
+                    # SDK บางรุ่น return dict
+                    return {"success": False, "error": str(res.get("error", "unknown")), "markdown": None, "metadata": None}
+                return {
+                    "success": True,
+                    "markdown": (res or {}).get("markdown") if isinstance(res, dict) else getattr(res, "markdown", None),
+                    "metadata": (res or {}).get("metadata") if isinstance(res, dict) else getattr(res, "metadata", None),
+                }
+            except Exception as exc:  # noqa: BLE001
+                last_err = exc
+                logger.warning("scrape attempt %s failed: %s", attempt, exc)
+        return {"success": False, "error": str(last_err or "unknown"), "markdown": None, "metadata": None}
+
+    # ---------- crawl (async ผ่าน Celery task) ----------
+    async def crawl_async(self, url: str, options: Optional[dict] = None) -> str:
+        """ส่งงาน crawl เข้าคิว Celery → return task_id (ไม่บล็อก)"""
+        from app.workers.tasks import crawl_url_task  # import ช้า กัน circular
+
+        options = options or {}
+        return crawl_url_task.delay(url, options).id
+
+    # ---------- job status ----------
+    async def crawl_status(self, crawl_id: str) -> dict:
+        if self._client is None:
+            return {"status": "unknown", "error": "FireCrawl ไม่ได้ตั้งค่า", "data": None}
+        try:
+            res = self._client.check_crawl_status(crawl_id)
+            return {
+                "status": (res or {}).get("status", "unknown"),
+                "data": (res or {}).get("data"),
+                "error": (res or {}).get("error"),
+            }
+        except Exception as exc:  # pragma: no cover
+            logger.warning("crawl status failed (fail-open): %s", exc)
+            return {"status": "unknown", "error": str(exc), "data": None}
+
+
+_service: Optional[FireCrawlService] = None
+
+
+def get_firecrawl_service(settings: Settings, cache=None) -> FireCrawlService:
+    global _service
+    if _service is None:
+        _service = FireCrawlService(settings, cache=cache)
+    return _service

deliverables/firecrawl-fastapi/app/core/logging.py

@@ -0,0 +1,41 @@
+"""ตั้งค่า log แบบมีโครงสร้าง — JSON (failsafe เป็น console ถ้าไม่มี structlog)"""
+import logging
+
+from app.config.settings import Settings
+
+# structlog เป็น optional — fallback เป็น stdlib logging
+try:
+    import structlog
+
+    _HAS_STRUCTLOG = True
+except Exception:  # pragma: no cover
+    _HAS_STRUCTLOG = False
+
+
+def setup_logging(settings: Settings) -> None:
+    if _HAS_STRUCTLOG:
+        processors = [
+            structlog.contextvars.merge_contextvars,
+            structlog.processors.add_log_level,
+            structlog.processors.StackInfoRenderer(),
+            structlog.processors.format_exc_info,
+        ]
+        if settings.log_format == "json":
+            processors.append(structlog.processors.JSONRenderer(ensure_ascii=False))
+        else:
+            processors.append(structlog.dev.ConsoleRenderer())
+
+        structlog.configure(
+            processors=processors,
+            wrapper_class=structlog.make_filtering_bound_logger(logging.INFO),
+            logger_factory=structlog.PrintLoggerFactory(),
+            cache_logger_on_first_use=True,
+        )
+    else:  # pragma: no cover
+        logging.basicConfig(level=logging.INFO)
+
+
+def get_logger(name: str = "firecrawl"):
+    if _HAS_STRUCTLOG:
+        return structlog.get_logger(name)
+    return logging.getLogger(name)