diff --git a/.claude/commands/prepare-pr.md b/.claude/commands/prepare-pr.md new file mode 100644 index 0000000000..02e5808acf --- /dev/null +++ b/.claude/commands/prepare-pr.md @@ -0,0 +1,18 @@ +Prepare, commit, and open a pull request for the current branch. Do the following in order: + +1. Run `git status`, `git diff --cached`, `git log main..HEAD --oneline`, and `git diff main..HEAD` to understand what is staged and what the full branch diff looks like. + +2. Read `CHANGES.md` to understand the current upstream divergence state. + +3. Craft a conventional commit message for the staged changes: + - Format: `(): ` (under 72 chars) + - Types: feat, fix, chore, refactor, docs, style, test + - Focus on WHY and WHAT changed logically, not which files were touched + +4. Commit the staged changes with that message. + +5. Write the PR description — short, human-readable prose (2–4 sentences max). Explain what the change does and why, as if telling a teammate. No bullet lists of files. Capture intent and logic, not implementation details. + +6. Update `CHANGES.md` if any newly touched upstream files are not yet logged. Follow the existing table format. + +7. Push the branch and create the PR using `gh pr create` with the prose description. diff --git a/.env b/.env deleted file mode 100644 index 5628e7d2d8..0000000000 --- a/.env +++ /dev/null @@ -1,45 +0,0 @@ -# Domain -# This would be set to the production domain with an env var on deployment -# used by Traefik to transmit traffic and acquire TLS certificates -DOMAIN=localhost -# To test the local Traefik config -# DOMAIN=localhost.tiangolo.com - -# Used by the backend to generate links in emails to the frontend -FRONTEND_HOST=http://localhost:5173 -# In staging and production, set this env var to the frontend host, e.g. -# FRONTEND_HOST=https://dashboard.example.com - -# Environment: local, staging, production -ENVIRONMENT=local - -PROJECT_NAME="Full Stack FastAPI Project" -STACK_NAME=full-stack-fastapi-project - -# Backend -BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173,https://localhost,https://localhost:5173,http://localhost.tiangolo.com" -SECRET_KEY=changethis -FIRST_SUPERUSER=admin@example.com -FIRST_SUPERUSER_PASSWORD=changethis - -# Emails -SMTP_HOST= -SMTP_USER= -SMTP_PASSWORD= -EMAILS_FROM_EMAIL=info@example.com -SMTP_TLS=True -SMTP_SSL=False -SMTP_PORT=587 - -# Postgres -POSTGRES_SERVER=localhost -POSTGRES_PORT=5432 -POSTGRES_DB=app -POSTGRES_USER=postgres -POSTGRES_PASSWORD=changethis - -SENTRY_DSN= - -# Configure these with your own Docker registry images -DOCKER_IMAGE_BACKEND=backend -DOCKER_IMAGE_FRONTEND=frontend diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index 35d089860c..1f6e64b0fa 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -1,11 +1,10 @@ name: Add to Project +# Disabled in this fork: targets the fastapi org project board and needs a +# PROJECTS_TOKEN we don't have. Manual-only so it stays merge-clean with upstream. +# Original triggers: pull_request_target; issues (opened, reopened). on: - pull_request_target: # zizmor: ignore[dangerous-triggers] - issues: - types: - - opened - - reopened + workflow_dispatch: permissions: {} diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index ff441d502c..543f2a8df6 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -1,22 +1,68 @@ name: Deploy to Production on: - release: - types: - - published + push: + branches: + - master -permissions: {} +permissions: + contents: read jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push backend + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + file: backend/Dockerfile + push: true + tags: ${{ secrets.DOCKER_IMAGE_BACKEND }}:latest + cache-from: type=gha,scope=backend + cache-to: type=gha,mode=max,scope=backend + + - name: Build and push frontend + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + file: frontend/Dockerfile + push: true + tags: ${{ secrets.DOCKER_IMAGE_FRONTEND }}:latest + build-args: | + VITE_API_URL=https://api.${{ secrets.DOMAIN_PRODUCTION }} + NODE_ENV=production + cache-from: type=gha,scope=frontend + cache-to: type=gha,mode=max,scope=frontend + deploy: - environment: production - # Do not deploy in the main repository, only in user projects - if: github.repository_owner != 'fastapi' + needs: build runs-on: - self-hosted - production + environment: production + if: github.repository_owner != 'fastapi' env: ENVIRONMENT: production + PROJECT_NAME: Agentique DOMAIN: ${{ secrets.DOMAIN_PRODUCTION }} STACK_NAME: ${{ secrets.STACK_NAME_PRODUCTION }} SECRET_KEY: ${{ secrets.SECRET_KEY }} @@ -26,12 +72,27 @@ jobs: SMTP_USER: ${{ secrets.SMTP_USER }} SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} EMAILS_FROM_EMAIL: ${{ secrets.EMAILS_FROM_EMAIL }} + RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} + RESEND_AUDIENCE_ID: ${{ secrets.RESEND_AUDIENCE_ID }} POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_DB: ${{ secrets.POSTGRES_DB }} + POSTGRES_PORT: ${{ secrets.POSTGRES_PORT }} + POSTGRES_SERVER: ${{ secrets.POSTGRES_SERVER }} + DOCKER_IMAGE_BACKEND: ${{ secrets.DOCKER_IMAGE_BACKEND }} + DOCKER_IMAGE_FRONTEND: ${{ secrets.DOCKER_IMAGE_FRONTEND }} + FRONTEND_HOST: ${{ secrets.FRONTEND_HOST }} + BACKEND_CORS_ORIGINS: ${{ secrets.BACKEND_CORS_ORIGINS }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} + BAML_LOG: ${{ secrets.BAML_LOG }} + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + RESIDENTIAL_PROXY_URL: ${{ secrets.RESIDENTIAL_PROXY_URL }} steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - run: docker compose -f compose.yml --project-name ${{ secrets.STACK_NAME_PRODUCTION }} build + - run: touch .env + - run: docker network create traefik-public 2>/dev/null || true + - run: docker compose -f compose.yml --project-name ${{ secrets.STACK_NAME_PRODUCTION }} pull - run: docker compose -f compose.yml --project-name ${{ secrets.STACK_NAME_PRODUCTION }} up -d diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index b438649861..642344f2b4 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -33,5 +33,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false + - run: touch .env + - run: docker network create traefik-public 2>/dev/null || true - run: docker compose -f compose.yml --project-name ${{ secrets.STACK_NAME_STAGING }} build - run: docker compose -f compose.yml --project-name ${{ secrets.STACK_NAME_STAGING }} up -d diff --git a/.github/workflows/detect-conflicts.yml b/.github/workflows/detect-conflicts.yml index ecd288ba19..0da80207e7 100644 --- a/.github/workflows/detect-conflicts.yml +++ b/.github/workflows/detect-conflicts.yml @@ -1,8 +1,9 @@ name: "Conflict detector" +# Disabled in this fork: auto-labels conflicting PRs, only useful with many +# concurrent contributors. Manual-only so it stays merge-clean with upstream. +# Original triggers: push; pull_request_target (synchronize). on: - push: - pull_request_target: # zizmor: ignore[dangerous-triggers] - types: [synchronize] + workflow_dispatch: permissions: {} diff --git a/.github/workflows/guard-dependencies.yml b/.github/workflows/guard-dependencies.yml index c3f97c3752..ff266dbe03 100644 --- a/.github/workflows/guard-dependencies.yml +++ b/.github/workflows/guard-dependencies.yml @@ -1,11 +1,10 @@ name: Guard Dependencies +# Disabled in this fork: auto-closes dependency PRs from non-org members, only +# relevant with external contributors. Manual-only so it stays merge-clean with upstream. +# Original trigger: pull_request_target on master, paths pyproject.toml / uv.lock. on: - pull_request_target: # zizmor: ignore[dangerous-triggers] -- This workflow only reads context.payload metadata, never checks out PR code - branches: [master] - paths: - - pyproject.toml - - uv.lock + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/issue-manager.yml b/.github/workflows/issue-manager.yml index 586f768cce..0ed9f5266b 100644 --- a/.github/workflows/issue-manager.yml +++ b/.github/workflows/issue-manager.yml @@ -1,17 +1,11 @@ name: Issue Manager +# Disabled in this fork: the job is already gated to `repository_owner == 'fastapi'` +# so it never runs here. Triggers neutered to manual-only to stop scheduled/event +# runs spinning up. Stays merge-clean with upstream. +# Original triggers: schedule (cron 21 17 * * *); issue_comment (created); +# issues (labeled); pull_request_target (labeled); workflow_dispatch. on: - schedule: - - cron: "21 17 * * *" - issue_comment: - types: - - created - issues: - types: - - labeled - pull_request_target: # zizmor: ignore[dangerous-triggers] - types: - - labeled workflow_dispatch: permissions: {} diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 5b7524f25e..aed520b763 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,13 +1,10 @@ name: Labels +# Disabled in this fork: enforces upstream's label taxonomy and fails PRs lacking +# one of its labels. Unneeded friction for a small fork. +# Manual-only so it stays merge-clean with upstream. +# Original trigger: pull_request_target (opened, synchronize, reopened, labeled, unlabeled). on: - pull_request_target: # zizmor: ignore[dangerous-triggers] - types: - - opened - - synchronize - - reopened - # For label-checker - - labeled - - unlabeled + workflow_dispatch: permissions: {} diff --git a/.github/workflows/latest-changes.yml b/.github/workflows/latest-changes.yml index 2304a4c430..ff2b455a4b 100644 --- a/.github/workflows/latest-changes.yml +++ b/.github/workflows/latest-changes.yml @@ -1,11 +1,10 @@ name: Latest Changes +# Disabled in this fork: tiangolo's changelog bot (needs LATEST_CHANGES secret, +# commits to release-notes.md we don't maintain). Manual-only so it stays +# merge-clean with upstream. +# Original trigger also included: pull_request_target on master (closed). on: - pull_request_target: # zizmor: ignore[dangerous-triggers] - branches: - - master - types: - - closed workflow_dispatch: inputs: number: diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index d5aeae0369..ac9ef72de2 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -69,11 +69,30 @@ jobs: # Before upgrading uv version, make sure astral-sh/setup-uv knows its checksum. # See: https://github.com/astral-sh/setup-uv/issues/851#issuecomment-4282017837 version: "0.11.18" + - name: Create .env for CI + run: | + cat > .env << 'EOF' + DOMAIN=localhost + STACK_NAME=agentique-ci + DOCKER_IMAGE_BACKEND=agentique-backend + DOCKER_IMAGE_FRONTEND=agentique-frontend + PROJECT_NAME=Agentique + ENVIRONMENT=development + FRONTEND_HOST=http://localhost:5173 + SECRET_KEY=ci-test-secret-key-not-for-production + FIRST_SUPERUSER=admin@example.com + FIRST_SUPERUSER_PASSWORD=ci-test-password + POSTGRES_SERVER=localhost + POSTGRES_DB=app + POSTGRES_USER=postgres + POSTGRES_PASSWORD=ci-test-password + EOF - run: uv sync working-directory: backend - run: bun ci working-directory: frontend - run: bash scripts/generate-client.sh + - run: docker network create traefik-public 2>/dev/null || true - run: docker compose build - run: docker compose down -v --remove-orphans - name: Run Playwright tests diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index e34d578dd3..5770e53c91 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -59,6 +59,24 @@ jobs: run: uv sync --all-packages - name: Install frontend dependencies run: bun ci + - name: Create .env for CI + run: | + cat > .env << 'EOF' + DOMAIN=localhost + STACK_NAME=agentique-ci + DOCKER_IMAGE_BACKEND=agentique-backend + DOCKER_IMAGE_FRONTEND=agentique-frontend + PROJECT_NAME=Agentique + ENVIRONMENT=development + FRONTEND_HOST=http://localhost:5173 + SECRET_KEY=ci-test-secret-key-not-for-production + FIRST_SUPERUSER=admin@example.com + FIRST_SUPERUSER_PASSWORD=ci-test-password + POSTGRES_SERVER=localhost + POSTGRES_DB=app + POSTGRES_USER=postgres + POSTGRES_PASSWORD=ci-test-password + EOF - name: Run prek - pre-commit id: precommit run: uv run prek run --from-ref origin/${GITHUB_BASE_REF} --to-ref HEAD --show-diff-on-failure diff --git a/.github/workflows/smokeshow.yml b/.github/workflows/smokeshow.yml index ceec1e71a8..e5d9a86589 100644 --- a/.github/workflows/smokeshow.yml +++ b/.github/workflows/smokeshow.yml @@ -1,9 +1,11 @@ name: Smokeshow +# Disabled in this fork: uploads a coverage badge to smokeshow.com (needs +# SMOKESHOW_AUTH_KEY). Coverage is already enforced in Test Backend (--fail-under=90). +# Manual-only so it stays merge-clean with upstream. +# Original trigger: workflow_run after [Test Backend] completed. on: - workflow_run: # zizmor: ignore[dangerous-triggers] - workflows: [Test Backend] - types: [completed] + workflow_dispatch: permissions: {} diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 8add95789e..1a0a550032 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -26,6 +26,24 @@ jobs: # Before upgrading uv version, make sure astral-sh/setup-uv knows its checksum. # See: https://github.com/astral-sh/setup-uv/issues/851#issuecomment-4282017837 version: "0.11.18" + - name: Create .env for CI + run: | + cat > .env << 'EOF' + DOMAIN=localhost + STACK_NAME=agentique-ci + DOCKER_IMAGE_BACKEND=agentique-backend + DOCKER_IMAGE_FRONTEND=agentique-frontend + PROJECT_NAME=Agentique + ENVIRONMENT=development + FRONTEND_HOST=http://localhost:5173 + SECRET_KEY=ci-test-secret-key-not-for-production + FIRST_SUPERUSER=admin@example.com + FIRST_SUPERUSER_PASSWORD=ci-test-password + POSTGRES_SERVER=localhost + POSTGRES_DB=app + POSTGRES_USER=postgres + POSTGRES_PASSWORD=ci-test-password + EOF - run: docker compose down -v --remove-orphans - run: docker compose up -d db mailcatcher - name: Migrate DB diff --git a/.github/workflows/test-docker-compose.yml b/.github/workflows/test-docker-compose.yml index 73078c1e01..eaf553eeb3 100644 --- a/.github/workflows/test-docker-compose.yml +++ b/.github/workflows/test-docker-compose.yml @@ -1,10 +1,10 @@ name: Test Docker Compose +# Disabled in this fork: Playwright already builds and exercises the full stack, +# so this smoke test is redundant. Manual-only so it stays merge-clean with upstream. +# Original triggers: push on master; pull_request. on: - push: - branches: - - master - pull_request: + workflow_dispatch: permissions: {} jobs: diff --git a/.gitignore b/.gitignore index f903ab6066..1761a70b8d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ node_modules/ /playwright-report/ /blob-report/ /playwright/.cache/ +.ignored/ +.DS_Store +.env \ No newline at end of file diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000000..8510667ef6 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,107 @@ +# Changes on top of `fastapi/full-stack-fastapi-template` + +--- + +## 2026-07-07 — newsletter + RSS pipeline sources + +- `backend/pyproject.toml` — `imap-tools` dependency. +- New files: `backend/pipeline/sources/email.py` (IMAP newsletter source, wired into `SOURCES`), `backend/pipeline/sources/rss.py` (generic multi-feed RSS source, not wired in — mirrors original TS, which also had it disabled). + +## 2026-07-05 + +- `frontend/src/main.tsx` — `currentUser` query `onError` clears token and redirects to `/login` on `400/401/403/404`; sets `retry=false` for immediate redirect. +- `frontend/tests/login.spec.ts` — mocks `/users/me` → 404, asserts token clearing + redirect. +- `backend/app/api/routes/login.py` — in dev, logs reset link to console instead of asserting `emails_enabled` (fails locally). + +## 2026-07-04 — merge upstream/master (10 commits) + +Merged `fastapi/full-stack-fastapi-template@a758585` (release-notes + dep bumps only). + +- `uv.lock` — re-locked from ours in a `uv:python3.14-bookworm-slim` container. Minimal bump: `emails` 0.6 → 1.1.2. +- `backend/app/utils.py`, `backend/pyproject.toml` — took upstream `emails` 1.1.2 API change, layered on our Resend early-return. +- Workflows — `actions/checkout` v6.0.3 → v7.0.0. + +## 2026-07-02 + +- `frontend/src/components/Common/Footer.tsx` — hand-rolled `fetch` to `/articles/stats` was missing `/api/v1` prefix, 404ing silently. Fixed. +- `backend/app/core/config.py` — `RESEND_API_KEY: str | None`; `emails_enabled` now `bool(EMAILS_FROM_EMAIL and (RESEND_API_KEY or SMTP_HOST))`. +- `backend/app/utils.py` — `send_email` uses `resend.Emails.send(...)` when `RESEND_API_KEY` set, falls back to SMTP. +- `backend/tests/api/routes/test_login.py` — `test_recovery_password` configures `RESEND_API_KEY`, asserts `resend.Emails.send`; added `test_recovery_password_smtp_fallback`. +- `backend/app/api/routes/likes.py`, `backend/app/api/deps_agentique.py`, `backend/app/alembic/versions/d4e5f6a7b8c9_add_article_like_table.py`, `backend/tests/api/routes/test_likes.py`, `backend/tests/utils/article.py` — new files: article likes (PUT/DELETE/GET auth-required, idempotent composite PK), `CurrentUserOptional` (returns `None` on any auth failure), alembic migration. +- `frontend/src/components/Articles/LikeButton.tsx`, `ArticleRow.tsx` — new: optimistic TanStack Query like toggle with cache patching, article row extracted. +- `frontend/src/routes/_layout/profile.tsx` — new: tabs Liked (default) / My profile / Password / Danger zone. +- `backend/app/models_agentique.py` — `ArticleLike` table; `ArticlePublic` gains `like_count`, `liked_by_me`. +- `backend/app/api/routes/articles.py` — LEFT JOIN `article_like` aggregate for `like_count`; per-user liked-id for `liked_by_me`; `sort=likes-desc`. Counts at query time, not denormalized. +- `frontend/src/components/Articles/ArticlesList.tsx` — renders `ArticleRow`; `data-testid`s added. +- `frontend/src/components/Sidebar/Filters.tsx` — "Popular" (`likes-desc`) sort option. +- `backend/app/api/main.py` — mounts `likes.router`. +- `backend/tests/conftest.py` — `db` teardown deletes `article_like` before users. +- `frontend/src/routes/_layout/settings.tsx` — `beforeLoad` redirect to `/profile`. +- `frontend/src/routes/login.tsx` — `redirect` search param via `Route.useSearch()`. +- `frontend/src/hooks/useAuth.ts` — `loginMutation` accepts optional `redirectTo`. +- `frontend/src/components/Sidebar/User.tsx` — "Log in" when logged out; "Profile" menu item → `/profile`. +- `frontend/src/routeTree.gen.ts`, `frontend/src/client/{schemas,sdk,types}.gen.ts` — regenerated. +- `frontend/tests/user-settings.spec.ts` — `/settings` → `/profile`; removed `test.skip`. +- `frontend/tests/login.spec.ts`, `frontend/tests/sign-up.spec.ts` — removed `test.skip`. +- `frontend/tests/utils/user.ts` — post-login assertion to sidebar user-menu visibility. +- `frontend/tests/likes.spec.ts` — new: fire button, anonymous→login redirect, optimistic toggle, sort, Liked tab; serialized. +- `backend/pyproject.toml` — dropped `login.py`, `users.py` from coverage omit; 92% coverage. +- `backend/tests/api/routes/test_login.py`, `test_users.py` — removed module-level skip; `test_recovery_password` patches `EMAILS_FROM_EMAIL`. +- `backend/app/models_agentique.py`, `backend/app/api/routes/articles.py` — `# type: ignore[import-untyped]` on `pgvector.sqlalchemy`; reordered imports for `ruff check`. +- `backend/app/api/routes/articles.py` — `# ty: ignore[...]` on `Article.score`/`Article.embedding` query lines (known `ty` false positive). +- `backend/app/seed_articles.py` — `session.execute(delete(Article))` → `session.exec(...)` (`ty` deprecation). +- `frontend/src/client/{schemas,sdk,types}.gen.ts` — regenerated; was stale, missing `private` router types. + +## 2026-07-01 + +- `.github/workflows/test-backend.yml`, `playwright.yml`, `pre-commit.yml` — added "Create .env for CI" step (fixed non-secret values). +- `.github/workflows/deploy-production.yml` — moved `packages: write` to `build` job only (`zizmor`). +- `backend/app/api/main.py` — `"local"` → `"development"` for `private` router mount (missed in 2026-06-29 rename). +- `backend/app/api/routes/articles.py` — removed unused `sqlalchemy.or_` import. `# pragma: no cover` on `get_model()`/`_embed()` (monkeypatched in tests). +- `backend/app/seed_articles.py` — 50 deterministic sample articles, every filter dimension, 256-dim embeddings. Idempotent wipe-and-reinsert; refuses production. +- `backend/scripts/prestart.sh` — guarded `python -m app.seed_articles` after `initial_data.py`. +- `backend/pyproject.toml` — `[tool.coverage.report] omit` for upstream modules unused by Agentique; `--fail-under=90` measures only exercised code. +- `backend/tests/api/routes/test_newsletter.py` — `monkeypatch.setenv("RESEND_API_KEY"`/`"RESEND_AUDIENCE_ID")` so `resend.Contacts.create` branch is reached. +- `backend/tests/api/routes/test_articles.py` — `since=` fallback test. +- `frontend/src/components/Newsletter/SubscribeForm.tsx` — `noValidate` on `
` (browser HTML5 validation intercepted submit before React/zod). +- `tests/api/routes/test_login.py`, `test_users.py`, `test_items.py`, `test_private.py`, `tests/crud/test_user.py` — module-level skip (files kept). +- `frontend/tests/{login,sign-up,reset-password,admin,user-settings,items}.spec.ts` — file-level skip. +- New files: `backend/tests/api/routes/test_articles.py`, `test_newsletter.py` (monkeypatches `resend.Contacts.create`), `frontend/tests/newsletter.spec.ts`, `frontend/tests/articles.spec.ts`. + +## 2026-06-30 + +- `compose.yml` — `www-http`/`www-https` Traefik routers + `redirectregex` to 301 `www.${DOMAIN}` → bare domain. +- `compose.yml` — `SHELL=/bin/sh` for pipeline (was inheriting host `zsh`, crashing). +- `compose.yml` — pipeline command `supercronic -no-reap` (PID 1 reaper crash without it). +- `backend/pipeline/crontab` — `python` → `/app/.venv/bin/python`. +- `backend/app/main.py` — `newsletter.router` mounted on `app` under `/api`. +- `backend/pyproject.toml` — `resend` dependency. +- `compose.yml` — `RESEND_API_KEY`/`RESEND_AUDIENCE_ID` for `prestart` and `backend`. + +## 2026-06-29 + +- `backend/app/core/config.py` — `ENVIRONMENT` Literal `"local"` → `"development"`; matching guard updated. + +## 2026-06-28 + +- `frontend/src/routes/_layout.tsx` — `beforeLoad` auth guard commented out. +- `frontend/src/routes/_layout/index.tsx` — Dashboard → `ArticlesList`. +- `.env` — deleted and gitignored. +- `compose.yml` — frontend Traefik rule `dashboard.${DOMAIN}` → `${DOMAIN}` (root domain). Added `PROJECT_NAME` to `prestart`/`backend` env. +- `.github/workflows/deploy-production.yml` — split into build (GitHub runner, pushes to ghcr.io) and deploy (self-hosted runner, pulls/restarts). Added buildx + GHA layer caching. Added missing compose env vars. +- GitHub secrets — `DOCKER_IMAGE_*` → ghcr.io URLs; `BACKEND_CORS_ORIGINS`/`FRONTEND_HOST` → `https://next.agentique.ch`; added `STACK_NAME_PRODUCTION=agentique-next`. +- `.github/workflows/deploy-staging.yml` — reverted to upstream, disabled (`workflow_dispatch` only). +- `.github/workflows/deploy-production.yml` — trigger `release: published` → `push: [master]`; added `touch .env`; added missing compose env vars. +- Neutered upstream CI `on:` triggers to `workflow_dispatch` (manual-only): `add-to-project`, `smokeshow`, `labeler`, `detect-conflicts`, `guard-dependencies`, `issue-manager`, `latest-changes`, `test-docker-compose`. + +## 2026-06-27 — pipeline migration + +- `backend/pyproject.toml` — `baml-py`, `trafilatura`, `feedparser`, `dnspython`, `regex`. +- `backend/Dockerfile` — supercronic install; COPY `baml_client` + `pipeline`. +- `compose.yml` — `pipeline` service (supercronic, daily 04:00). + +## 2026-06-27 + +- `compose.yml` — db image `postgres:18` → `pgvector/pgvector:pg17`. +- `backend/pyproject.toml` — `pgvector`, `model2vec`. +- `backend/app/api/main.py` — `articles` router (items router kept). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..b399db0730 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,30 @@ +# agentique-next — working agreements + +## Upstream fork +This repo is a fork of `fastapi/full-stack-fastapi-template`. Keep that in mind for every change: + +- **Prefer adding new files over modifying upstream ones.** New Agentique code goes in `models_agentique.py`, `routes/articles.py`, etc. — not patched into `models.py`, `crud.py`, or other upstream files. +- **When you must touch an upstream file**, make the smallest possible change and note it in `CHANGES.md` with the conflict risk level. Entries in `CHANGES.md` should be concise: one line per file changed, minimal prose, no narrative or deep explanations — just the fact, file path, and any key technical detail another dev needs to know for a future upstream merge. See existing entries for style. +- **Never delete upstream files** (routes, tests, utils). They may be needed for upstream merges and for the template to stay coherent. +- **Backend especially**: the Python backend is where upstream activity is highest. Treat every file in `backend/app/` that existed in the original template as upstream-owned. New domain logic belongs in new files. +- **Generated frontend files** (`types.gen.ts`, `sdk.gen.ts`, `routeTree.gen.ts`) are safe to patch manually — they get fully regenerated by `generate-client` and `vite dev`, so merge conflicts there are not real conflicts. +- Before starting any non-trivial backend task, check `CHANGES.md` to understand what is already diverged. + +## Client generation +- `types.gen.ts`, `sdk.gen.ts`, `schemas.gen.ts` must be committed — Docker does not run `generate-client`, it builds from committed files. To regenerate after an API schema change: start the local Docker stack, then run `bash ./scripts/generate-client.sh` from repo root. Commit the result. + +## Plans +- When asked for a plan, save it to `plans/.md`. Keep the chat reply to a short summary + file path. + +## Skills +- `/prepare-pr` — generates a PR description (conventional commit format) and updates `CHANGES.md`. + +## Tone — how to write to me +Applies to chat replies, plans, docs, PR descriptions, commit messages, code comments — anywhere you're writing for a human, not just code. + +- Short bullet points over paragraphs. If it can be a bullet, make it a bullet. +- Plain language. Explain like to a smart person who isn't in the weeds of this codebase — no jargon dumped without translation. +- Lead with the point, not the setup. Skip preamble like "let me explain" or "here's what's going on." +- Cut filler words and hedging. Say the thing directly. +- Still be precise — short doesn't mean vague. Name the actual file/thing when it matters, drop it when it doesn't. +- When a plan or explanation has several parts, list them as bullets, not numbered prose paragraphs. diff --git a/README.md b/README.md index b3837f1e65..3c589275e0 100644 --- a/README.md +++ b/README.md @@ -1,233 +1,36 @@ -# Full Stack FastAPI Template +# agentique -Test Docker Compose -Test Backend -Coverage +AI-powered article aggregation and intelligence feed. Fetches articles from configured sources, scores and deduplicates them with an LLM, extracts full content, summarizes and categorizes each piece, and stores vector embeddings for semantic search. -## Technology Stack and Features +## How it works -- ⚡ [**FastAPI**](https://fastapi.tiangolo.com) for the Python backend API. - - 🧰 [SQLModel](https://sqlmodel.tiangolo.com) for the Python SQL database interactions (ORM). - - 🔍 [Pydantic](https://docs.pydantic.dev), used by FastAPI, for the data validation and settings management. - - 💾 [PostgreSQL](https://www.postgresql.org) as the SQL database. -- 🚀 [React](https://react.dev) for the frontend. - - 💃 Using TypeScript, hooks, [Vite](https://vitejs.dev), and other parts of a modern frontend stack. - - 🎨 [Tailwind CSS](https://tailwindcss.com) and [shadcn/ui](https://ui.shadcn.com) for the frontend components. - - 🤖 An automatically generated frontend client. - - 🧪 [Playwright](https://playwright.dev) for End-to-End testing. - - 🦇 Dark mode support. -- 🐋 [Docker Compose](https://www.docker.com) for development and production. -- 🔒 Secure password hashing by default. -- 🔑 JWT (JSON Web Token) authentication. -- 📫 Email based password recovery. -- 📬 [Mailcatcher](https://mailcatcher.me) for local email testing during development. -- ✅ Tests with [Pytest](https://pytest.org). -- 📞 [Traefik](https://traefik.io) as a reverse proxy / load balancer. -- 🚢 Deployment instructions using Docker Compose, including how to set up a frontend Traefik proxy to handle automatic HTTPS certificates. -- 🏭 CI (continuous integration) and CD (continuous deployment) based on GitHub Actions. +A cron-scheduled pipeline fetches articles from configured sources and runs each batch through a sequence of BAML-powered steps: deduplication, LLM scoring, content extraction, summarization, categorization, and vector embedding. Results are served via a FastAPI REST API and a React frontend. -### Dashboard Login +Docker Compose services: -[![Dashboard login screenshot](img/login.png)](https://github.com/fastapi/full-stack-fastapi-template) +- `db` — PostgreSQL with pgvector extension +- `backend` — FastAPI app serving the REST API +- `pipeline` — runs the article pipeline on a cron schedule via supercronic +- `frontend` — React app served via Vite +- `adminer` — lightweight web UI for browsing and querying the database directly +- `prestart` — one-shot container that runs DB migrations before the backend starts -### Dashboard - Admin +## Stack -[![Admin dashboard screenshot](img/dashboard.png)](https://github.com/fastapi/full-stack-fastapi-template) +- [FastAPI](https://fastapi.tiangolo.com) — Python backend API +- [BAML](https://docs.boundaryml.com) — structured LLM function definitions +- [PostgreSQL + pgvector](https://github.com/pgvector/pgvector) — article storage and vector search +- [model2vec](https://github.com/MinishLab/model2vec) — fast static embeddings +- [React](https://react.dev) + [Vite](https://vitejs.dev) + [Tailwind CSS](https://tailwindcss.com) — frontend +- [Docker Compose](https://docs.docker.com/compose/) — local dev and production -### Dashboard - Items +## Docs -[![Items dashboard screenshot](img/dashboard-items.png)](https://github.com/fastapi/full-stack-fastapi-template) +- [Backend](./backend/README.md) +- [Frontend](./frontend/README.md) +- [Deployment](./deployment.md) +- [Development](./development.md) -### Dashboard - Dark Mode +## Upstream -[![Dark mode dashboard screenshot](img/dashboard-dark.png)](https://github.com/fastapi/full-stack-fastapi-template) - -### Interactive API Documentation - -[![API docs](img/docs.png)](https://github.com/fastapi/full-stack-fastapi-template) - -## How To Use It - -You can **just fork or clone** this repository and use it as is. - -✨ It just works. ✨ - -### How to Use a Private Repository - -If you want to have a private repository, GitHub won't allow you to simply fork it as it doesn't allow changing the visibility of forks. - -But you can do the following: - -- Create a new GitHub repo, for example `my-full-stack`. -- Clone this repository manually, set the name with the name of the project you want to use, for example `my-full-stack`: - -```bash -git clone git@github.com:fastapi/full-stack-fastapi-template.git my-full-stack -``` - -- Enter into the new directory: - -```bash -cd my-full-stack -``` - -- Set the new origin to your new repository, copy it from the GitHub interface, for example: - -```bash -git remote set-url origin git@github.com:octocat/my-full-stack.git -``` - -- Add this repo as another "remote" to allow you to get updates later: - -```bash -git remote add upstream git@github.com:fastapi/full-stack-fastapi-template.git -``` - -- Push the code to your new repository: - -```bash -git push -u origin master -``` - -### Update From the Original Template - -After cloning the repository, and after doing changes, you might want to get the latest changes from this original template. - -- Make sure you added the original repository as a remote, you can check it with: - -```bash -git remote -v - -origin git@github.com:octocat/my-full-stack.git (fetch) -origin git@github.com:octocat/my-full-stack.git (push) -upstream git@github.com:fastapi/full-stack-fastapi-template.git (fetch) -upstream git@github.com:fastapi/full-stack-fastapi-template.git (push) -``` - -- Pull the latest changes without merging: - -```bash -git pull --no-commit upstream master -``` - -This will download the latest changes from this template without committing them, that way you can check everything is right before committing. - -- If there are conflicts, solve them in your editor. - -- Once you are done, commit the changes: - -```bash -git merge --continue -``` - -### Configure - -You can then update configs in the `.env` files to customize your configurations. - -Before deploying it, make sure you change at least the values for: - -- `SECRET_KEY` -- `FIRST_SUPERUSER_PASSWORD` -- `POSTGRES_PASSWORD` - -You can (and should) pass these as environment variables from secrets. - -Read the [deployment.md](./deployment.md) docs for more details. - -### Generate Secret Keys - -Some environment variables in the `.env` file have a default value of `changethis`. - -You have to change them with a secret key, to generate secret keys you can run the following command: - -```bash -python -c "import secrets; print(secrets.token_urlsafe(32))" -``` - -Copy the content and use that as password / secret key. And run that again to generate another secure key. - -## How To Use It - Alternative With Copier - -This repository also supports generating a new project using [Copier](https://copier.readthedocs.io). - -It will copy all the files, ask you configuration questions, and update the `.env` files with your answers. - -### Install Copier - -You can install Copier with: - -```bash -pip install copier -``` - -Or better, if you have [`pipx`](https://pipx.pypa.io/), you can run it with: - -```bash -pipx install copier -``` - -**Note**: If you have `pipx`, installing copier is optional, you could run it directly. - -### Generate a Project With Copier - -Decide a name for your new project's directory, you will use it below. For example, `my-awesome-project`. - -Go to the directory that will be the parent of your project, and run the command with your project's name: - -```bash -copier copy https://github.com/fastapi/full-stack-fastapi-template my-awesome-project --trust -``` - -If you have `pipx` and you didn't install `copier`, you can run it directly: - -```bash -pipx run copier copy https://github.com/fastapi/full-stack-fastapi-template my-awesome-project --trust -``` - -**Note** the `--trust` option is necessary to be able to execute a [post-creation script](https://github.com/fastapi/full-stack-fastapi-template/blob/master/.copier/update_dotenv.py) that updates your `.env` files. - -### Input Variables - -Copier will ask you for some data, you might want to have at hand before generating the project. - -But don't worry, you can just update any of that in the `.env` files afterwards. - -The input variables, with their default values (some auto generated) are: - -- `project_name`: (default: `"FastAPI Project"`) The name of the project, shown to API users (in .env). -- `stack_name`: (default: `"fastapi-project"`) The name of the stack used for Docker Compose labels and project name (no spaces, no periods) (in .env). -- `secret_key`: (default: `"changethis"`) The secret key for the project, used for security, stored in .env, you can generate one with the method above. -- `first_superuser`: (default: `"admin@example.com"`) The email of the first superuser (in .env). -- `first_superuser_password`: (default: `"changethis"`) The password of the first superuser (in .env). -- `smtp_host`: (default: "") The SMTP server host to send emails, you can set it later in .env. -- `smtp_user`: (default: "") The SMTP server user to send emails, you can set it later in .env. -- `smtp_password`: (default: "") The SMTP server password to send emails, you can set it later in .env. -- `emails_from_email`: (default: `"info@example.com"`) The email account to send emails from, you can set it later in .env. -- `postgres_password`: (default: `"changethis"`) The password for the PostgreSQL database, stored in .env, you can generate one with the method above. -- `sentry_dsn`: (default: "") The DSN for Sentry, if you are using it, you can set it later in .env. - -## Backend Development - -Backend docs: [backend/README.md](./backend/README.md). - -## Frontend Development - -Frontend docs: [frontend/README.md](./frontend/README.md). - -## Deployment - -Deployment docs: [deployment.md](./deployment.md). - -## Development - -General development docs: [development.md](./development.md). - -This includes using Docker Compose, custom local domains, `.env` configurations, etc. - -## Release Notes - -Check the file [release-notes.md](./release-notes.md). - -## License - -The Full Stack FastAPI Template is licensed under the terms of the MIT license. +Fork of [fastapi/full-stack-fastapi-template](https://github.com/fastapi/full-stack-fastapi-template). See [CHANGES.md](./CHANGES.md) for divergences. diff --git a/backend/.python-version b/backend/.python-version new file mode 100644 index 0000000000..6324d401a0 --- /dev/null +++ b/backend/.python-version @@ -0,0 +1 @@ +3.14 diff --git a/backend/Dockerfile b/backend/Dockerfile index edef2849f4..e317da16a2 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -2,6 +2,11 @@ FROM python:3.14 ENV PYTHONUNBUFFERED=1 +# Install supercronic for container-friendly cron +RUN SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.2.33/supercronic-linux-amd64 \ + && curl -fsSL "$SUPERCRONIC_URL" -o /usr/local/bin/supercronic \ + && chmod +x /usr/local/bin/supercronic + # Install uv # Ref: https://docs.astral.sh/uv/guides/integration/docker/#installing-uv COPY --from=ghcr.io/astral-sh/uv:0.9.26 /uv /uvx /bin/ @@ -32,6 +37,8 @@ COPY ./backend/scripts /app/backend/scripts COPY ./backend/pyproject.toml ./backend/alembic.ini /app/backend/ COPY ./backend/app /app/backend/app +COPY ./backend/baml_client /app/backend/baml_client +COPY ./backend/pipeline /app/backend/pipeline # Sync the project # Ref: https://docs.astral.sh/uv/guides/integration/docker/#intermediate-layers diff --git a/backend/app/alembic/versions/a1b2c3d4e5f6_add_articles_table.py b/backend/app/alembic/versions/a1b2c3d4e5f6_add_articles_table.py new file mode 100644 index 0000000000..4972eb0d6f --- /dev/null +++ b/backend/app/alembic/versions/a1b2c3d4e5f6_add_articles_table.py @@ -0,0 +1,56 @@ +"""Add articles table with pgvector + +Revision ID: a1b2c3d4e5f6 +Revises: fe56fa70289e +Create Date: 2026-06-27 00:00:00.000000 + +""" +import sqlalchemy as sa +from alembic import op + +revision = "a1b2c3d4e5f6" +down_revision = "fe56fa70289e" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("CREATE EXTENSION IF NOT EXISTS vector") + + op.create_table( + "article", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column("title", sa.String(), nullable=False), + sa.Column("source", sa.String(), nullable=False), + sa.Column("source_type", sa.String(), nullable=False), + sa.Column("url", sa.String(), nullable=True), + sa.Column("published_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("score", sa.Integer(), nullable=True), + sa.Column("summary", sa.Text(), nullable=True), + sa.Column("categories", sa.JSON(), nullable=True), + sa.Column("kind", sa.String(), nullable=True), + sa.Column("content", sa.Text(), nullable=True), + sa.Column( + "embedding", + sa.VARCHAR(length=256), # placeholder; vector type applied below + nullable=True, + ), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=True), + ) + + # Replace placeholder column with the real vector type + op.execute("ALTER TABLE article ALTER COLUMN embedding TYPE vector(256) USING NULL") + + op.create_index("ix_article_score", "article", ["score"]) + op.create_index("ix_article_published_at", "article", ["published_at"]) + op.execute( + """ + CREATE INDEX ix_article_embedding_hnsw + ON article + USING hnsw (embedding vector_cosine_ops) + """ + ) + + +def downgrade() -> None: + op.drop_table("article") diff --git a/backend/app/alembic/versions/b7c8d9e0f1a2_add_scored_url_table.py b/backend/app/alembic/versions/b7c8d9e0f1a2_add_scored_url_table.py new file mode 100644 index 0000000000..ddcebbdb21 --- /dev/null +++ b/backend/app/alembic/versions/b7c8d9e0f1a2_add_scored_url_table.py @@ -0,0 +1,28 @@ +"""add scored_url table + +Revision ID: b7c8d9e0f1a2 +Revises: a1b2c3d4e5f6 +Create Date: 2026-06-27 00:00:00.000000 + +""" +from alembic import op + +revision = "b7c8d9e0f1a2" +down_revision = "a1b2c3d4e5f6" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + """ + CREATE TABLE IF NOT EXISTS scored_url ( + url TEXT PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + + +def downgrade(): + op.execute("DROP TABLE IF EXISTS scored_url") diff --git a/backend/app/alembic/versions/c8d9e0f1a2b3_add_newsletter_subscriber_table.py b/backend/app/alembic/versions/c8d9e0f1a2b3_add_newsletter_subscriber_table.py new file mode 100644 index 0000000000..c9afab369e --- /dev/null +++ b/backend/app/alembic/versions/c8d9e0f1a2b3_add_newsletter_subscriber_table.py @@ -0,0 +1,32 @@ +"""add newsletter_subscriber table + +Revision ID: c8d9e0f1a2b3 +Revises: b7c8d9e0f1a2 +Create Date: 2026-06-30 00:00:00.000000 + +""" +from alembic import op + +revision = "c8d9e0f1a2b3" +down_revision = "b7c8d9e0f1a2" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + """ + CREATE TABLE IF NOT EXISTS newsletter_subscriber ( + email TEXT PRIMARY KEY, + categories JSONB NOT NULL DEFAULT '["all"]', + custom_category TEXT NOT NULL DEFAULT '', + utm_source TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + + +def downgrade(): + op.execute("DROP TABLE IF EXISTS newsletter_subscriber") diff --git a/backend/app/alembic/versions/d4e5f6a7b8c9_add_article_like_table.py b/backend/app/alembic/versions/d4e5f6a7b8c9_add_article_like_table.py new file mode 100644 index 0000000000..22e8a580dc --- /dev/null +++ b/backend/app/alembic/versions/d4e5f6a7b8c9_add_article_like_table.py @@ -0,0 +1,30 @@ +"""add article_like table + +Revision ID: d4e5f6a7b8c9 +Revises: c8d9e0f1a2b3 +Create Date: 2026-07-02 00:00:00.000000 + +""" +from alembic import op + +revision = "d4e5f6a7b8c9" +down_revision = "c8d9e0f1a2b3" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + """ + CREATE TABLE IF NOT EXISTS article_like ( + user_id UUID NOT NULL REFERENCES "user" (id), + article_id INTEGER NOT NULL REFERENCES article (id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, article_id) + ) + """ + ) + + +def downgrade(): + op.execute("DROP TABLE IF EXISTS article_like") diff --git a/backend/app/api/deps.py b/backend/app/api/deps.py index 5f28ec692a..4fb1c6574b 100644 --- a/backend/app/api/deps.py +++ b/backend/app/api/deps.py @@ -33,7 +33,7 @@ def get_current_user(session: SessionDep, token: TokenDep) -> User: token, settings.SECRET_KEY, algorithms=[security.ALGORITHM] ) token_data = TokenPayload(**payload) - except InvalidTokenError, ValidationError: + except (InvalidTokenError, ValidationError): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Could not validate credentials", diff --git a/backend/app/api/deps_agentique.py b/backend/app/api/deps_agentique.py new file mode 100644 index 0000000000..0da915dd70 --- /dev/null +++ b/backend/app/api/deps_agentique.py @@ -0,0 +1,34 @@ +from typing import Annotated + +import jwt +from fastapi import Depends, Request +from jwt.exceptions import InvalidTokenError +from pydantic import ValidationError + +from app.api.deps import SessionDep +from app.core import security +from app.core.config import settings +from app.models import TokenPayload, User + + +def get_current_user_optional(request: Request, session: SessionDep) -> User | None: + authorization = request.headers.get("Authorization") + if not authorization or not authorization.lower().startswith("bearer "): + return None + token = authorization.split(" ", 1)[1] + + try: + payload = jwt.decode( + token, settings.SECRET_KEY, algorithms=[security.ALGORITHM] + ) + token_data = TokenPayload(**payload) + except (InvalidTokenError, ValidationError): + return None + + user = session.get(User, token_data.sub) + if not user or not user.is_active: + return None + return user + + +CurrentUserOptional = Annotated[User | None, Depends(get_current_user_optional)] diff --git a/backend/app/api/main.py b/backend/app/api/main.py index eac18c8e8f..790ff7f905 100644 --- a/backend/app/api/main.py +++ b/backend/app/api/main.py @@ -1,6 +1,6 @@ from fastapi import APIRouter -from app.api.routes import items, login, private, users, utils +from app.api.routes import articles, items, likes, login, private, users, utils from app.core.config import settings api_router = APIRouter() @@ -8,7 +8,9 @@ api_router.include_router(users.router) api_router.include_router(utils.router) api_router.include_router(items.router) +api_router.include_router(articles.router) +api_router.include_router(likes.router) -if settings.ENVIRONMENT == "local": +if settings.ENVIRONMENT == "development": api_router.include_router(private.router) diff --git a/backend/app/api/routes/articles.py b/backend/app/api/routes/articles.py new file mode 100644 index 0000000000..001892760e --- /dev/null +++ b/backend/app/api/routes/articles.py @@ -0,0 +1,171 @@ +from datetime import UTC, datetime, timedelta +from typing import Any + +from fastapi import APIRouter, Query +from model2vec import StaticModel +from pgvector.sqlalchemy import Vector # type: ignore[import-untyped] +from sqlalchemy import cast, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlmodel import col, select + +from app.api.deps import SessionDep +from app.api.deps_agentique import CurrentUserOptional +from app.models_agentique import Article, ArticleLike, ArticlePublic, ArticlesPublic + +router = APIRouter(prefix="/articles", tags=["articles"]) + +# Loaded once at module import — model2vec is CPU-only and tiny (~30 MB) +_model: StaticModel | None = None + + +def get_model() -> StaticModel: # pragma: no cover + global _model + if _model is None: + _model = StaticModel.from_pretrained("minishlab/potion-base-8M") + return _model + + +def _embed(text: str) -> list[float]: # pragma: no cover + import numpy as np + + model = get_model() + vec = model.encode([text])[0] + norm = np.linalg.norm(vec) + if norm > 0: + vec = vec / norm + return vec.tolist() + + +def _liked_article_ids(session: SessionDep, user_id: Any) -> set[int]: + return set( + session.exec( + select(ArticleLike.article_id).where(ArticleLike.user_id == user_id) + ).all() + ) + + +def _like_counts_subquery() -> Any: + return ( + select(ArticleLike.article_id, func.count().label("like_count")) + .group_by(ArticleLike.article_id) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + .subquery() + ) + + +@router.get("/", response_model=ArticlesPublic) +def read_articles( + session: SessionDep, + current_user: CurrentUserOptional, + limit: int = Query(default=20, ge=1, le=50), + since: str | None = None, + min_score: int | None = Query(default=None, ge=1, le=10), + category: str | None = None, + kind: str | None = None, + sort: str = Query(default="score-desc"), +) -> Any: + since_dt: datetime + if since: + try: + since_dt = datetime.fromisoformat(since) + except ValueError: + since_dt = datetime.now(UTC) - timedelta(days=30) + else: + since_dt = datetime.now(UTC) - timedelta(days=30) + + conditions = [ + Article.score.is_not(None), # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + col(Article.published_at) >= since_dt, + ] + if min_score is not None: + conditions.append(Article.score >= min_score) # type: ignore[operator] # ty: ignore[unsupported-operator] + if kind is not None: + conditions.append(Article.kind == kind) + if category is not None: + conditions.append( + cast(Article.categories, JSONB).contains([category]) # type: ignore[arg-type] + ) + + count_statement = select(func.count()).select_from(Article).where(*conditions) + count = session.exec(count_statement).one() + + like_counts_subq = _like_counts_subquery() + like_count_expr = func.coalesce(like_counts_subq.c.like_count, 0) + joined_statement = ( + select(Article, like_count_expr.label("like_count")) + .outerjoin(like_counts_subq, like_counts_subq.c.article_id == Article.id) + .where(*conditions) + ) + + if sort == "published_at-desc": + joined_statement = joined_statement.order_by( + col(Article.published_at).desc(), col(Article.id).desc() + ) + elif sort == "likes-desc": + joined_statement = joined_statement.order_by( + like_count_expr.desc(), col(Article.score).desc(), col(Article.id).desc() + ) + else: + joined_statement = joined_statement.order_by( + col(Article.score).desc(), col(Article.id).desc() + ) + joined_statement = joined_statement.limit(limit) + + rows = session.exec(joined_statement).all() + + liked_ids = _liked_article_ids(session, current_user.id) if current_user else set() + + data = [] + for article, like_count in rows: + pub = ArticlePublic.model_validate(article) + pub.like_count = like_count + pub.liked_by_me = article.id in liked_ids + data.append(pub) + + return ArticlesPublic(data=data, count=count) + + +@router.get("/search", response_model=ArticlesPublic) +def search_articles( + session: SessionDep, + current_user: CurrentUserOptional, + q: str, + limit: int = Query(default=20, ge=1, le=50), +) -> Any: + query_vec = _embed(q) + + like_counts_subq = _like_counts_subquery() + like_count_expr = func.coalesce(like_counts_subq.c.like_count, 0) + + statement = ( + select(Article, like_count_expr.label("like_count")) + .outerjoin(like_counts_subq, like_counts_subq.c.article_id == Article.id) + .where(Article.score.is_not(None)) # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + .where(Article.embedding.is_not(None)) # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + .order_by( + cast(Article.embedding, Vector(256)).cosine_distance(query_vec), + col(Article.id).desc(), + ) + .limit(limit) + ) + + rows = session.exec(statement).all() + + liked_ids = _liked_article_ids(session, current_user.id) if current_user else set() + + data = [] + for article, like_count in rows: + pub = ArticlePublic.model_validate(article) + pub.like_count = like_count + pub.liked_by_me = article.id in liked_ids + data.append(pub) + + return ArticlesPublic(data=data, count=len(data)) + + +@router.get("/stats") +def article_stats(session: SessionDep) -> Any: + total = session.exec(select(func.count()).select_from(Article)).one() + last = session.exec( + select(func.max(Article.created_at)) # type: ignore[arg-type] + ).one() + return {"total": total, "lastUpdated": last.isoformat() if last else None} diff --git a/backend/app/api/routes/likes.py b/backend/app/api/routes/likes.py new file mode 100644 index 0000000000..7575c83f66 --- /dev/null +++ b/backend/app/api/routes/likes.py @@ -0,0 +1,66 @@ +from typing import Any + +from fastapi import APIRouter, HTTPException +from sqlalchemy import func +from sqlmodel import col, select + +from app.api.deps import CurrentUser, SessionDep +from app.models_agentique import Article, ArticleLike, ArticlePublic, ArticlesPublic + +router = APIRouter(tags=["likes"]) + + +@router.put("/articles/{article_id}/like") +def like_article( + session: SessionDep, current_user: CurrentUser, article_id: int +) -> Any: + article = session.get(Article, article_id) + if not article: + raise HTTPException(status_code=404, detail="Article not found") + + existing = session.get(ArticleLike, (current_user.id, article_id)) + if not existing: + session.add(ArticleLike(user_id=current_user.id, article_id=article_id)) + session.commit() + + return {"ok": True} + + +@router.delete("/articles/{article_id}/like") +def unlike_article( + session: SessionDep, current_user: CurrentUser, article_id: int +) -> Any: + existing = session.get(ArticleLike, (current_user.id, article_id)) + if existing: + session.delete(existing) + session.commit() + + return {"ok": True} + + +@router.get("/me/liked-articles", response_model=ArticlesPublic) +def read_liked_articles(session: SessionDep, current_user: CurrentUser) -> Any: + like_counts_subq = ( + select(ArticleLike.article_id, func.count().label("like_count")) + .group_by(ArticleLike.article_id) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + .subquery() + ) + like_count_expr = func.coalesce(like_counts_subq.c.like_count, 0) + + statement = ( + select(Article, like_count_expr.label("like_count")) + .join(ArticleLike, col(ArticleLike.article_id) == col(Article.id)) + .outerjoin(like_counts_subq, like_counts_subq.c.article_id == Article.id) + .where(ArticleLike.user_id == current_user.id) + .order_by(col(ArticleLike.created_at).desc(), col(Article.id).desc()) + ) + rows = session.exec(statement).all() + + data = [] + for article, like_count in rows: + pub = ArticlePublic.model_validate(article) + pub.like_count = like_count + pub.liked_by_me = True + data.append(pub) + + return ArticlesPublic(data=data, count=len(data)) diff --git a/backend/app/api/routes/login.py b/backend/app/api/routes/login.py index 58441e37e9..2647350b02 100644 --- a/backend/app/api/routes/login.py +++ b/backend/app/api/routes/login.py @@ -1,3 +1,4 @@ +import logging from datetime import timedelta from typing import Annotated, Any @@ -17,6 +18,8 @@ verify_password_reset_token, ) +logger = logging.getLogger(__name__) + router = APIRouter(tags=["login"]) @@ -64,11 +67,19 @@ def recover_password(email: str, session: SessionDep) -> Message: email_data = generate_reset_password_email( email_to=user.email, email=email, token=password_reset_token ) - send_email( - email_to=user.email, - subject=email_data.subject, - html_content=email_data.html_content, - ) + if settings.ENVIRONMENT == "development": + logger.info( + "[dev] Password reset link for %s: %s/reset-password?token=%s", + email, + settings.FRONTEND_HOST, + password_reset_token, + ) + if settings.emails_enabled: + send_email( + email_to=user.email, + subject=email_data.subject, + html_content=email_data.html_content, + ) return Message( message="If that email is registered, we sent a password recovery link" ) diff --git a/backend/app/api/routes/newsletter.py b/backend/app/api/routes/newsletter.py new file mode 100644 index 0000000000..19c3d1e320 --- /dev/null +++ b/backend/app/api/routes/newsletter.py @@ -0,0 +1,50 @@ +import logging +import os +from typing import Any + +import resend +from fastapi import APIRouter, HTTPException + +from app.api.deps import SessionDep +from app.models_agentique import ( + NewsletterSubscribeRequest, + NewsletterSubscribeResponse, + NewsletterSubscriber, + get_datetime_utc, +) + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/newsletter", tags=["newsletter"]) + + +@router.post("/subscribe", response_model=NewsletterSubscribeResponse) +def subscribe(session: SessionDep, body: NewsletterSubscribeRequest) -> Any: + if "@" not in body.email: + raise HTTPException(status_code=400, detail="Valid email is required") + + categories = body.categories if body.categories else ["all"] + + existing = session.get(NewsletterSubscriber, body.email) + subscriber = NewsletterSubscriber( + email=body.email, + categories=categories, + custom_category=body.customCategory, + utm_source=body.utm_source, + created_at=existing.created_at if existing else get_datetime_utc(), + updated_at=get_datetime_utc(), + ) + session.merge(subscriber) + session.commit() + + api_key = os.getenv("RESEND_API_KEY") + audience_id = os.getenv("RESEND_AUDIENCE_ID") + if api_key and audience_id: + resend.api_key = api_key + try: + resend.Contacts.create({"email": body.email, "audience_id": audience_id}) + except Exception as e: + if "already exists" not in str(e): + logger.error(f"Resend contact create failed for {body.email}: {e}") + + return NewsletterSubscribeResponse(ok=True) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 8710696271..a2f6c975e2 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -34,7 +34,7 @@ class Settings(BaseSettings): # 60 minutes * 24 hours * 8 days = 8 days ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8 FRONTEND_HOST: str = "http://localhost:5173" - ENVIRONMENT: Literal["local", "staging", "production"] = "local" + ENVIRONMENT: Literal["development", "staging", "production"] = "development" BACKEND_CORS_ORIGINS: Annotated[ list[AnyUrl] | str, BeforeValidator(parse_cors) @@ -73,6 +73,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn: SMTP_HOST: str | None = None SMTP_USER: str | None = None SMTP_PASSWORD: str | None = None + RESEND_API_KEY: str | None = None EMAILS_FROM_EMAIL: EmailStr | None = None EMAILS_FROM_NAME: str | None = None @@ -87,7 +88,7 @@ def _set_default_emails_from(self) -> Self: @computed_field # type: ignore[prop-decorator] @property def emails_enabled(self) -> bool: - return bool(self.SMTP_HOST and self.EMAILS_FROM_EMAIL) + return bool(self.EMAILS_FROM_EMAIL and (self.RESEND_API_KEY or self.SMTP_HOST)) EMAIL_TEST_USER: EmailStr = "test@example.com" FIRST_SUPERUSER: EmailStr @@ -99,7 +100,7 @@ def _check_default_secret(self, var_name: str, value: str | None) -> None: f'The value of {var_name} is "changethis", ' "for security, please change it, at least for deployments." ) - if self.ENVIRONMENT == "local": + if self.ENVIRONMENT == "development": warnings.warn(message, stacklevel=1) else: raise ValueError(message) diff --git a/backend/app/main.py b/backend/app/main.py index 9a95801e74..39f4b59e4b 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -4,6 +4,7 @@ from starlette.middleware.cors import CORSMiddleware from app.api.main import api_router +from app.api.routes import newsletter from app.core.config import settings @@ -31,3 +32,4 @@ def custom_generate_unique_id(route: APIRoute) -> str: ) app.include_router(api_router, prefix=settings.API_V1_STR) +app.include_router(newsletter.router, prefix="/api") diff --git a/backend/app/models_agentique.py b/backend/app/models_agentique.py new file mode 100644 index 0000000000..a13d04d1a5 --- /dev/null +++ b/backend/app/models_agentique.py @@ -0,0 +1,85 @@ +import uuid +from datetime import UTC, datetime + +from pgvector.sqlalchemy import Vector # type: ignore[import-untyped] +from sqlalchemy import JSON, Column, DateTime +from sqlmodel import Field, SQLModel + + +def get_datetime_utc() -> datetime: + return datetime.now(UTC) + + +class ArticleBase(SQLModel): + title: str + source: str + source_type: str + url: str | None = None + published_at: datetime | None = None + score: int | None = None + summary: str | None = None + categories: list[str] = Field( + default_factory=list, sa_column=Column(JSON, nullable=True) + ) + kind: str | None = None + + +class Article(ArticleBase, table=True): + id: int | None = Field(default=None, primary_key=True) + content: str | None = Field(default="") + # 256-dim model2vec vectors; nullable until the import script runs + embedding: list[float] | None = Field( + default=None, sa_column=Column(Vector(256), nullable=True) + ) + created_at: datetime | None = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) + + +class ArticlePublic(ArticleBase): + id: int + created_at: datetime | None = None + like_count: int = 0 + liked_by_me: bool = False + + +class ArticlesPublic(SQLModel): + data: list[ArticlePublic] + count: int + + +class ArticleLike(SQLModel, table=True): + __tablename__ = "article_like" + user_id: uuid.UUID = Field(foreign_key="user.id", primary_key=True) + article_id: int = Field(foreign_key="article.id", primary_key=True) + created_at: datetime = Field(default_factory=get_datetime_utc) + + +class ScoredUrl(SQLModel, table=True): + __tablename__ = "scored_url" + url: str = Field(primary_key=True) + created_at: datetime = Field(default_factory=get_datetime_utc) + + +class NewsletterSubscriber(SQLModel, table=True): + __tablename__ = "newsletter_subscriber" + email: str = Field(primary_key=True) + categories: list[str] = Field( + default_factory=lambda: ["all"], sa_column=Column(JSON, nullable=False) + ) + custom_category: str = Field(default="") + utm_source: str | None = Field(default=None) + created_at: datetime = Field(default_factory=get_datetime_utc) + updated_at: datetime = Field(default_factory=get_datetime_utc) + + +class NewsletterSubscribeRequest(SQLModel): + email: str + categories: list[str] = Field(default_factory=lambda: ["all"]) + customCategory: str = "" + utm_source: str | None = None + + +class NewsletterSubscribeResponse(SQLModel): + ok: bool = True diff --git a/backend/app/seed_articles.py b/backend/app/seed_articles.py new file mode 100644 index 0000000000..8b56642e85 --- /dev/null +++ b/backend/app/seed_articles.py @@ -0,0 +1,83 @@ +import math +import random +from datetime import UTC, datetime, timedelta + +from sqlmodel import Session, delete + +from app.core.config import settings +from app.core.db import engine +from app.models_agentique import Article, ArticleLike + +# Fixed seed so local dev, CI, and the Playwright stack all get the same 50 rows. +SEED = 20260701 + +CATEGORIES = ["models", "dev", "research"] +KINDS = ["repo", "paper", "model", "blog", "product", "announcement"] +SOURCE_TYPES = ["aiNews", "rss", "hackerNews"] +SOURCES = ["AI News", "Hacker News", "The Batch", "Import AI", "Latent Space"] + +EMBEDDING_DIM = 256 +ARTICLE_COUNT = 50 + + +def _normalized_embedding(rng: random.Random) -> list[float]: + vec = [rng.gauss(0, 1) for _ in range(EMBEDDING_DIM)] + norm = math.sqrt(sum(v * v for v in vec)) + return [v / norm for v in vec] + + +def _published_at(rng: random.Random, index: int) -> datetime: + now = datetime.now(UTC) + if index < 40: + # bulk: today down to a week ago + return now - timedelta(days=rng.uniform(0, 7)) + elif index < 48: + # a week ago down to just under the 30-day default window + return now - timedelta(days=rng.uniform(7, 30)) + else: + # just past the default `since` window, to exercise it + return now - timedelta(days=rng.uniform(31, 40)) + + +def make_sample_articles() -> list[Article]: + rng = random.Random(SEED) + articles = [] + for i in range(ARTICLE_COUNT): + categories = rng.sample(CATEGORIES, k=rng.choice([1, 2])) + articles.append( + Article( + title=f"Sample article {i + 1}", + source=rng.choice(SOURCES), + source_type=rng.choice(SOURCE_TYPES), + url=f"https://example.com/articles/{i + 1}", + published_at=_published_at(rng, i), + score=rng.randint(1, 10), + summary=f"Summary for sample article {i + 1}.", + categories=categories, + kind=rng.choice(KINDS), + content=f"Content body for sample article {i + 1}.", + embedding=_normalized_embedding(rng), + ) + ) + return articles + + +def seed(session: Session) -> None: + # article_like FK-references article, so clear it before wiping articles. + session.exec(delete(ArticleLike)) + session.exec(delete(Article)) + session.add_all(make_sample_articles()) + session.commit() + + +def main() -> None: + if settings.ENVIRONMENT == "production": + raise RuntimeError( + "Refusing to run seed_articles in production; it wipes the articles table." + ) + with Session(engine) as session: + seed(session) + + +if __name__ == "__main__": + main() diff --git a/backend/app/utils.py b/backend/app/utils.py index 687b73095c..762a1642d2 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -6,6 +6,7 @@ import emails import jwt +import resend from jinja2 import Template from jwt.exceptions import InvalidTokenError @@ -37,6 +38,18 @@ def send_email( html_content: str = "", ) -> None: assert settings.emails_enabled, "no provided configuration for email variables" + if settings.RESEND_API_KEY: + resend.api_key = settings.RESEND_API_KEY + response = resend.Emails.send( + { + "from": f"{settings.EMAILS_FROM_NAME} <{settings.EMAILS_FROM_EMAIL}>", + "to": email_to, + "subject": subject, + "html": html_content, + } + ) + logger.info(f"send email result: {response}") + return assert settings.EMAILS_FROM_EMAIL # For type checker message = emails.message.Message( subject=subject, diff --git a/backend/baml_client/__init__.py b/backend/baml_client/__init__.py new file mode 100644 index 0000000000..b86cfcbbfd --- /dev/null +++ b/backend/baml_client/__init__.py @@ -0,0 +1,60 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +__version__ = "0.223.0" + +try: + from baml_py.safe_import import EnsureBamlPyImport +except ImportError: + raise ImportError(f"""Update to baml-py required. +Version of baml_client generator (see generators.baml): {__version__} + +Please upgrade baml-py to version "{__version__}". + +$ pip install baml-py=={__version__} +$ uv add baml-py=={__version__} + +If nothing else works, please ask for help: + +https://github.com/boundaryml/baml/issues +https://boundaryml.com/discord +""") from None + + +with EnsureBamlPyImport(__version__) as e: + e.raise_if_incompatible_version(__version__) + + from . import types + from . import tracing + from . import stream_types + from . import config + from .config import reset_baml_env_vars + + from .async_client import b + + from . import watchers + + +# FOR LEGACY COMPATIBILITY, expose "partial_types" as an alias for "stream_types" +# WE RECOMMEND USERS TO USE "stream_types" INSTEAD +partial_types = stream_types + +__all__ = [ + "b", + "stream_types", + "partial_types", + "tracing", + "types", + "reset_baml_env_vars", + "config", + "watchers", +] \ No newline at end of file diff --git a/backend/baml_client/async_client.py b/backend/baml_client/async_client.py new file mode 100644 index 0000000000..3da46d80ff --- /dev/null +++ b/backend/baml_client/async_client.py @@ -0,0 +1,563 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +import typing +import typing_extensions +import baml_py + +from . import stream_types, types, type_builder +from .parser import LlmResponseParser, LlmStreamParser +from .runtime import DoNotUseDirectlyCallManager, BamlCallOptions +from .globals import DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME as __runtime__ + + +class BamlAsyncClient: + __options: DoNotUseDirectlyCallManager + __stream_client: "BamlStreamClient" + __http_request: "BamlHttpRequestClient" + __http_stream_request: "BamlHttpStreamRequestClient" + __llm_response_parser: LlmResponseParser + __llm_stream_parser: LlmStreamParser + + def __init__(self, options: DoNotUseDirectlyCallManager): + self.__options = options + self.__stream_client = BamlStreamClient(options) + self.__http_request = BamlHttpRequestClient(options) + self.__http_stream_request = BamlHttpStreamRequestClient(options) + self.__llm_response_parser = LlmResponseParser(options) + self.__llm_stream_parser = LlmStreamParser(options) + + def with_options(self, + tb: typing.Optional[type_builder.TypeBuilder] = None, + client_registry: typing.Optional[baml_py.baml_py.ClientRegistry] = None, + client: typing.Optional[str] = None, + collector: typing.Optional[typing.Union[baml_py.baml_py.Collector, typing.List[baml_py.baml_py.Collector]]] = None, + env: typing.Optional[typing.Dict[str, typing.Optional[str]]] = None, + tags: typing.Optional[typing.Dict[str, str]] = None, + on_tick: typing.Optional[typing.Callable[[str, baml_py.baml_py.FunctionLog], None]] = None, + ) -> "BamlAsyncClient": + options: BamlCallOptions = {} + if tb is not None: + options["tb"] = tb + if client_registry is not None: + options["client_registry"] = client_registry + if client is not None: + options["client"] = client + if collector is not None: + options["collector"] = collector + if env is not None: + options["env"] = env + if tags is not None: + options["tags"] = tags + if on_tick is not None: + options["on_tick"] = on_tick + return BamlAsyncClient(self.__options.merge_options(options)) + + @property + def stream(self): + return self.__stream_client + + @property + def request(self): + return self.__http_request + + @property + def stream_request(self): + return self.__http_stream_request + + @property + def parse(self): + return self.__llm_response_parser + + @property + def parse_stream(self): + return self.__llm_stream_parser + + async def CategorizeOnly(self, title: str, + baml_options: BamlCallOptions = {}, + ) -> types.CategorizeOnlyResult: + # Check if on_tick is provided + if 'on_tick' in baml_options: + # Use streaming internally when on_tick is provided + __stream__ = self.stream.CategorizeOnly(title=title, + baml_options=baml_options) + return await __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = await self.__options.merge_options(baml_options).call_function_async(function_name="CategorizeOnly", args={ + "title": title, + }) + return typing.cast(types.CategorizeOnlyResult, __result__.cast_to(types, types, stream_types, False, __runtime__)) + async def ClassifyKind(self, title: str,url: str,summary: typing.Optional[str] = None, + baml_options: BamlCallOptions = {}, + ) -> types.ClassifyKindResult: + # Check if on_tick is provided + if 'on_tick' in baml_options: + # Use streaming internally when on_tick is provided + __stream__ = self.stream.ClassifyKind(title=title,url=url,summary=summary, + baml_options=baml_options) + return await __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = await self.__options.merge_options(baml_options).call_function_async(function_name="ClassifyKind", args={ + "title": title,"url": url,"summary": summary, + }) + return typing.cast(types.ClassifyKindResult, __result__.cast_to(types, types, stream_types, False, __runtime__)) + async def ClassifyProfiles(self, profiles: typing.List["types.ProfileInput"], + baml_options: BamlCallOptions = {}, + ) -> typing.List["types.ProfileVerdict"]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + # Use streaming internally when on_tick is provided + __stream__ = self.stream.ClassifyProfiles(profiles=profiles, + baml_options=baml_options) + return await __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = await self.__options.merge_options(baml_options).call_function_async(function_name="ClassifyProfiles", args={ + "profiles": profiles, + }) + return typing.cast(typing.List["types.ProfileVerdict"], __result__.cast_to(types, types, stream_types, False, __runtime__)) + async def ExtractLinks(self, emailHtml: str, + baml_options: BamlCallOptions = {}, + ) -> typing.List["types.ExtractedLink"]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + # Use streaming internally when on_tick is provided + __stream__ = self.stream.ExtractLinks(emailHtml=emailHtml, + baml_options=baml_options) + return await __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = await self.__options.merge_options(baml_options).call_function_async(function_name="ExtractLinks", args={ + "emailHtml": emailHtml, + }) + return typing.cast(typing.List["types.ExtractedLink"], __result__.cast_to(types, types, stream_types, False, __runtime__)) + async def ExtractProducts(self, newsletterText: str, + baml_options: BamlCallOptions = {}, + ) -> typing.List["types.NewsletterProduct"]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + # Use streaming internally when on_tick is provided + __stream__ = self.stream.ExtractProducts(newsletterText=newsletterText, + baml_options=baml_options) + return await __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = await self.__options.merge_options(baml_options).call_function_async(function_name="ExtractProducts", args={ + "newsletterText": newsletterText, + }) + return typing.cast(typing.List["types.NewsletterProduct"], __result__.cast_to(types, types, stream_types, False, __runtime__)) + async def ImproveTitles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> typing.List["types.TitleFix"]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + # Use streaming internally when on_tick is provided + __stream__ = self.stream.ImproveTitles(articles=articles, + baml_options=baml_options) + return await __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = await self.__options.merge_options(baml_options).call_function_async(function_name="ImproveTitles", args={ + "articles": articles, + }) + return typing.cast(typing.List["types.TitleFix"], __result__.cast_to(types, types, stream_types, False, __runtime__)) + async def ScoreArticles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> typing.List["types.ScoredArticle"]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + # Use streaming internally when on_tick is provided + __stream__ = self.stream.ScoreArticles(articles=articles, + baml_options=baml_options) + return await __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = await self.__options.merge_options(baml_options).call_function_async(function_name="ScoreArticles", args={ + "articles": articles, + }) + return typing.cast(typing.List["types.ScoredArticle"], __result__.cast_to(types, types, stream_types, False, __runtime__)) + async def SelectNotableProducts(self, products: typing.List["types.NewsletterProduct"], + baml_options: BamlCallOptions = {}, + ) -> typing.List[int]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + # Use streaming internally when on_tick is provided + __stream__ = self.stream.SelectNotableProducts(products=products, + baml_options=baml_options) + return await __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = await self.__options.merge_options(baml_options).call_function_async(function_name="SelectNotableProducts", args={ + "products": products, + }) + return typing.cast(typing.List[int], __result__.cast_to(types, types, stream_types, False, __runtime__)) + async def SelectProductLink(self, productName: str,productDescription: str,candidates: typing.List["types.SearchCandidate"], + baml_options: BamlCallOptions = {}, + ) -> types.ProductLinkChoice: + # Check if on_tick is provided + if 'on_tick' in baml_options: + # Use streaming internally when on_tick is provided + __stream__ = self.stream.SelectProductLink(productName=productName,productDescription=productDescription,candidates=candidates, + baml_options=baml_options) + return await __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = await self.__options.merge_options(baml_options).call_function_async(function_name="SelectProductLink", args={ + "productName": productName,"productDescription": productDescription,"candidates": candidates, + }) + return typing.cast(types.ProductLinkChoice, __result__.cast_to(types, types, stream_types, False, __runtime__)) + async def SemanticDedup(self, newArticles: typing.List["types.ArticleInput"],existingArticles: typing.List["types.ExistingArticle"], + baml_options: BamlCallOptions = {}, + ) -> typing.List["types.DedupMatch"]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + # Use streaming internally when on_tick is provided + __stream__ = self.stream.SemanticDedup(newArticles=newArticles,existingArticles=existingArticles, + baml_options=baml_options) + return await __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = await self.__options.merge_options(baml_options).call_function_async(function_name="SemanticDedup", args={ + "newArticles": newArticles,"existingArticles": existingArticles, + }) + return typing.cast(typing.List["types.DedupMatch"], __result__.cast_to(types, types, stream_types, False, __runtime__)) + async def SummarizeAndCategorize(self, title: str,content: str, + baml_options: BamlCallOptions = {}, + ) -> types.SummarizeAndCategorizeResult: + # Check if on_tick is provided + if 'on_tick' in baml_options: + # Use streaming internally when on_tick is provided + __stream__ = self.stream.SummarizeAndCategorize(title=title,content=content, + baml_options=baml_options) + return await __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = await self.__options.merge_options(baml_options).call_function_async(function_name="SummarizeAndCategorize", args={ + "title": title,"content": content, + }) + return typing.cast(types.SummarizeAndCategorizeResult, __result__.cast_to(types, types, stream_types, False, __runtime__)) + + + +class BamlStreamClient: + __options: DoNotUseDirectlyCallManager + + def __init__(self, options: DoNotUseDirectlyCallManager): + self.__options = options + + def CategorizeOnly(self, title: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlStream[stream_types.CategorizeOnlyResult, types.CategorizeOnlyResult]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_async_stream(function_name="CategorizeOnly", args={ + "title": title, + }) + return baml_py.BamlStream[stream_types.CategorizeOnlyResult, types.CategorizeOnlyResult]( + __result__, + lambda x: typing.cast(stream_types.CategorizeOnlyResult, x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(types.CategorizeOnlyResult, x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def ClassifyKind(self, title: str,url: str,summary: typing.Optional[str] = None, + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlStream[stream_types.ClassifyKindResult, types.ClassifyKindResult]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_async_stream(function_name="ClassifyKind", args={ + "title": title,"url": url,"summary": summary, + }) + return baml_py.BamlStream[stream_types.ClassifyKindResult, types.ClassifyKindResult]( + __result__, + lambda x: typing.cast(stream_types.ClassifyKindResult, x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(types.ClassifyKindResult, x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def ClassifyProfiles(self, profiles: typing.List["types.ProfileInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlStream[typing.List["stream_types.ProfileVerdict"], typing.List["types.ProfileVerdict"]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_async_stream(function_name="ClassifyProfiles", args={ + "profiles": profiles, + }) + return baml_py.BamlStream[typing.List["stream_types.ProfileVerdict"], typing.List["types.ProfileVerdict"]]( + __result__, + lambda x: typing.cast(typing.List["stream_types.ProfileVerdict"], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List["types.ProfileVerdict"], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def ExtractLinks(self, emailHtml: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlStream[typing.List["stream_types.ExtractedLink"], typing.List["types.ExtractedLink"]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_async_stream(function_name="ExtractLinks", args={ + "emailHtml": emailHtml, + }) + return baml_py.BamlStream[typing.List["stream_types.ExtractedLink"], typing.List["types.ExtractedLink"]]( + __result__, + lambda x: typing.cast(typing.List["stream_types.ExtractedLink"], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List["types.ExtractedLink"], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def ExtractProducts(self, newsletterText: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlStream[typing.List["stream_types.NewsletterProduct"], typing.List["types.NewsletterProduct"]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_async_stream(function_name="ExtractProducts", args={ + "newsletterText": newsletterText, + }) + return baml_py.BamlStream[typing.List["stream_types.NewsletterProduct"], typing.List["types.NewsletterProduct"]]( + __result__, + lambda x: typing.cast(typing.List["stream_types.NewsletterProduct"], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List["types.NewsletterProduct"], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def ImproveTitles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlStream[typing.List["stream_types.TitleFix"], typing.List["types.TitleFix"]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_async_stream(function_name="ImproveTitles", args={ + "articles": articles, + }) + return baml_py.BamlStream[typing.List["stream_types.TitleFix"], typing.List["types.TitleFix"]]( + __result__, + lambda x: typing.cast(typing.List["stream_types.TitleFix"], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List["types.TitleFix"], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def ScoreArticles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlStream[typing.List["stream_types.ScoredArticle"], typing.List["types.ScoredArticle"]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_async_stream(function_name="ScoreArticles", args={ + "articles": articles, + }) + return baml_py.BamlStream[typing.List["stream_types.ScoredArticle"], typing.List["types.ScoredArticle"]]( + __result__, + lambda x: typing.cast(typing.List["stream_types.ScoredArticle"], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List["types.ScoredArticle"], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def SelectNotableProducts(self, products: typing.List["types.NewsletterProduct"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlStream[typing.List[int], typing.List[int]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_async_stream(function_name="SelectNotableProducts", args={ + "products": products, + }) + return baml_py.BamlStream[typing.List[int], typing.List[int]]( + __result__, + lambda x: typing.cast(typing.List[int], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List[int], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def SelectProductLink(self, productName: str,productDescription: str,candidates: typing.List["types.SearchCandidate"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlStream[stream_types.ProductLinkChoice, types.ProductLinkChoice]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_async_stream(function_name="SelectProductLink", args={ + "productName": productName,"productDescription": productDescription,"candidates": candidates, + }) + return baml_py.BamlStream[stream_types.ProductLinkChoice, types.ProductLinkChoice]( + __result__, + lambda x: typing.cast(stream_types.ProductLinkChoice, x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(types.ProductLinkChoice, x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def SemanticDedup(self, newArticles: typing.List["types.ArticleInput"],existingArticles: typing.List["types.ExistingArticle"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlStream[typing.List["stream_types.DedupMatch"], typing.List["types.DedupMatch"]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_async_stream(function_name="SemanticDedup", args={ + "newArticles": newArticles,"existingArticles": existingArticles, + }) + return baml_py.BamlStream[typing.List["stream_types.DedupMatch"], typing.List["types.DedupMatch"]]( + __result__, + lambda x: typing.cast(typing.List["stream_types.DedupMatch"], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List["types.DedupMatch"], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def SummarizeAndCategorize(self, title: str,content: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlStream[stream_types.SummarizeAndCategorizeResult, types.SummarizeAndCategorizeResult]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_async_stream(function_name="SummarizeAndCategorize", args={ + "title": title,"content": content, + }) + return baml_py.BamlStream[stream_types.SummarizeAndCategorizeResult, types.SummarizeAndCategorizeResult]( + __result__, + lambda x: typing.cast(stream_types.SummarizeAndCategorizeResult, x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(types.SummarizeAndCategorizeResult, x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + + +class BamlHttpRequestClient: + __options: DoNotUseDirectlyCallManager + + def __init__(self, options: DoNotUseDirectlyCallManager): + self.__options = options + + async def CategorizeOnly(self, title: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="CategorizeOnly", args={ + "title": title, + }, mode="request") + return __result__ + async def ClassifyKind(self, title: str,url: str,summary: typing.Optional[str] = None, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="ClassifyKind", args={ + "title": title,"url": url,"summary": summary, + }, mode="request") + return __result__ + async def ClassifyProfiles(self, profiles: typing.List["types.ProfileInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="ClassifyProfiles", args={ + "profiles": profiles, + }, mode="request") + return __result__ + async def ExtractLinks(self, emailHtml: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="ExtractLinks", args={ + "emailHtml": emailHtml, + }, mode="request") + return __result__ + async def ExtractProducts(self, newsletterText: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="ExtractProducts", args={ + "newsletterText": newsletterText, + }, mode="request") + return __result__ + async def ImproveTitles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="ImproveTitles", args={ + "articles": articles, + }, mode="request") + return __result__ + async def ScoreArticles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="ScoreArticles", args={ + "articles": articles, + }, mode="request") + return __result__ + async def SelectNotableProducts(self, products: typing.List["types.NewsletterProduct"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="SelectNotableProducts", args={ + "products": products, + }, mode="request") + return __result__ + async def SelectProductLink(self, productName: str,productDescription: str,candidates: typing.List["types.SearchCandidate"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="SelectProductLink", args={ + "productName": productName,"productDescription": productDescription,"candidates": candidates, + }, mode="request") + return __result__ + async def SemanticDedup(self, newArticles: typing.List["types.ArticleInput"],existingArticles: typing.List["types.ExistingArticle"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="SemanticDedup", args={ + "newArticles": newArticles,"existingArticles": existingArticles, + }, mode="request") + return __result__ + async def SummarizeAndCategorize(self, title: str,content: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="SummarizeAndCategorize", args={ + "title": title,"content": content, + }, mode="request") + return __result__ + + +class BamlHttpStreamRequestClient: + __options: DoNotUseDirectlyCallManager + + def __init__(self, options: DoNotUseDirectlyCallManager): + self.__options = options + + async def CategorizeOnly(self, title: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="CategorizeOnly", args={ + "title": title, + }, mode="stream") + return __result__ + async def ClassifyKind(self, title: str,url: str,summary: typing.Optional[str] = None, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="ClassifyKind", args={ + "title": title,"url": url,"summary": summary, + }, mode="stream") + return __result__ + async def ClassifyProfiles(self, profiles: typing.List["types.ProfileInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="ClassifyProfiles", args={ + "profiles": profiles, + }, mode="stream") + return __result__ + async def ExtractLinks(self, emailHtml: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="ExtractLinks", args={ + "emailHtml": emailHtml, + }, mode="stream") + return __result__ + async def ExtractProducts(self, newsletterText: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="ExtractProducts", args={ + "newsletterText": newsletterText, + }, mode="stream") + return __result__ + async def ImproveTitles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="ImproveTitles", args={ + "articles": articles, + }, mode="stream") + return __result__ + async def ScoreArticles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="ScoreArticles", args={ + "articles": articles, + }, mode="stream") + return __result__ + async def SelectNotableProducts(self, products: typing.List["types.NewsletterProduct"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="SelectNotableProducts", args={ + "products": products, + }, mode="stream") + return __result__ + async def SelectProductLink(self, productName: str,productDescription: str,candidates: typing.List["types.SearchCandidate"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="SelectProductLink", args={ + "productName": productName,"productDescription": productDescription,"candidates": candidates, + }, mode="stream") + return __result__ + async def SemanticDedup(self, newArticles: typing.List["types.ArticleInput"],existingArticles: typing.List["types.ExistingArticle"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="SemanticDedup", args={ + "newArticles": newArticles,"existingArticles": existingArticles, + }, mode="stream") + return __result__ + async def SummarizeAndCategorize(self, title: str,content: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="SummarizeAndCategorize", args={ + "title": title,"content": content, + }, mode="stream") + return __result__ + + +b = BamlAsyncClient(DoNotUseDirectlyCallManager({})) \ No newline at end of file diff --git a/backend/baml_client/config.py b/backend/baml_client/config.py new file mode 100644 index 0000000000..64b7fff31b --- /dev/null +++ b/backend/baml_client/config.py @@ -0,0 +1,102 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +from __future__ import annotations + +import os +import warnings +import typing_extensions +import typing +import functools + +from baml_py.logging import ( + get_log_level as baml_get_log_level, + set_log_level as baml_set_log_level, +) +from .globals import reset_baml_env_vars + +rT = typing_extensions.TypeVar("rT") # return type +pT = typing_extensions.ParamSpec("pT") # parameters type + + +def _deprecated(message: str): + def decorator(func: typing.Callable[pT, rT]) -> typing.Callable[pT, rT]: + """Use this decorator to mark functions as deprecated. + Every time the decorated function runs, it will emit + a "deprecation" warning.""" + + @functools.wraps(func) + def new_func(*args: pT.args, **kwargs: pT.kwargs): + warnings.simplefilter("always", DeprecationWarning) # turn off filter + warnings.warn( + "Call to a deprecated function {}.".format(func.__name__) + message, + category=DeprecationWarning, + stacklevel=2, + ) + warnings.simplefilter("default", DeprecationWarning) # reset filter + return func(*args, **kwargs) + + return new_func + + return decorator + + +@_deprecated("Use os.environ['BAML_LOG'] instead") +def get_log_level(): + """ + Get the log level for the BAML Python client. + """ + return baml_get_log_level() + + +@_deprecated("Use os.environ['BAML_LOG'] instead") +def set_log_level( + level: typing_extensions.Literal["DEBUG", "INFO", "WARN", "ERROR", "OFF"] | str, +): + """ + Set the log level for the BAML Python client + """ + baml_set_log_level(level) + os.environ["BAML_LOG"] = level + + +@_deprecated("Use os.environ['BAML_LOG_JSON_MODE'] instead") +def set_log_json_mode(): + """ + Set the log JSON mode for the BAML Python client. + """ + os.environ["BAML_LOG_JSON_MODE"] = "true" + + +@_deprecated("Use os.environ['BAML_LOG_MAX_CHUNK_LENGTH'] instead") +def set_log_max_chunk_length(): + """ + Set the maximum log chunk length for the BAML Python client. + """ + os.environ["BAML_LOG_MAX_CHUNK_LENGTH"] = "1000" + + +def set_log_max_message_length(*args, **kwargs): + """ + Alias for set_log_max_chunk_length for compatibility with docs. + """ + return set_log_max_chunk_length(*args, **kwargs) + + +__all__ = [ + "set_log_level", + "get_log_level", + "set_log_json_mode", + "reset_baml_env_vars", + "set_log_max_message_length", + "set_log_max_chunk_length", +] diff --git a/backend/baml_client/globals.py b/backend/baml_client/globals.py new file mode 100644 index 0000000000..769e055bb3 --- /dev/null +++ b/backend/baml_client/globals.py @@ -0,0 +1,35 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +from __future__ import annotations +import os +import warnings + +from baml_py import BamlCtxManager, BamlRuntime +from .inlinedbaml import get_baml_files +from typing import Dict + +DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME = BamlRuntime.from_files( + "baml_src", + get_baml_files(), + os.environ.copy() +) +DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX = BamlCtxManager(DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME) + +def reset_baml_env_vars(env_vars: Dict[str, str]): + warnings.warn( + "reset_baml_env_vars is deprecated and should be removed. Environment variables are now lazily loaded on each function call", + DeprecationWarning, + stacklevel=2 + ) + +__all__ = [] diff --git a/backend/baml_client/inlinedbaml.py b/backend/baml_client/inlinedbaml.py new file mode 100644 index 0000000000..85fd7651d0 --- /dev/null +++ b/backend/baml_client/inlinedbaml.py @@ -0,0 +1,28 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +_file_map = { + + "clients.baml": "client Nvidia {\n provider openai-generic\n options {\n base_url \"https://integrate.api.nvidia.com/v1\"\n api_key env.NVIDIA_NIM_API_KEY\n model \"qwen/qwen3-next-80b-a3b-instruct\"\n timeout 60\n }\n}\n\nclient NvidiaGptOss {\n provider openai-generic\n options {\n base_url \"https://integrate.api.nvidia.com/v1\"\n api_key env.NVIDIA_NIM_API_KEY\n model \"openai/gpt-oss-120b\"\n timeout 60\n }\n}\n", + "dedup.baml": "class DedupMatch {\n url string @description(\"URL of the new article\")\n existingUrl string @description(\"URL of the existing article it duplicates\")\n}\n\nfunction SemanticDedup(\n newArticles: ArticleInput[],\n existingArticles: ExistingArticle[]\n) -> DedupMatch[] {\n client NvidiaGptOss\n prompt #\"\n {{ _.role(\"system\") }}\n You are a semantic similarity engine. You compare article pairs and determine whether they cover the same underlying story. You focus on substance, not surface-level keyword overlap.\n\n {{ _.role(\"user\") }}\n You are given two lists:\n 1. NEW articles that were just fetched\n 2. EXISTING articles already in our database from the last 14 days\n\n Your job: identify which NEW articles are about the same story as an EXISTING article. Two articles are \"the same story\" if they cover the same announcement, release, event, or topic - even if from different sources or with different angles.\n\n NEW articles:\n {% for a in newArticles %}\n {{ loop.index }}. [{{ a.source }}] \"{{ a.title }}\" - {{ a.url }}{% if a.trust %} [trust: {{ a.trust }}]{% endif %}\n {% if a.snippet %} {{ a.snippet }}{% endif %}\n {% endfor %}\n\n EXISTING articles:\n {% for a in existingArticles %}\n {{ loop.index }}. [{{ a.source }}] \"{{ a.title }}\" - {{ a.url }}\n {% endfor %}\n\n For each match, return the new article's URL and the existing article's URL.\n If there are no duplicates, return an empty list.\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest dedup_no_matches {\n functions [SemanticDedup]\n args {\n newArticles [\n {\n url \"https://a.com/1\"\n title \"Claude 4 released\"\n source \"HN\"\n }\n ]\n existingArticles [\n {\n url \"https://b.com/1\"\n title \"The state of Rust in 2026\"\n source \"RSS\"\n }\n ]\n }\n}\n", + "discover.baml": "// Batched relevance classifier for the Substack discovery crawler.\n// One LLM call judges many candidate profiles at once (see ScoreArticles for\n// the array-in/array-out batching pattern). Classification runs on cheap\n// metadata (name + description) BEFORE any per-profile post fetch, and gates\n// both DB insertion and whether the crawler recurses into that profile.\n\nclass ProfileInput {\n handle string @description(\"Substack owner handle, echoed back in the verdict\")\n name string @description(\"Publication name\")\n description string? @description(\"Publication hero text and/or author bio, if known\")\n}\n\nclass ProfileVerdict {\n handle string @description(\"Must exactly match the input handle\")\n isAiRelated bool\n confidence int @description(\"0-100 confidence that AI/ML/LLMs is the PRIMARY focus\")\n}\n\nfunction ClassifyProfiles(profiles: ProfileInput[]) -> ProfileVerdict[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"system\") }}\n You curate sources for a developer-focused AI news platform. For each Substack\n publication, decide whether artificial intelligence is its PRIMARY subject:\n AI/ML/LLMs, foundation models, AI engineering, AI products, or AI research.\n\n Mark isAiRelated = true ONLY when AI is clearly the main topic. Mark false for\n general tech, productivity, marketing, business, crypto, politics, lifestyle, or\n publications that merely mention AI in passing. When the signal is thin (sparse\n description, generic name), lean false and assign low confidence - the crawler\n prunes everything you reject, so false positives pollute the whole subtree.\n\n {{ _.role(\"user\") }}\n Classify every profile. Echo each handle back EXACTLY as given.\n\n {% for p in profiles %}\n {{ loop.index }}. handle={{ p.handle }} | {{ p.name }}{% if p.description %} — {{ p.description }}{% endif %}\n {% endfor %}\n\n {{ ctx.output_format }}\n \"#\n}\n", + "extract_links.baml": "class ExtractedLink {\n title string\n url string\n snippet string @description(\"One-sentence description of the article\")\n}\n\nfunction ExtractLinks(emailHtml: string) -> ExtractedLink[] {\n client NvidiaGptOss\n prompt #\"\n {{ _.role(\"system\") }}\n You are a precise link extraction engine. You parse newsletter HTML and identify external article links while filtering out tracking, admin, and promotional URLs.\n\n {{ _.role(\"user\") }}\n You are given the HTML body of a newsletter email. Extract all article links that point to external news stories, blog posts, research papers, or product announcements.\n\n For each article, return its title, URL, and a one-sentence snippet describing it.\n\n IGNORE these types of links:\n - Tracking/redirect URLs (e.g. links through email marketing platforms)\n - Unsubscribe, manage preferences, or email settings links\n - Social media profile links (twitter, linkedin, etc.)\n - Links back to the newsletter's own homepage, archive index, or subscription page (but DO include article/post links hosted on the newsletter's domain - curated newsletters often host their own content)\n - Sponsor/advertisement links\n - Job posting links\n\n RESOLVE redirect URLs: if a link is clearly a tracking redirect (contains /click, /track, utm_ params, etc.), try to extract the final destination URL from the href. Strip all UTM parameters from URLs.\n\n Newsletter HTML:\n {{ emailHtml }}\n\n If no relevant article links are found, return an empty list.\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest extract_links_basic {\n functions [ExtractLinks]\n args {\n emailHtml \"Welcome to today's issue! [Read OpenAI's new paper](https://openai.com/papers/gpt-5). [Unsubscribe](https://newsletter.com/unsubscribe).\"\n }\n}\n", + "extract_products.baml": "// Product-centric newsletter extraction.\n//\n// Replaces the per-newsletter tracking-URL resolvers: instead of decoding each\n// provider's redirect scheme to recover the real link, we extract the *named\n// products* a newsletter reports on (from anchor text + surrounding prose),\n// keep only the notable ones, and rediscover their canonical URLs via web\n// search (src/sources/email.ts). SelectProductLink then picks the best\n// first-party result among the candidates.\n\nclass NewsletterProduct {\n name string @description(\"Exact product / tool / model / launch name\")\n description string @description(\"Concise phrase: what it is and does, usable verbatim as a web search query\")\n}\n\nfunction ExtractProducts(newsletterText: string) -> NewsletterProduct[] {\n client NvidiaGptOss\n prompt #\"\n {{ _.role(\"system\") }}\n You extract the AI/tech products, tools, models, and launches that a newsletter is reporting on. Work only from the text; never invent products that are not mentioned.\n\n {{ _.role(\"user\") }}\n Below is the text of a newsletter email. Links appear as [anchor text](url) but the URLs are often tracking redirects, so judge by the words, not the links.\n\n Identify every distinct product, tool, model, SDK, dataset, or launch the newsletter is reporting as news. For each, return:\n - name: the exact product/tool/model name\n - description: a concise phrase describing what it is and does, suitable as a web search query\n\n INCLUDE only real, named launches a reader could go use or read about.\n\n IGNORE:\n - the author's intro letter, essay, or personal commentary\n - sponsor blurbs, advertisements, \"brought to you by\", upgrade / subscribe / manage-preferences prompts\n - social-media embeds (an @handle with like / repost / reply counts) and the newsletter's own social links\n - section headers, navigation, and unsubscribe links\n\n Newsletter text:\n {{ newsletterText }}\n\n If the text names no qualifying products, return an empty list.\n\n {{ ctx.output_format }}\n \"#\n}\n\n// Notability gate: trims the extracted list to the launches actually worth\n// resolving, so we don't spend a web search on every incremental SDK bump.\n// Returns the 1-based indices of the products to KEEP.\nfunction SelectNotableProducts(products: NewsletterProduct[]) -> int[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n Below is a numbered list of products/tools/models a newsletter mentioned. Return the numbers of the ones that are NOTABLE enough to feature in an AI-news aggregator.\n\n KEEP a product if it is a distinct, real, individually newsworthy launch a reader would want to click: new models, significant new products or platforms, notable agents/tools, major releases.\n\n DROP:\n - minor or incremental items: small SDK/utility bumps, niche helper libraries, design skills, config tweaks\n - vague or marketing-only entries with no concrete product behind them\n - duplicates or sub-features of another item in the list\n\n Products:\n {% for p in products %}\n {{ loop.index }}. {{ p.name }} — {{ p.description }}\n {% endfor %}\n\n Return only the numbers to keep, as a JSON array of integers. If none qualify, return [].\n\n {{ ctx.output_format }}\n \"#\n}\n\nclass SearchCandidate {\n title string\n url string\n snippet string\n}\n\nclass ProductLinkChoice {\n index int @description(\"1-based index of the best first-party result, or 0 if none qualifies\")\n confidence float @description(\"0.0 - 1.0\")\n}\n\n// Picks the best canonical URL for a product among search candidates, biased\n// hard toward the official / first-party source and away from social posts and\n// SEO/aggregator pages. Returns index 0 when nothing is clearly the product.\nfunction SelectProductLink(\n productName: string,\n productDescription: string,\n candidates: SearchCandidate[]\n) -> ProductLinkChoice {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n A newsletter featured this item:\n name: {{ productName }}\n description: {{ productDescription }}\n\n Web search returned these candidate results:\n {% for c in candidates %}\n {{ loop.index }}. {{ c.title }}\n url: {{ c.url }}\n snippet: {{ c.snippet }}\n {% endfor %}\n\n Pick the ONE candidate that is the canonical, first-party source for this exact product — its official site, product page, model card, repo, or the vendor's own announcement.\n\n PREFER the official / first-party domain that matches the product or its maker.\n REJECT (do not pick) when it is the best available:\n - social-media posts (x.com, twitter.com, linkedin.com, reddit.com, threads, mastodon)\n - videos (youtube.com)\n - content-farm / SEO / generic aggregator or listicle pages that merely mention the product\n - a DIFFERENT product that happens to share the name\n\n Return the 1-based index of the best first-party result. If none of the candidates is clearly a first-party source for THIS product, return index 0.\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest extract_products_basic {\n functions [ExtractProducts]\n args {\n newsletterText \"Headlines: [OpenAI ships GPT-5](https://sub.stack/redirect/2/abc), a new flagship model. Brought to you by Acme. [Unsubscribe](https://newsletter.com/unsub).\"\n }\n}\n", + "fix_titles.baml": "class TitleFix {\n url string @description(\"The exact URL from the matching input article - copy verbatim\")\n title string @description(\"Plain title text only. No surrounding quotes. No leading source name or bracket tag. Max 12 words. Return the input title verbatim if it is already clear, specific, and informative.\")\n}\n\nfunction ImproveTitles(articles: ArticleInput[]) -> TitleFix[] {\n client NvidiaGptOss\n prompt #\"\n {{ _.role(\"system\") }}\n You are a technical editor. You write clear, informative article titles for an AI developer news platform.\n\n Output rules (strict):\n - For each input article, return one TitleFix whose `url` exactly matches that article's URL.\n - The `title` field contains plain title text ONLY. Do NOT wrap it in quotes. Do NOT prefix it with \"[Source]\" or any source name. Do NOT include the URL.\n - Use only the snippet of the SAME article to inform the rewrite. Never blend information across articles.\n\n {{ _.role(\"user\") }}\n Decide for each article whether the existing title is already good. A good title is specific, informative, and technical/neutral in register - it tells a developer what the article is actually about.\n\n - If the existing title is already good, return it verbatim.\n - Otherwise rewrite it using only that article's snippet: specific, informative, max 12 words, technical/neutral register, no hype or filler.\n\n {% for a in articles %}\n --- Article {{ loop.index }} ---\n url: {{ a.url }}\n source: {{ a.source }}\n current_title: {{ a.title }}\n {% if a.snippet %}snippet: {{ a.snippet }}{% endif %}\n {% endfor %}\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest improve_titles_rewrite_short {\n functions [ImproveTitles]\n args {\n articles [\n {\n url \"https://example.com/claude\"\n title \"Claude 4\"\n source \"HN\"\n snippet \"Anthropic today released Claude 4, a new flagship model with improved reasoning and a 1M-token context window.\"\n }\n ]\n }\n}\n\ntest improve_titles_keep_unchanged {\n functions [ImproveTitles]\n args {\n articles [\n {\n url \"https://example.com/post\"\n title \"Anthropic releases Claude 4 with 1M-token context window\"\n source \"HN\"\n snippet \"Anthropic today released Claude 4, a new flagship model with improved reasoning.\"\n }\n ]\n }\n}\n", + "generators.baml": "generator python {\n output_type \"python/pydantic\"\n output_dir \"../backend/\"\n version \"0.223.0\"\n}\n", + "score.baml": "class ScoredArticle {\n url string\n score int @description(\"1-100 developer-actionability rating\")\n}\n\nfunction ScoreArticles(articles: ArticleInput[]) -> ScoredArticle[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"system\") }}\n You are a senior editorial analyst at a developer news platform. Your job is to evaluate articles for relevance and actionability for AI builders. You are precise, consistent, and ruthless about filtering noise.\n\n {{ _.role(\"user\") }}\n Score every article for developer-actionability on a 1-100 scale.\n\n STEP 1 - SCOPE CHECK (apply first, before scoring):\n Is the article about AI, ML, LLMs, foundation models, or AI-adjacent developer tools/infrastructure?\n - NO → score 1-20 and move on. Do NOT apply the scoring guide below.\n - Out-of-scope examples: text editors, parsers, version control tools, data structures (CRDTs, B-trees), memory management, general programming tutorials, non-AI open source projects, codebase audits, legacy code. These are general software engineering - not AI.\n\n STEP 2 - SCORE (only for in-scope articles):\n Audience: developers, startup founders, and tech leaders building with AI who want to ship faster.\n\n Score 91-100 is reserved for exceptional articles that are BOTH immediately actionable today AND fit one of these themes: local/on-device inference, privacy-preserving AI, efficiency breakthroughs (quantization, token optimization, running models on less hardware), or deep technical dives that explain how something actually works. A score in this range means \"a developer could open a terminal and act on this within the hour, AND it reveals a non-obvious insight or technique.\" Zero is a valid outcome - do not give 91+ just because something is the best of a weak batch.\n\n Models (new releases, benchmarks, evals, model cards, comparisons):\n - 81-90: New model release or major update, model system cards, open-weight models a developer can run today\n - 71-80: Meaningful benchmark or eval, model comparison with actionable conclusions\n\n Dev (SDKs, frameworks, CLI tools, IDE integrations, open-source projects, tutorials, deployment):\n - 81-90: New AI API or SDK, open-source AI tool/library a developer can use today\n - 71-80: Fine-tuning guide, RAG pattern, significant framework update, AI infrastructure (training, inference, memory optimization), deployment recipes\n\n Research (papers, breakthroughs, novel techniques):\n - 71-80: Research with immediate practical implications, AI security research\n - 51-60: Research with near-term practical implications\n\n Industry (funding, platform changes, acquisitions, partnerships, market shifts):\n - 51-60: Industry analysis with actionable takeaways, AI platform partnerships\n - 31-40: Funding rounds without product details, opinion/commentary with no actionable steps, \"AI is changing everything\" think pieces, personal anecdotes about using AI\n\n Upgrade to 91-100: if an article would score 81-90 AND fits the top-tier criteria above (local inference, privacy-preserving, efficiency breakthrough, or deep how-it-works dive), give it 91-100.\n\n Bottom tier:\n - 1-20: Non-AI topics (see step 1), regulation, politics, lawsuits, corporate drama\n\n Trust bonus: entries may include a [trust: high/medium/low] tag. Give +1 to high-trust sources when content quality is comparable.\n\n Articles:\n {% for a in articles %}\n {{ loop.index }}. [{{ a.source }}] \"{{ a.title }}\" - {{ a.url }}{% if a.trust %} [trust: {{ a.trust }}]{% endif %}\n {% if a.snippet %} {{ a.snippet }}{% endif %}\n {% endfor %}\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest score_gpt5_release {\n functions [ScoreArticles]\n args {\n articles [\n {\n url \"https://openai.com/index/introducing-gpt-5\"\n title \"Introducing GPT-5\"\n source \"Hacker News\"\n trust \"medium\"\n }\n ]\n }\n}\n\ntest score_out_of_scope {\n functions [ScoreArticles]\n args {\n articles [\n {\n url \"https://example.com/crdt-deep-dive\"\n title \"A Deep Dive Into CRDTs and Conflict Resolution\"\n source \"Some Blog\"\n trust \"low\"\n }\n ]\n }\n}\n", + "shared.baml": "// Shared types used by multiple BAML functions.\n\nenum ArticleCategory {\n Models\n Dev\n Research\n}\n\nenum ArticleKind {\n Repo // GitHub/GitLab repository\n Paper // Academic paper, preprint, technical report\n Model // Model card or model release page\n Blog // Developer blog post, tutorial, how-to, opinion piece\n Product // Product launch page or product website\n Announcement // Official release post from a major AI lab\n}\n\n// Used as input to ScoreArticles, ImproveTitles, SemanticDedup.\n// Mirrors the fields the existing formatArticles() helper emits.\nclass ArticleInput {\n url string\n title string\n source string\n snippet string? @description(\"First ~200 chars of the article content, if any\")\n trust string? @description(\"\\\"high\\\" | \\\"medium\\\" | \\\"low\\\" - source trust tag\")\n}\n\n// Used as input to SemanticDedup for rows already in the DB.\nclass ExistingArticle {\n url string\n title string\n source string\n}\n", + "summarize.baml": "class SummarizeAndCategorizeResult {\n summary string @description(\"2-3 short factual lines separated by \\\\n\")\n categories ArticleCategory[] @description(\"1-2 categories\")\n kind ArticleKind\n}\n\nclass CategorizeOnlyResult {\n categories ArticleCategory[] @description(\"1-2 categories\")\n}\n\nclass ClassifyKindResult {\n kind ArticleKind\n}\n\nfunction SummarizeAndCategorize(title: string, content: string) -> SummarizeAndCategorizeResult {\n client NvidiaGptOss\n prompt #\"\n {{ _.role(\"system\") }}\n You are a technical writer for a developer news digest. Your readers are tech-savvy but busy. Write in plain, direct language.\n\n {{ _.role(\"user\") }}\n Summarize this article. Be plain and brief.\n\n Title: {{ title }}\n\n Content:\n {{ content }}\n\n 1. Write 2-3 lines. Keep each line to one short sentence. Rules:\n - Only state facts from the article.\n - No passive voice. No \"enables\", \"allows\", \"provides\".\n - Separate lines with a newline.\n 2. Assign 1-2 categories.\n 3. Assign a kind.\n\n Category definitions:\n - Models: New model releases, benchmarks, evals, model cards, model comparisons\n - Dev: Repos, tools, product launches, developer blogs, tutorials, how-tos, fine-tuning guides, deployment recipes, infrastructure updates\n - Research: Papers, breakthroughs, novel techniques with practical implications\n\n Kind definitions:\n - Repo: GitHub or GitLab repository\n - Paper: Academic paper, preprint, or long-form technical report\n - Model: Model card or model release page (e.g. HuggingFace)\n - Blog: Developer blog post, tutorial, how-to, or opinion piece\n - Product: Product launch page or product website\n - Announcement: Official release post from a major AI lab (OpenAI, Anthropic, Google, Meta, Mistral, etc.)\n\n {{ ctx.output_format }}\n \"#\n}\n\nfunction ClassifyKind(title: string, url: string, summary: string?) -> ClassifyKindResult {\n client NvidiaGptOss\n prompt #\"\n {{ _.role(\"system\") }}\n You are a technical librarian who classifies AI developer content by format.\n\n {{ _.role(\"user\") }}\n Classify this article's kind.\n\n Title: {{ title }}\n URL: {{ url }}\n {% if summary %}Summary: {{ summary }}{% endif %}\n\n Kind definitions:\n - Repo: GitHub or GitLab repository\n - Paper: Academic paper, preprint, or long-form technical report\n - Model: Model card or model release page (e.g. HuggingFace)\n - Blog: Developer blog post, tutorial, how-to, or opinion piece\n - Product: Product launch page or product website\n - Announcement: Official release post from a major AI lab (OpenAI, Anthropic, Google, Meta, Mistral, etc.)\n\n {{ ctx.output_format }}\n \"#\n}\n\nfunction CategorizeOnly(title: string) -> CategorizeOnlyResult {\n client NvidiaGptOss\n prompt #\"\n {{ _.role(\"system\") }}\n You are a technical librarian who classifies developer-focused AI articles.\n\n {{ _.role(\"user\") }}\n Assign 1-2 categories to this article title.\n\n Title: {{ title }}\n\n Category definitions:\n - Models: New model releases, benchmarks, evals, model cards, model comparisons\n - Dev: Repos, tools, product launches, developer blogs, tutorials, how-tos, fine-tuning guides, deployment recipes, infrastructure updates\n - Research: Papers, breakthroughs, novel techniques with practical implications\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest summarize_basic {\n functions [SummarizeAndCategorize]\n args {\n title \"Anthropic releases Claude 4\"\n content \"Anthropic today launched Claude 4, a new flagship model. It features improved reasoning, a 1M-token context window, and native tool use. The model is available via API and Claude.ai starting today.\"\n }\n}\n\ntest categorize_only_basic {\n functions [CategorizeOnly]\n args {\n title \"OpenAI releases GPT-5 with 2M context\"\n }\n}\n", +} + +def get_baml_files(): + return _file_map \ No newline at end of file diff --git a/backend/baml_client/parser.py b/backend/baml_client/parser.py new file mode 100644 index 0000000000..f41c08fa06 --- /dev/null +++ b/backend/baml_client/parser.py @@ -0,0 +1,166 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +import typing +import typing_extensions + + +from . import stream_types, types +from .runtime import DoNotUseDirectlyCallManager, BamlCallOptions + +class LlmResponseParser: + __options: DoNotUseDirectlyCallManager + + def __init__(self, options: DoNotUseDirectlyCallManager): + self.__options = options + + def CategorizeOnly( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> types.CategorizeOnlyResult: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="CategorizeOnly", llm_response=llm_response, mode="request") + return typing.cast(types.CategorizeOnlyResult, __result__) + + def ClassifyKind( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> types.ClassifyKindResult: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="ClassifyKind", llm_response=llm_response, mode="request") + return typing.cast(types.ClassifyKindResult, __result__) + + def ClassifyProfiles( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List["types.ProfileVerdict"]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="ClassifyProfiles", llm_response=llm_response, mode="request") + return typing.cast(typing.List["types.ProfileVerdict"], __result__) + + def ExtractLinks( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List["types.ExtractedLink"]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="ExtractLinks", llm_response=llm_response, mode="request") + return typing.cast(typing.List["types.ExtractedLink"], __result__) + + def ExtractProducts( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List["types.NewsletterProduct"]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="ExtractProducts", llm_response=llm_response, mode="request") + return typing.cast(typing.List["types.NewsletterProduct"], __result__) + + def ImproveTitles( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List["types.TitleFix"]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="ImproveTitles", llm_response=llm_response, mode="request") + return typing.cast(typing.List["types.TitleFix"], __result__) + + def ScoreArticles( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List["types.ScoredArticle"]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="ScoreArticles", llm_response=llm_response, mode="request") + return typing.cast(typing.List["types.ScoredArticle"], __result__) + + def SelectNotableProducts( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List[int]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="SelectNotableProducts", llm_response=llm_response, mode="request") + return typing.cast(typing.List[int], __result__) + + def SelectProductLink( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> types.ProductLinkChoice: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="SelectProductLink", llm_response=llm_response, mode="request") + return typing.cast(types.ProductLinkChoice, __result__) + + def SemanticDedup( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List["types.DedupMatch"]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="SemanticDedup", llm_response=llm_response, mode="request") + return typing.cast(typing.List["types.DedupMatch"], __result__) + + def SummarizeAndCategorize( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> types.SummarizeAndCategorizeResult: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="SummarizeAndCategorize", llm_response=llm_response, mode="request") + return typing.cast(types.SummarizeAndCategorizeResult, __result__) + + + +class LlmStreamParser: + __options: DoNotUseDirectlyCallManager + + def __init__(self, options: DoNotUseDirectlyCallManager): + self.__options = options + + def CategorizeOnly( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> stream_types.CategorizeOnlyResult: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="CategorizeOnly", llm_response=llm_response, mode="stream") + return typing.cast(stream_types.CategorizeOnlyResult, __result__) + + def ClassifyKind( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> stream_types.ClassifyKindResult: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="ClassifyKind", llm_response=llm_response, mode="stream") + return typing.cast(stream_types.ClassifyKindResult, __result__) + + def ClassifyProfiles( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List["stream_types.ProfileVerdict"]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="ClassifyProfiles", llm_response=llm_response, mode="stream") + return typing.cast(typing.List["stream_types.ProfileVerdict"], __result__) + + def ExtractLinks( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List["stream_types.ExtractedLink"]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="ExtractLinks", llm_response=llm_response, mode="stream") + return typing.cast(typing.List["stream_types.ExtractedLink"], __result__) + + def ExtractProducts( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List["stream_types.NewsletterProduct"]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="ExtractProducts", llm_response=llm_response, mode="stream") + return typing.cast(typing.List["stream_types.NewsletterProduct"], __result__) + + def ImproveTitles( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List["stream_types.TitleFix"]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="ImproveTitles", llm_response=llm_response, mode="stream") + return typing.cast(typing.List["stream_types.TitleFix"], __result__) + + def ScoreArticles( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List["stream_types.ScoredArticle"]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="ScoreArticles", llm_response=llm_response, mode="stream") + return typing.cast(typing.List["stream_types.ScoredArticle"], __result__) + + def SelectNotableProducts( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List[int]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="SelectNotableProducts", llm_response=llm_response, mode="stream") + return typing.cast(typing.List[int], __result__) + + def SelectProductLink( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> stream_types.ProductLinkChoice: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="SelectProductLink", llm_response=llm_response, mode="stream") + return typing.cast(stream_types.ProductLinkChoice, __result__) + + def SemanticDedup( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> typing.List["stream_types.DedupMatch"]: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="SemanticDedup", llm_response=llm_response, mode="stream") + return typing.cast(typing.List["stream_types.DedupMatch"], __result__) + + def SummarizeAndCategorize( + self, llm_response: str, baml_options: BamlCallOptions = {}, + ) -> stream_types.SummarizeAndCategorizeResult: + __result__ = self.__options.merge_options(baml_options).parse_response(function_name="SummarizeAndCategorize", llm_response=llm_response, mode="stream") + return typing.cast(stream_types.SummarizeAndCategorizeResult, __result__) + + \ No newline at end of file diff --git a/backend/baml_client/runtime.py b/backend/baml_client/runtime.py new file mode 100644 index 0000000000..27fc3a9b45 --- /dev/null +++ b/backend/baml_client/runtime.py @@ -0,0 +1,361 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +import os +import typing +import typing_extensions + +import baml_py + +from . import types, stream_types, type_builder +from .globals import DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME as __runtime__, DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX as __ctx__manager__ + + +class BamlCallOptions(typing.TypedDict, total=False): + tb: typing_extensions.NotRequired[type_builder.TypeBuilder] + client_registry: typing_extensions.NotRequired[baml_py.baml_py.ClientRegistry] + client: typing_extensions.NotRequired[str] + env: typing_extensions.NotRequired[typing.Dict[str, typing.Optional[str]]] + tags: typing_extensions.NotRequired[typing.Dict[str, str]] + collector: typing_extensions.NotRequired[ + typing.Union[baml_py.baml_py.Collector, typing.List[baml_py.baml_py.Collector]] + ] + abort_controller: typing_extensions.NotRequired[baml_py.baml_py.AbortController] + on_tick: typing_extensions.NotRequired[typing.Callable[[str, baml_py.baml_py.FunctionLog], None]] + watchers: typing_extensions.NotRequired[typing.Any] # EventCollector type, will be overridden in generated clients + + +class _ResolvedBamlOptions: + tb: typing.Optional[baml_py.baml_py.TypeBuilder] + client_registry: typing.Optional[baml_py.baml_py.ClientRegistry] + collectors: typing.List[baml_py.baml_py.Collector] + env_vars: typing.Dict[str, str] + tags: typing.Dict[str, str] + abort_controller: typing.Optional[baml_py.baml_py.AbortController] + on_tick: typing.Optional[typing.Callable[[], None]] + watchers: typing.Optional[typing.Any] + + def __init__( + self, + tb: typing.Optional[baml_py.baml_py.TypeBuilder], + client_registry: typing.Optional[baml_py.baml_py.ClientRegistry], + collectors: typing.List[baml_py.baml_py.Collector], + env_vars: typing.Dict[str, str], + tags: typing.Dict[str, str], + abort_controller: typing.Optional[baml_py.baml_py.AbortController], + on_tick: typing.Optional[typing.Callable[[], None]], + watchers: typing.Optional[typing.Any], + ): + self.tb = tb + self.client_registry = client_registry + self.collectors = collectors + self.env_vars = env_vars + self.tags = tags + self.abort_controller = abort_controller + self.on_tick = on_tick + self.watchers = watchers + + + + +class DoNotUseDirectlyCallManager: + def __init__(self, baml_options: BamlCallOptions): + self.__baml_options = baml_options + + def __getstate__(self): + # Return state needed for pickling + return {"baml_options": self.__baml_options} + + def __setstate__(self, state): + # Restore state from pickling + self.__baml_options = state["baml_options"] + + def __resolve(self) -> _ResolvedBamlOptions: + tb = self.__baml_options.get("tb") + if tb is not None: + baml_tb = tb._tb # type: ignore (we know how to use this private attribute) + else: + baml_tb = None + client_registry = self.__baml_options.get("client_registry") + client = self.__baml_options.get("client") + + # If client is provided, it takes precedence (creates/overrides client_registry primary) + if client is not None: + if client_registry is None: + client_registry = baml_py.baml_py.ClientRegistry() + client_registry.set_primary(client) + + collector = self.__baml_options.get("collector") + collectors_as_list = ( + collector + if isinstance(collector, list) + else [collector] if collector is not None else [] + ) + env_vars = os.environ.copy() + for k, v in self.__baml_options.get("env", {}).items(): + if v is not None: + env_vars[k] = v + else: + env_vars.pop(k, None) + + tags = self.__baml_options.get("tags", {}) or {} + + abort_controller = self.__baml_options.get("abort_controller") + + on_tick = self.__baml_options.get("on_tick") + if on_tick is not None: + collector = baml_py.baml_py.Collector("on-tick-collector") + collectors_as_list.append(collector) + def on_tick_wrapper(): + log = collector.last + if log is not None: + on_tick("Unknown", log) + else: + on_tick_wrapper = None + + watchers = self.__baml_options.get("watchers") + + return _ResolvedBamlOptions( + baml_tb, + client_registry, + collectors_as_list, + env_vars, + tags, + abort_controller, + on_tick_wrapper, + watchers, + ) + + def merge_options(self, options: BamlCallOptions) -> "DoNotUseDirectlyCallManager": + return DoNotUseDirectlyCallManager({**self.__baml_options, **options}) + + async def call_function_async( + self, *, function_name: str, args: typing.Dict[str, typing.Any] + ) -> baml_py.baml_py.FunctionResult: + resolved_options = self.__resolve() + + # Check if already aborted + if resolved_options.abort_controller is not None and resolved_options.abort_controller.aborted: + raise baml_py.baml_py.BamlAbortError("Operation was aborted") + + return await __runtime__.call_function( + function_name, + args, + # ctx + __ctx__manager__.clone_context(), + # tb + resolved_options.tb, + # cr + resolved_options.client_registry, + # collectors + resolved_options.collectors, + # env_vars + resolved_options.env_vars, + # tags + resolved_options.tags, + # abort_controller + resolved_options.abort_controller, + # watchers + resolved_options.watchers, + ) + + def call_function_sync( + self, *, function_name: str, args: typing.Dict[str, typing.Any] + ) -> baml_py.baml_py.FunctionResult: + resolved_options = self.__resolve() + + # Check if already aborted + if resolved_options.abort_controller is not None and resolved_options.abort_controller.aborted: + raise baml_py.baml_py.BamlAbortError("Operation was aborted") + + ctx = __ctx__manager__.get() + return __runtime__.call_function_sync( + function_name, + args, + # ctx + ctx, + # tb + resolved_options.tb, + # cr + resolved_options.client_registry, + # collectors + resolved_options.collectors, + # env_vars + resolved_options.env_vars, + # tags + resolved_options.tags, + # abort_controller + resolved_options.abort_controller, + # watchers + resolved_options.watchers, + ) + + def create_async_stream( + self, + *, + function_name: str, + args: typing.Dict[str, typing.Any], + ) -> typing.Tuple[baml_py.baml_py.RuntimeContextManager, baml_py.baml_py.FunctionResultStream]: + resolved_options = self.__resolve() + ctx = __ctx__manager__.clone_context() + result = __runtime__.stream_function( + function_name, + args, + # this is always None, we set this later! + # on_event + None, + # ctx + ctx, + # tb + resolved_options.tb, + # cr + resolved_options.client_registry, + # collectors + resolved_options.collectors, + # env_vars + resolved_options.env_vars, + # tags + resolved_options.tags, + # on_tick + resolved_options.on_tick, + # abort_controller + resolved_options.abort_controller, + ) + return ctx, result + + def create_sync_stream( + self, + *, + function_name: str, + args: typing.Dict[str, typing.Any], + ) -> typing.Tuple[baml_py.baml_py.RuntimeContextManager, baml_py.baml_py.SyncFunctionResultStream]: + resolved_options = self.__resolve() + if resolved_options.on_tick is not None: + raise ValueError("on_tick is not supported for sync streams. Please use async streams instead.") + ctx = __ctx__manager__.get() + result = __runtime__.stream_function_sync( + function_name, + args, + # this is always None, we set this later! + # on_event + None, + # ctx + ctx, + # tb + resolved_options.tb, + # cr + resolved_options.client_registry, + # collectors + resolved_options.collectors, + # env_vars + resolved_options.env_vars, + # tags + resolved_options.tags, + # on_tick + # always None! sync streams don't support on_tick + None, + # abort_controller + resolved_options.abort_controller, + ) + return ctx, result + + async def create_http_request_async( + self, + *, + function_name: str, + args: typing.Dict[str, typing.Any], + mode: typing_extensions.Literal["stream", "request"], + ) -> baml_py.baml_py.HTTPRequest: + resolved_options = self.__resolve() + return await __runtime__.build_request( + function_name, + args, + # ctx + __ctx__manager__.clone_context(), + # tb + resolved_options.tb, + # cr + resolved_options.client_registry, + # env_vars + resolved_options.env_vars, + # is_stream + mode == "stream", + ) + + def create_http_request_sync( + self, + *, + function_name: str, + args: typing.Dict[str, typing.Any], + mode: typing_extensions.Literal["stream", "request"], + ) -> baml_py.baml_py.HTTPRequest: + resolved_options = self.__resolve() + return __runtime__.build_request_sync( + function_name, + args, + # ctx + __ctx__manager__.get(), + # tb + resolved_options.tb, + # cr + resolved_options.client_registry, + # env_vars + resolved_options.env_vars, + # is_stream + mode == "stream", + ) + + def parse_response(self, *, function_name: str, llm_response: str, mode: typing_extensions.Literal["stream", "request"]) -> typing.Any: + resolved_options = self.__resolve() + return __runtime__.parse_llm_response( + function_name, + llm_response, + # enum_module + types, + # cls_module + types, + # partial_cls_module + stream_types, + # allow_partials + mode == "stream", + # ctx + __ctx__manager__.get(), + # tb + resolved_options.tb, + # cr + resolved_options.client_registry, + # env_vars + resolved_options.env_vars, + ) + + +def disassemble(function: typing.Callable) -> None: + import inspect + from . import b + + if not callable(function): + print(f"disassemble: object {function} is not a Baml function") + return + + is_client_method = False + + for (method_name, _) in inspect.getmembers(b, predicate=inspect.ismethod): + if method_name == function.__name__: + is_client_method = True + break + + if not is_client_method: + print(f"disassemble: function {function.__name__} is not a Baml function") + return + + print(f"----- function {function.__name__} -----") + __runtime__.disassemble(function.__name__) \ No newline at end of file diff --git a/backend/baml_client/stream_types.py b/backend/baml_client/stream_types.py new file mode 100644 index 0000000000..2032f01469 --- /dev/null +++ b/backend/baml_client/stream_types.py @@ -0,0 +1,94 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +import typing +import typing_extensions +from pydantic import BaseModel, ConfigDict, Field + +import baml_py + +from . import types + +StreamStateValueT = typing.TypeVar('StreamStateValueT') +class StreamState(BaseModel, typing.Generic[StreamStateValueT]): + value: StreamStateValueT + state: typing_extensions.Literal["Pending", "Incomplete", "Complete"] +# ######################################################################### +# Generated classes (14) +# ######################################################################### + +class ArticleInput(BaseModel): + url: typing.Optional[str] = None + title: typing.Optional[str] = None + source: typing.Optional[str] = None + snippet: typing.Optional[str] = Field(default=None, description='First ~200 chars of the article content, if any') + trust: typing.Optional[str] = Field(default=None, description='"high" | "medium" | "low" - source trust tag') + +class CategorizeOnlyResult(BaseModel): + categories: typing.List[types.ArticleCategory] = Field(description='1-2 categories') + +class ClassifyKindResult(BaseModel): + kind: typing.Optional[types.ArticleKind] = None + +class DedupMatch(BaseModel): + url: typing.Optional[str] = Field(default=None, description='URL of the new article') + existingUrl: typing.Optional[str] = Field(default=None, description='URL of the existing article it duplicates') + +class ExistingArticle(BaseModel): + url: typing.Optional[str] = None + title: typing.Optional[str] = None + source: typing.Optional[str] = None + +class ExtractedLink(BaseModel): + title: typing.Optional[str] = None + url: typing.Optional[str] = None + snippet: typing.Optional[str] = Field(default=None, description='One-sentence description of the article') + +class NewsletterProduct(BaseModel): + name: typing.Optional[str] = Field(default=None, description='Exact product / tool / model / launch name') + description: typing.Optional[str] = Field(default=None, description='Concise phrase: what it is and does, usable verbatim as a web search query') + +class ProductLinkChoice(BaseModel): + index: typing.Optional[int] = Field(default=None, description='1-based index of the best first-party result, or 0 if none qualifies') + confidence: typing.Optional[float] = Field(default=None, description='0.0 - 1.0') + +class ProfileInput(BaseModel): + handle: typing.Optional[str] = Field(default=None, description='Substack owner handle, echoed back in the verdict') + name: typing.Optional[str] = Field(default=None, description='Publication name') + description: typing.Optional[str] = Field(default=None, description='Publication hero text and/or author bio, if known') + +class ProfileVerdict(BaseModel): + handle: typing.Optional[str] = Field(default=None, description='Must exactly match the input handle') + isAiRelated: typing.Optional[bool] = None + confidence: typing.Optional[int] = Field(default=None, description='0-100 confidence that AI/ML/LLMs is the PRIMARY focus') + +class ScoredArticle(BaseModel): + url: typing.Optional[str] = None + score: typing.Optional[int] = Field(default=None, description='1-100 developer-actionability rating') + +class SearchCandidate(BaseModel): + title: typing.Optional[str] = None + url: typing.Optional[str] = None + snippet: typing.Optional[str] = None + +class SummarizeAndCategorizeResult(BaseModel): + summary: typing.Optional[str] = Field(default=None, description='2-3 short factual lines separated by \\n') + categories: typing.List[types.ArticleCategory] = Field(description='1-2 categories') + kind: typing.Optional[types.ArticleKind] = None + +class TitleFix(BaseModel): + url: typing.Optional[str] = Field(default=None, description='The exact URL from the matching input article - copy verbatim') + title: typing.Optional[str] = Field(default=None, description='Plain title text only. No surrounding quotes. No leading source name or bracket tag. Max 12 words. Return the input title verbatim if it is already clear, specific, and informative.') + +# ######################################################################### +# Generated type aliases (0) +# ######################################################################### diff --git a/backend/baml_client/sync_client.py b/backend/baml_client/sync_client.py new file mode 100644 index 0000000000..0654247051 --- /dev/null +++ b/backend/baml_client/sync_client.py @@ -0,0 +1,564 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +import typing +import typing_extensions +import baml_py + +from . import stream_types, types, type_builder +from .parser import LlmResponseParser, LlmStreamParser +from .runtime import DoNotUseDirectlyCallManager, BamlCallOptions +from .globals import DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME as __runtime__ + +class BamlSyncClient: + __options: DoNotUseDirectlyCallManager + __stream_client: "BamlStreamClient" + __http_request: "BamlHttpRequestClient" + __http_stream_request: "BamlHttpStreamRequestClient" + __llm_response_parser: LlmResponseParser + __llm_stream_parser: LlmStreamParser + + def __init__(self, options: DoNotUseDirectlyCallManager): + self.__options = options + self.__stream_client = BamlStreamClient(options) + self.__http_request = BamlHttpRequestClient(options) + self.__http_stream_request = BamlHttpStreamRequestClient(options) + self.__llm_response_parser = LlmResponseParser(options) + self.__llm_stream_parser = LlmStreamParser(options) + + def __getstate__(self): + # Return state needed for pickling + return {"options": self.__options} + + def __setstate__(self, state): + # Restore state from pickling + self.__options = state["options"] + self.__stream_client = BamlStreamClient(self.__options) + self.__http_request = BamlHttpRequestClient(self.__options) + self.__http_stream_request = BamlHttpStreamRequestClient(self.__options) + self.__llm_response_parser = LlmResponseParser(self.__options) + self.__llm_stream_parser = LlmStreamParser(self.__options) + + def with_options(self, + tb: typing.Optional[type_builder.TypeBuilder] = None, + client_registry: typing.Optional[baml_py.baml_py.ClientRegistry] = None, + client: typing.Optional[str] = None, + collector: typing.Optional[typing.Union[baml_py.baml_py.Collector, typing.List[baml_py.baml_py.Collector]]] = None, + env: typing.Optional[typing.Dict[str, typing.Optional[str]]] = None, + tags: typing.Optional[typing.Dict[str, str]] = None, + on_tick: typing.Optional[typing.Callable[[str, baml_py.baml_py.FunctionLog], None]] = None, + ) -> "BamlSyncClient": + options: BamlCallOptions = {} + if tb is not None: + options["tb"] = tb + if client_registry is not None: + options["client_registry"] = client_registry + if client is not None: + options["client"] = client + if collector is not None: + options["collector"] = collector + if env is not None: + options["env"] = env + if tags is not None: + options["tags"] = tags + if on_tick is not None: + options["on_tick"] = on_tick + return BamlSyncClient(self.__options.merge_options(options)) + + @property + def stream(self): + return self.__stream_client + + @property + def request(self): + return self.__http_request + + @property + def stream_request(self): + return self.__http_stream_request + + @property + def parse(self): + return self.__llm_response_parser + + @property + def parse_stream(self): + return self.__llm_stream_parser + + def CategorizeOnly(self, title: str, + baml_options: BamlCallOptions = {}, + ) -> types.CategorizeOnlyResult: + # Check if on_tick is provided + if 'on_tick' in baml_options: + __stream__ = self.stream.CategorizeOnly(title=title, + baml_options=baml_options) + return __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = self.__options.merge_options(baml_options).call_function_sync(function_name="CategorizeOnly", args={ + "title": title, + }) + return typing.cast(types.CategorizeOnlyResult, __result__.cast_to(types, types, stream_types, False, __runtime__)) + def ClassifyKind(self, title: str,url: str,summary: typing.Optional[str] = None, + baml_options: BamlCallOptions = {}, + ) -> types.ClassifyKindResult: + # Check if on_tick is provided + if 'on_tick' in baml_options: + __stream__ = self.stream.ClassifyKind(title=title,url=url,summary=summary, + baml_options=baml_options) + return __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = self.__options.merge_options(baml_options).call_function_sync(function_name="ClassifyKind", args={ + "title": title,"url": url,"summary": summary, + }) + return typing.cast(types.ClassifyKindResult, __result__.cast_to(types, types, stream_types, False, __runtime__)) + def ClassifyProfiles(self, profiles: typing.List["types.ProfileInput"], + baml_options: BamlCallOptions = {}, + ) -> typing.List["types.ProfileVerdict"]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + __stream__ = self.stream.ClassifyProfiles(profiles=profiles, + baml_options=baml_options) + return __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = self.__options.merge_options(baml_options).call_function_sync(function_name="ClassifyProfiles", args={ + "profiles": profiles, + }) + return typing.cast(typing.List["types.ProfileVerdict"], __result__.cast_to(types, types, stream_types, False, __runtime__)) + def ExtractLinks(self, emailHtml: str, + baml_options: BamlCallOptions = {}, + ) -> typing.List["types.ExtractedLink"]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + __stream__ = self.stream.ExtractLinks(emailHtml=emailHtml, + baml_options=baml_options) + return __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = self.__options.merge_options(baml_options).call_function_sync(function_name="ExtractLinks", args={ + "emailHtml": emailHtml, + }) + return typing.cast(typing.List["types.ExtractedLink"], __result__.cast_to(types, types, stream_types, False, __runtime__)) + def ExtractProducts(self, newsletterText: str, + baml_options: BamlCallOptions = {}, + ) -> typing.List["types.NewsletterProduct"]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + __stream__ = self.stream.ExtractProducts(newsletterText=newsletterText, + baml_options=baml_options) + return __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = self.__options.merge_options(baml_options).call_function_sync(function_name="ExtractProducts", args={ + "newsletterText": newsletterText, + }) + return typing.cast(typing.List["types.NewsletterProduct"], __result__.cast_to(types, types, stream_types, False, __runtime__)) + def ImproveTitles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> typing.List["types.TitleFix"]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + __stream__ = self.stream.ImproveTitles(articles=articles, + baml_options=baml_options) + return __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = self.__options.merge_options(baml_options).call_function_sync(function_name="ImproveTitles", args={ + "articles": articles, + }) + return typing.cast(typing.List["types.TitleFix"], __result__.cast_to(types, types, stream_types, False, __runtime__)) + def ScoreArticles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> typing.List["types.ScoredArticle"]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + __stream__ = self.stream.ScoreArticles(articles=articles, + baml_options=baml_options) + return __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = self.__options.merge_options(baml_options).call_function_sync(function_name="ScoreArticles", args={ + "articles": articles, + }) + return typing.cast(typing.List["types.ScoredArticle"], __result__.cast_to(types, types, stream_types, False, __runtime__)) + def SelectNotableProducts(self, products: typing.List["types.NewsletterProduct"], + baml_options: BamlCallOptions = {}, + ) -> typing.List[int]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + __stream__ = self.stream.SelectNotableProducts(products=products, + baml_options=baml_options) + return __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = self.__options.merge_options(baml_options).call_function_sync(function_name="SelectNotableProducts", args={ + "products": products, + }) + return typing.cast(typing.List[int], __result__.cast_to(types, types, stream_types, False, __runtime__)) + def SelectProductLink(self, productName: str,productDescription: str,candidates: typing.List["types.SearchCandidate"], + baml_options: BamlCallOptions = {}, + ) -> types.ProductLinkChoice: + # Check if on_tick is provided + if 'on_tick' in baml_options: + __stream__ = self.stream.SelectProductLink(productName=productName,productDescription=productDescription,candidates=candidates, + baml_options=baml_options) + return __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = self.__options.merge_options(baml_options).call_function_sync(function_name="SelectProductLink", args={ + "productName": productName,"productDescription": productDescription,"candidates": candidates, + }) + return typing.cast(types.ProductLinkChoice, __result__.cast_to(types, types, stream_types, False, __runtime__)) + def SemanticDedup(self, newArticles: typing.List["types.ArticleInput"],existingArticles: typing.List["types.ExistingArticle"], + baml_options: BamlCallOptions = {}, + ) -> typing.List["types.DedupMatch"]: + # Check if on_tick is provided + if 'on_tick' in baml_options: + __stream__ = self.stream.SemanticDedup(newArticles=newArticles,existingArticles=existingArticles, + baml_options=baml_options) + return __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = self.__options.merge_options(baml_options).call_function_sync(function_name="SemanticDedup", args={ + "newArticles": newArticles,"existingArticles": existingArticles, + }) + return typing.cast(typing.List["types.DedupMatch"], __result__.cast_to(types, types, stream_types, False, __runtime__)) + def SummarizeAndCategorize(self, title: str,content: str, + baml_options: BamlCallOptions = {}, + ) -> types.SummarizeAndCategorizeResult: + # Check if on_tick is provided + if 'on_tick' in baml_options: + __stream__ = self.stream.SummarizeAndCategorize(title=title,content=content, + baml_options=baml_options) + return __stream__.get_final_response() + else: + # Original non-streaming code + __result__ = self.__options.merge_options(baml_options).call_function_sync(function_name="SummarizeAndCategorize", args={ + "title": title,"content": content, + }) + return typing.cast(types.SummarizeAndCategorizeResult, __result__.cast_to(types, types, stream_types, False, __runtime__)) + + + +class BamlStreamClient: + __options: DoNotUseDirectlyCallManager + + def __init__(self, options: DoNotUseDirectlyCallManager): + self.__options = options + + def CategorizeOnly(self, title: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlSyncStream[stream_types.CategorizeOnlyResult, types.CategorizeOnlyResult]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_sync_stream(function_name="CategorizeOnly", args={ + "title": title, + }) + return baml_py.BamlSyncStream[stream_types.CategorizeOnlyResult, types.CategorizeOnlyResult]( + __result__, + lambda x: typing.cast(stream_types.CategorizeOnlyResult, x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(types.CategorizeOnlyResult, x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def ClassifyKind(self, title: str,url: str,summary: typing.Optional[str] = None, + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlSyncStream[stream_types.ClassifyKindResult, types.ClassifyKindResult]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_sync_stream(function_name="ClassifyKind", args={ + "title": title,"url": url,"summary": summary, + }) + return baml_py.BamlSyncStream[stream_types.ClassifyKindResult, types.ClassifyKindResult]( + __result__, + lambda x: typing.cast(stream_types.ClassifyKindResult, x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(types.ClassifyKindResult, x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def ClassifyProfiles(self, profiles: typing.List["types.ProfileInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlSyncStream[typing.List["stream_types.ProfileVerdict"], typing.List["types.ProfileVerdict"]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_sync_stream(function_name="ClassifyProfiles", args={ + "profiles": profiles, + }) + return baml_py.BamlSyncStream[typing.List["stream_types.ProfileVerdict"], typing.List["types.ProfileVerdict"]]( + __result__, + lambda x: typing.cast(typing.List["stream_types.ProfileVerdict"], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List["types.ProfileVerdict"], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def ExtractLinks(self, emailHtml: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlSyncStream[typing.List["stream_types.ExtractedLink"], typing.List["types.ExtractedLink"]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_sync_stream(function_name="ExtractLinks", args={ + "emailHtml": emailHtml, + }) + return baml_py.BamlSyncStream[typing.List["stream_types.ExtractedLink"], typing.List["types.ExtractedLink"]]( + __result__, + lambda x: typing.cast(typing.List["stream_types.ExtractedLink"], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List["types.ExtractedLink"], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def ExtractProducts(self, newsletterText: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlSyncStream[typing.List["stream_types.NewsletterProduct"], typing.List["types.NewsletterProduct"]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_sync_stream(function_name="ExtractProducts", args={ + "newsletterText": newsletterText, + }) + return baml_py.BamlSyncStream[typing.List["stream_types.NewsletterProduct"], typing.List["types.NewsletterProduct"]]( + __result__, + lambda x: typing.cast(typing.List["stream_types.NewsletterProduct"], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List["types.NewsletterProduct"], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def ImproveTitles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlSyncStream[typing.List["stream_types.TitleFix"], typing.List["types.TitleFix"]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_sync_stream(function_name="ImproveTitles", args={ + "articles": articles, + }) + return baml_py.BamlSyncStream[typing.List["stream_types.TitleFix"], typing.List["types.TitleFix"]]( + __result__, + lambda x: typing.cast(typing.List["stream_types.TitleFix"], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List["types.TitleFix"], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def ScoreArticles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlSyncStream[typing.List["stream_types.ScoredArticle"], typing.List["types.ScoredArticle"]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_sync_stream(function_name="ScoreArticles", args={ + "articles": articles, + }) + return baml_py.BamlSyncStream[typing.List["stream_types.ScoredArticle"], typing.List["types.ScoredArticle"]]( + __result__, + lambda x: typing.cast(typing.List["stream_types.ScoredArticle"], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List["types.ScoredArticle"], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def SelectNotableProducts(self, products: typing.List["types.NewsletterProduct"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlSyncStream[typing.List[int], typing.List[int]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_sync_stream(function_name="SelectNotableProducts", args={ + "products": products, + }) + return baml_py.BamlSyncStream[typing.List[int], typing.List[int]]( + __result__, + lambda x: typing.cast(typing.List[int], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List[int], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def SelectProductLink(self, productName: str,productDescription: str,candidates: typing.List["types.SearchCandidate"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlSyncStream[stream_types.ProductLinkChoice, types.ProductLinkChoice]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_sync_stream(function_name="SelectProductLink", args={ + "productName": productName,"productDescription": productDescription,"candidates": candidates, + }) + return baml_py.BamlSyncStream[stream_types.ProductLinkChoice, types.ProductLinkChoice]( + __result__, + lambda x: typing.cast(stream_types.ProductLinkChoice, x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(types.ProductLinkChoice, x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def SemanticDedup(self, newArticles: typing.List["types.ArticleInput"],existingArticles: typing.List["types.ExistingArticle"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlSyncStream[typing.List["stream_types.DedupMatch"], typing.List["types.DedupMatch"]]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_sync_stream(function_name="SemanticDedup", args={ + "newArticles": newArticles,"existingArticles": existingArticles, + }) + return baml_py.BamlSyncStream[typing.List["stream_types.DedupMatch"], typing.List["types.DedupMatch"]]( + __result__, + lambda x: typing.cast(typing.List["stream_types.DedupMatch"], x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(typing.List["types.DedupMatch"], x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + def SummarizeAndCategorize(self, title: str,content: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.BamlSyncStream[stream_types.SummarizeAndCategorizeResult, types.SummarizeAndCategorizeResult]: + __ctx__, __result__ = self.__options.merge_options(baml_options).create_sync_stream(function_name="SummarizeAndCategorize", args={ + "title": title,"content": content, + }) + return baml_py.BamlSyncStream[stream_types.SummarizeAndCategorizeResult, types.SummarizeAndCategorizeResult]( + __result__, + lambda x: typing.cast(stream_types.SummarizeAndCategorizeResult, x.cast_to(types, types, stream_types, True, __runtime__)), + lambda x: typing.cast(types.SummarizeAndCategorizeResult, x.cast_to(types, types, stream_types, False, __runtime__)), + __ctx__, + ) + + +class BamlHttpRequestClient: + __options: DoNotUseDirectlyCallManager + + def __init__(self, options: DoNotUseDirectlyCallManager): + self.__options = options + + def CategorizeOnly(self, title: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="CategorizeOnly", args={ + "title": title, + }, mode="request") + return __result__ + def ClassifyKind(self, title: str,url: str,summary: typing.Optional[str] = None, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="ClassifyKind", args={ + "title": title,"url": url,"summary": summary, + }, mode="request") + return __result__ + def ClassifyProfiles(self, profiles: typing.List["types.ProfileInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="ClassifyProfiles", args={ + "profiles": profiles, + }, mode="request") + return __result__ + def ExtractLinks(self, emailHtml: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="ExtractLinks", args={ + "emailHtml": emailHtml, + }, mode="request") + return __result__ + def ExtractProducts(self, newsletterText: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="ExtractProducts", args={ + "newsletterText": newsletterText, + }, mode="request") + return __result__ + def ImproveTitles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="ImproveTitles", args={ + "articles": articles, + }, mode="request") + return __result__ + def ScoreArticles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="ScoreArticles", args={ + "articles": articles, + }, mode="request") + return __result__ + def SelectNotableProducts(self, products: typing.List["types.NewsletterProduct"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="SelectNotableProducts", args={ + "products": products, + }, mode="request") + return __result__ + def SelectProductLink(self, productName: str,productDescription: str,candidates: typing.List["types.SearchCandidate"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="SelectProductLink", args={ + "productName": productName,"productDescription": productDescription,"candidates": candidates, + }, mode="request") + return __result__ + def SemanticDedup(self, newArticles: typing.List["types.ArticleInput"],existingArticles: typing.List["types.ExistingArticle"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="SemanticDedup", args={ + "newArticles": newArticles,"existingArticles": existingArticles, + }, mode="request") + return __result__ + def SummarizeAndCategorize(self, title: str,content: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="SummarizeAndCategorize", args={ + "title": title,"content": content, + }, mode="request") + return __result__ + + +class BamlHttpStreamRequestClient: + __options: DoNotUseDirectlyCallManager + + def __init__(self, options: DoNotUseDirectlyCallManager): + self.__options = options + + def CategorizeOnly(self, title: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="CategorizeOnly", args={ + "title": title, + }, mode="stream") + return __result__ + def ClassifyKind(self, title: str,url: str,summary: typing.Optional[str] = None, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="ClassifyKind", args={ + "title": title,"url": url,"summary": summary, + }, mode="stream") + return __result__ + def ClassifyProfiles(self, profiles: typing.List["types.ProfileInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="ClassifyProfiles", args={ + "profiles": profiles, + }, mode="stream") + return __result__ + def ExtractLinks(self, emailHtml: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="ExtractLinks", args={ + "emailHtml": emailHtml, + }, mode="stream") + return __result__ + def ExtractProducts(self, newsletterText: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="ExtractProducts", args={ + "newsletterText": newsletterText, + }, mode="stream") + return __result__ + def ImproveTitles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="ImproveTitles", args={ + "articles": articles, + }, mode="stream") + return __result__ + def ScoreArticles(self, articles: typing.List["types.ArticleInput"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="ScoreArticles", args={ + "articles": articles, + }, mode="stream") + return __result__ + def SelectNotableProducts(self, products: typing.List["types.NewsletterProduct"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="SelectNotableProducts", args={ + "products": products, + }, mode="stream") + return __result__ + def SelectProductLink(self, productName: str,productDescription: str,candidates: typing.List["types.SearchCandidate"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="SelectProductLink", args={ + "productName": productName,"productDescription": productDescription,"candidates": candidates, + }, mode="stream") + return __result__ + def SemanticDedup(self, newArticles: typing.List["types.ArticleInput"],existingArticles: typing.List["types.ExistingArticle"], + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="SemanticDedup", args={ + "newArticles": newArticles,"existingArticles": existingArticles, + }, mode="stream") + return __result__ + def SummarizeAndCategorize(self, title: str,content: str, + baml_options: BamlCallOptions = {}, + ) -> baml_py.baml_py.HTTPRequest: + __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="SummarizeAndCategorize", args={ + "title": title,"content": content, + }, mode="stream") + return __result__ + + +b = BamlSyncClient(DoNotUseDirectlyCallManager({})) \ No newline at end of file diff --git a/backend/baml_client/tracing.py b/backend/baml_client/tracing.py new file mode 100644 index 0000000000..06725593c3 --- /dev/null +++ b/backend/baml_client/tracing.py @@ -0,0 +1,22 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +from .globals import DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX + +trace = DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX.trace_fn +set_tags = DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX.upsert_tags +def flush(): + DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX.flush() +on_log_event = DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX.on_log_event + + +__all__ = ['trace', 'set_tags', "flush", "on_log_event"] diff --git a/backend/baml_client/type_builder.py b/backend/baml_client/type_builder.py new file mode 100644 index 0000000000..e52762479a --- /dev/null +++ b/backend/baml_client/type_builder.py @@ -0,0 +1,844 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +import typing +from baml_py import type_builder +from baml_py import baml_py +# These are exports, not used here, hence the linter is disabled +from baml_py.baml_py import FieldType, EnumValueBuilder, EnumBuilder, ClassBuilder # noqa: F401 # pylint: disable=unused-import +from .globals import DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME + +class TypeBuilder(type_builder.TypeBuilder): + def __init__(self): + super().__init__(classes=set( + ["ArticleInput","CategorizeOnlyResult","ClassifyKindResult","DedupMatch","ExistingArticle","ExtractedLink","NewsletterProduct","ProductLinkChoice","ProfileInput","ProfileVerdict","ScoredArticle","SearchCandidate","SummarizeAndCategorizeResult","TitleFix",] + ), enums=set( + ["ArticleCategory","ArticleKind",] + ), runtime=DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME) + + # ######################################################################### + # Generated enums 2 + # ######################################################################### + + @property + def ArticleCategory(self) -> "ArticleCategoryViewer": + return ArticleCategoryViewer(self) + + @property + def ArticleKind(self) -> "ArticleKindViewer": + return ArticleKindViewer(self) + + + # ######################################################################### + # Generated classes 14 + # ######################################################################### + + @property + def ArticleInput(self) -> "ArticleInputViewer": + return ArticleInputViewer(self) + + @property + def CategorizeOnlyResult(self) -> "CategorizeOnlyResultViewer": + return CategorizeOnlyResultViewer(self) + + @property + def ClassifyKindResult(self) -> "ClassifyKindResultViewer": + return ClassifyKindResultViewer(self) + + @property + def DedupMatch(self) -> "DedupMatchViewer": + return DedupMatchViewer(self) + + @property + def ExistingArticle(self) -> "ExistingArticleViewer": + return ExistingArticleViewer(self) + + @property + def ExtractedLink(self) -> "ExtractedLinkViewer": + return ExtractedLinkViewer(self) + + @property + def NewsletterProduct(self) -> "NewsletterProductViewer": + return NewsletterProductViewer(self) + + @property + def ProductLinkChoice(self) -> "ProductLinkChoiceViewer": + return ProductLinkChoiceViewer(self) + + @property + def ProfileInput(self) -> "ProfileInputViewer": + return ProfileInputViewer(self) + + @property + def ProfileVerdict(self) -> "ProfileVerdictViewer": + return ProfileVerdictViewer(self) + + @property + def ScoredArticle(self) -> "ScoredArticleViewer": + return ScoredArticleViewer(self) + + @property + def SearchCandidate(self) -> "SearchCandidateViewer": + return SearchCandidateViewer(self) + + @property + def SummarizeAndCategorizeResult(self) -> "SummarizeAndCategorizeResultViewer": + return SummarizeAndCategorizeResultViewer(self) + + @property + def TitleFix(self) -> "TitleFixViewer": + return TitleFixViewer(self) + + + +# ######################################################################### +# Generated enums 2 +# ######################################################################### + +class ArticleCategoryAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.enum("ArticleCategory") + self._values: typing.Set[str] = set([ "Models", "Dev", "Research", ]) + self._vals = ArticleCategoryValues(self._bldr, self._values) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def values(self) -> "ArticleCategoryValues": + return self._vals + + +class ArticleCategoryViewer(ArticleCategoryAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_values(self) -> typing.List[typing.Tuple[str, type_builder.EnumValueViewer]]: + return [(name, type_builder.EnumValueViewer(self._bldr.value(name))) for name in self._values] + + +class ArticleCategoryValues: + def __init__(self, enum_bldr: baml_py.EnumBuilder, values: typing.Set[str]): + self.__bldr = enum_bldr + self.__values = values # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def Models(self) -> type_builder.EnumValueViewer: + return type_builder.EnumValueViewer(self.__bldr.value("Models")) + + @property + def Dev(self) -> type_builder.EnumValueViewer: + return type_builder.EnumValueViewer(self.__bldr.value("Dev")) + + @property + def Research(self) -> type_builder.EnumValueViewer: + return type_builder.EnumValueViewer(self.__bldr.value("Research")) + + + + +class ArticleKindAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.enum("ArticleKind") + self._values: typing.Set[str] = set([ "Repo", "Paper", "Model", "Blog", "Product", "Announcement", ]) + self._vals = ArticleKindValues(self._bldr, self._values) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def values(self) -> "ArticleKindValues": + return self._vals + + +class ArticleKindViewer(ArticleKindAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_values(self) -> typing.List[typing.Tuple[str, type_builder.EnumValueViewer]]: + return [(name, type_builder.EnumValueViewer(self._bldr.value(name))) for name in self._values] + + +class ArticleKindValues: + def __init__(self, enum_bldr: baml_py.EnumBuilder, values: typing.Set[str]): + self.__bldr = enum_bldr + self.__values = values # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def Repo(self) -> type_builder.EnumValueViewer: + return type_builder.EnumValueViewer(self.__bldr.value("Repo")) + + @property + def Paper(self) -> type_builder.EnumValueViewer: + return type_builder.EnumValueViewer(self.__bldr.value("Paper")) + + @property + def Model(self) -> type_builder.EnumValueViewer: + return type_builder.EnumValueViewer(self.__bldr.value("Model")) + + @property + def Blog(self) -> type_builder.EnumValueViewer: + return type_builder.EnumValueViewer(self.__bldr.value("Blog")) + + @property + def Product(self) -> type_builder.EnumValueViewer: + return type_builder.EnumValueViewer(self.__bldr.value("Product")) + + @property + def Announcement(self) -> type_builder.EnumValueViewer: + return type_builder.EnumValueViewer(self.__bldr.value("Announcement")) + + + + + +# ######################################################################### +# Generated classes 14 +# ######################################################################### + +class ArticleInputAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("ArticleInput") + self._properties: typing.Set[str] = set([ "url", "title", "source", "snippet", "trust", ]) + self._props = ArticleInputProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "ArticleInputProperties": + return self._props + + +class ArticleInputViewer(ArticleInputAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class ArticleInputProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def url(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("url")) + + @property + def title(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("title")) + + @property + def source(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("source")) + + @property + def snippet(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("snippet")) + + @property + def trust(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("trust")) + + + + +class CategorizeOnlyResultAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("CategorizeOnlyResult") + self._properties: typing.Set[str] = set([ "categories", ]) + self._props = CategorizeOnlyResultProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "CategorizeOnlyResultProperties": + return self._props + + +class CategorizeOnlyResultViewer(CategorizeOnlyResultAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class CategorizeOnlyResultProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def categories(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("categories")) + + + + +class ClassifyKindResultAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("ClassifyKindResult") + self._properties: typing.Set[str] = set([ "kind", ]) + self._props = ClassifyKindResultProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "ClassifyKindResultProperties": + return self._props + + +class ClassifyKindResultViewer(ClassifyKindResultAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class ClassifyKindResultProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def kind(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("kind")) + + + + +class DedupMatchAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("DedupMatch") + self._properties: typing.Set[str] = set([ "url", "existingUrl", ]) + self._props = DedupMatchProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "DedupMatchProperties": + return self._props + + +class DedupMatchViewer(DedupMatchAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class DedupMatchProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def url(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("url")) + + @property + def existingUrl(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("existingUrl")) + + + + +class ExistingArticleAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("ExistingArticle") + self._properties: typing.Set[str] = set([ "url", "title", "source", ]) + self._props = ExistingArticleProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "ExistingArticleProperties": + return self._props + + +class ExistingArticleViewer(ExistingArticleAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class ExistingArticleProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def url(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("url")) + + @property + def title(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("title")) + + @property + def source(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("source")) + + + + +class ExtractedLinkAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("ExtractedLink") + self._properties: typing.Set[str] = set([ "title", "url", "snippet", ]) + self._props = ExtractedLinkProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "ExtractedLinkProperties": + return self._props + + +class ExtractedLinkViewer(ExtractedLinkAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class ExtractedLinkProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def title(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("title")) + + @property + def url(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("url")) + + @property + def snippet(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("snippet")) + + + + +class NewsletterProductAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("NewsletterProduct") + self._properties: typing.Set[str] = set([ "name", "description", ]) + self._props = NewsletterProductProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "NewsletterProductProperties": + return self._props + + +class NewsletterProductViewer(NewsletterProductAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class NewsletterProductProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def name(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("name")) + + @property + def description(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("description")) + + + + +class ProductLinkChoiceAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("ProductLinkChoice") + self._properties: typing.Set[str] = set([ "index", "confidence", ]) + self._props = ProductLinkChoiceProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "ProductLinkChoiceProperties": + return self._props + + +class ProductLinkChoiceViewer(ProductLinkChoiceAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class ProductLinkChoiceProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def index(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("index")) + + @property + def confidence(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("confidence")) + + + + +class ProfileInputAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("ProfileInput") + self._properties: typing.Set[str] = set([ "handle", "name", "description", ]) + self._props = ProfileInputProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "ProfileInputProperties": + return self._props + + +class ProfileInputViewer(ProfileInputAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class ProfileInputProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def handle(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("handle")) + + @property + def name(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("name")) + + @property + def description(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("description")) + + + + +class ProfileVerdictAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("ProfileVerdict") + self._properties: typing.Set[str] = set([ "handle", "isAiRelated", "confidence", ]) + self._props = ProfileVerdictProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "ProfileVerdictProperties": + return self._props + + +class ProfileVerdictViewer(ProfileVerdictAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class ProfileVerdictProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def handle(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("handle")) + + @property + def isAiRelated(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("isAiRelated")) + + @property + def confidence(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("confidence")) + + + + +class ScoredArticleAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("ScoredArticle") + self._properties: typing.Set[str] = set([ "url", "score", ]) + self._props = ScoredArticleProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "ScoredArticleProperties": + return self._props + + +class ScoredArticleViewer(ScoredArticleAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class ScoredArticleProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def url(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("url")) + + @property + def score(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("score")) + + + + +class SearchCandidateAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("SearchCandidate") + self._properties: typing.Set[str] = set([ "title", "url", "snippet", ]) + self._props = SearchCandidateProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "SearchCandidateProperties": + return self._props + + +class SearchCandidateViewer(SearchCandidateAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class SearchCandidateProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def title(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("title")) + + @property + def url(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("url")) + + @property + def snippet(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("snippet")) + + + + +class SummarizeAndCategorizeResultAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("SummarizeAndCategorizeResult") + self._properties: typing.Set[str] = set([ "summary", "categories", "kind", ]) + self._props = SummarizeAndCategorizeResultProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "SummarizeAndCategorizeResultProperties": + return self._props + + +class SummarizeAndCategorizeResultViewer(SummarizeAndCategorizeResultAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class SummarizeAndCategorizeResultProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def summary(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("summary")) + + @property + def categories(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("categories")) + + @property + def kind(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("kind")) + + + + +class TitleFixAst: + def __init__(self, tb: type_builder.TypeBuilder): + _tb = tb._tb # type: ignore (we know how to use this private attribute) + self._bldr = _tb.class_("TitleFix") + self._properties: typing.Set[str] = set([ "url", "title", ]) + self._props = TitleFixProperties(self._bldr, self._properties) + + def type(self) -> baml_py.FieldType: + return self._bldr.field() + + @property + def props(self) -> "TitleFixProperties": + return self._props + + +class TitleFixViewer(TitleFixAst): + def __init__(self, tb: type_builder.TypeBuilder): + super().__init__(tb) + + + def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]: + return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties] + + + +class TitleFixProperties: + def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]): + self.__bldr = bldr + self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821 + + + + @property + def url(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("url")) + + @property + def title(self) -> type_builder.ClassPropertyViewer: + return type_builder.ClassPropertyViewer(self.__bldr.property("title")) + + + diff --git a/backend/baml_client/type_map.py b/backend/baml_client/type_map.py new file mode 100644 index 0000000000..72cf02e092 --- /dev/null +++ b/backend/baml_client/type_map.py @@ -0,0 +1,66 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +from . import types +from . import stream_types + + +type_map = { + + "types.ArticleInput": types.ArticleInput, + "stream_types.ArticleInput": stream_types.ArticleInput, + + "types.CategorizeOnlyResult": types.CategorizeOnlyResult, + "stream_types.CategorizeOnlyResult": stream_types.CategorizeOnlyResult, + + "types.ClassifyKindResult": types.ClassifyKindResult, + "stream_types.ClassifyKindResult": stream_types.ClassifyKindResult, + + "types.DedupMatch": types.DedupMatch, + "stream_types.DedupMatch": stream_types.DedupMatch, + + "types.ExistingArticle": types.ExistingArticle, + "stream_types.ExistingArticle": stream_types.ExistingArticle, + + "types.ExtractedLink": types.ExtractedLink, + "stream_types.ExtractedLink": stream_types.ExtractedLink, + + "types.NewsletterProduct": types.NewsletterProduct, + "stream_types.NewsletterProduct": stream_types.NewsletterProduct, + + "types.ProductLinkChoice": types.ProductLinkChoice, + "stream_types.ProductLinkChoice": stream_types.ProductLinkChoice, + + "types.ProfileInput": types.ProfileInput, + "stream_types.ProfileInput": stream_types.ProfileInput, + + "types.ProfileVerdict": types.ProfileVerdict, + "stream_types.ProfileVerdict": stream_types.ProfileVerdict, + + "types.ScoredArticle": types.ScoredArticle, + "stream_types.ScoredArticle": stream_types.ScoredArticle, + + "types.SearchCandidate": types.SearchCandidate, + "stream_types.SearchCandidate": stream_types.SearchCandidate, + + "types.SummarizeAndCategorizeResult": types.SummarizeAndCategorizeResult, + "stream_types.SummarizeAndCategorizeResult": stream_types.SummarizeAndCategorizeResult, + + "types.TitleFix": types.TitleFix, + "stream_types.TitleFix": stream_types.TitleFix, + + + "types.ArticleCategory": types.ArticleCategory, + + "types.ArticleKind": types.ArticleKind, + +} \ No newline at end of file diff --git a/backend/baml_client/types.py b/backend/baml_client/types.py new file mode 100644 index 0000000000..2886b228b7 --- /dev/null +++ b/backend/baml_client/types.py @@ -0,0 +1,125 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +import typing +import typing_extensions +from enum import Enum + + +from pydantic import BaseModel, ConfigDict, Field + + +import baml_py + +CheckT = typing_extensions.TypeVar('CheckT') +CheckName = typing_extensions.TypeVar('CheckName', bound=str) + +class Check(BaseModel): + name: str + expression: str + status: str +class Checked(BaseModel, typing.Generic[CheckT, CheckName]): + value: CheckT + checks: typing.Dict[CheckName, Check] + +def get_checks(checks: typing.Dict[CheckName, Check]) -> typing.List[Check]: + return list(checks.values()) + +def all_succeeded(checks: typing.Dict[CheckName, Check]) -> bool: + return all(check.status == "succeeded" for check in get_checks(checks)) +# ######################################################################### +# Generated enums (2) +# ######################################################################### + +class ArticleCategory(str, Enum): + Models = "Models" + Dev = "Dev" + Research = "Research" + +class ArticleKind(str, Enum): + Repo = "Repo" + Paper = "Paper" + Model = "Model" + Blog = "Blog" + Product = "Product" + Announcement = "Announcement" + +# ######################################################################### +# Generated classes (14) +# ######################################################################### + +class ArticleInput(BaseModel): + url: str + title: str + source: str + snippet: typing.Optional[str] = Field(default=None, description='First ~200 chars of the article content, if any') + trust: typing.Optional[str] = Field(default=None, description='"high" | "medium" | "low" - source trust tag') + +class CategorizeOnlyResult(BaseModel): + categories: typing.List[ArticleCategory] = Field(description='1-2 categories') + +class ClassifyKindResult(BaseModel): + kind: ArticleKind + +class DedupMatch(BaseModel): + url: str = Field(description='URL of the new article') + existingUrl: str = Field(description='URL of the existing article it duplicates') + +class ExistingArticle(BaseModel): + url: str + title: str + source: str + +class ExtractedLink(BaseModel): + title: str + url: str + snippet: str = Field(description='One-sentence description of the article') + +class NewsletterProduct(BaseModel): + name: str = Field(description='Exact product / tool / model / launch name') + description: str = Field(description='Concise phrase: what it is and does, usable verbatim as a web search query') + +class ProductLinkChoice(BaseModel): + index: int = Field(description='1-based index of the best first-party result, or 0 if none qualifies') + confidence: float = Field(description='0.0 - 1.0') + +class ProfileInput(BaseModel): + handle: str = Field(description='Substack owner handle, echoed back in the verdict') + name: str = Field(description='Publication name') + description: typing.Optional[str] = Field(default=None, description='Publication hero text and/or author bio, if known') + +class ProfileVerdict(BaseModel): + handle: str = Field(description='Must exactly match the input handle') + isAiRelated: bool + confidence: int = Field(description='0-100 confidence that AI/ML/LLMs is the PRIMARY focus') + +class ScoredArticle(BaseModel): + url: str + score: int = Field(description='1-100 developer-actionability rating') + +class SearchCandidate(BaseModel): + title: str + url: str + snippet: str + +class SummarizeAndCategorizeResult(BaseModel): + summary: str = Field(description='2-3 short factual lines separated by \\n') + categories: typing.List[ArticleCategory] = Field(description='1-2 categories') + kind: ArticleKind + +class TitleFix(BaseModel): + url: str = Field(description='The exact URL from the matching input article - copy verbatim') + title: str = Field(description='Plain title text only. No surrounding quotes. No leading source name or bracket tag. Max 12 words. Return the input title verbatim if it is already clear, specific, and informative.') + +# ######################################################################### +# Generated type aliases (0) +# ######################################################################### diff --git a/backend/baml_client/watchers.py b/backend/baml_client/watchers.py new file mode 100644 index 0000000000..347146f66a --- /dev/null +++ b/backend/baml_client/watchers.py @@ -0,0 +1,44 @@ +# ---------------------------------------------------------------------------- +# +# Welcome to Baml! To use this generated code, please run the following: +# +# $ pip install baml +# +# ---------------------------------------------------------------------------- + +# This file was generated by BAML: please do not edit it. Instead, edit the +# BAML files and re-generate this code using: baml-cli generate +# baml-cli is available with the baml package. + +from typing import Callable, Any, Protocol, Generic, TypeVar, overload, Literal +import threading + +T = TypeVar("T") + +class BlockEvent: + def __init__(self, block_label: str, event_type: str): + self.block_label = block_label + self.event_type = event_type # "enter" | "exit" + +class VarEvent(Generic[T]): + def __init__(self, variable_name: str, value: T, timestamp: str, function_name: str): + self.variable_name = variable_name + self.value = value + self.timestamp = timestamp + self.function_name = function_name + +BlockHandler = Callable[[BlockEvent], None] +VarEventHandler = Callable[[VarEvent[T]], None] +StreamHandler = Callable[[Any], None] # Stream will be an async iterator + +class InternalEventBindings(Protocol): + function_name: str + block: list[BlockHandler] + vars: dict[str, list[VarEventHandler[Any]]] + streams: dict[str, list[StreamHandler]] + functions: dict[str, "InternalEventBindings"] + +class EventCollectorInternal(Protocol): + def __handlers__(self) -> InternalEventBindings: + ... + diff --git a/backend/pipeline/__init__.py b/backend/pipeline/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/pipeline/crontab b/backend/pipeline/crontab new file mode 100644 index 0000000000..96abab5777 --- /dev/null +++ b/backend/pipeline/crontab @@ -0,0 +1 @@ +0 4 * * * cd /app/backend && /app/.venv/bin/python -m pipeline.run diff --git a/backend/pipeline/run.py b/backend/pipeline/run.py new file mode 100644 index 0000000000..7df42e792c --- /dev/null +++ b/backend/pipeline/run.py @@ -0,0 +1,461 @@ +"""Pipeline entry point. Run with: python -m pipeline.run""" + +from __future__ import annotations + +import os +import sys +import time +from datetime import UTC, datetime, timedelta + +from model2vec import StaticModel +from sqlmodel import Session, create_engine, select + +from app.models_agentique import Article, ScoredUrl +from baml_client.sync_client import b +from baml_client.types import ArticleInput, ExistingArticle +from pipeline.sources.ainews import fetch_ai_news +from pipeline.sources.email import fetch_newsletter +from pipeline.sources.extract_content import re_extract_full_content +from pipeline.sources.hn import fetch_hn +from pipeline.sources.substack import fetch_substack +from pipeline.steps import ( + PROMPT_CONTENT_CAP, + SCORE_THRESHOLD, + TRUST_BY_SOURCE, + github_repo_from_content, + kind_from_url, +) +from pipeline.utils import log, sanitize_llm_text, strip_title_wrappers, wait_ms + + +def _build_db_url() -> str: + server = os.environ["POSTGRES_SERVER"] + port = os.environ.get("POSTGRES_PORT", "5432") + user = os.environ["POSTGRES_USER"] + password = os.environ.get("POSTGRES_PASSWORD", "") + db = os.environ.get("POSTGRES_DB", "") + return f"postgresql+psycopg://{user}:{password}@{server}:{port}/{db}" + + +_engine = create_engine(_build_db_url()) + +_model: StaticModel | None = None + + +def _get_model() -> StaticModel: + global _model + if _model is None: + _model = StaticModel.from_pretrained("minishlab/potion-base-8M") + return _model + + +def _embed(text: str) -> list[float]: + return _get_model().encode([text])[0].tolist() + + +SOURCES = [ + {"label": "Hacker News", "fetcher": fetch_hn}, + {"label": "Newsletter", "fetcher": fetch_newsletter}, + {"label": "AI News", "fetcher": fetch_ai_news}, + {"label": "Substack", "fetcher": fetch_substack}, +] + + +def _to_baml_input(a: dict) -> ArticleInput: + """Build a BAML ArticleInput from a fetched-article dict (snippet capped at 200).""" + return ArticleInput( + url=a["url"], + title=a["title"], + source=a["source"], + snippet=a.get("content", "")[:200] if a.get("content") else None, + trust=TRUST_BY_SOURCE.get(a["source"]), + ) + + +def run_pipeline() -> None: + log("=== Pipeline start ===") + start = time.time() + + with Session(_engine) as session: + for src in SOURCES: + log(f"\n=== Processing {src['label']} ===") + + fetched = _fetch_source(src["fetcher"], src["label"]) + fresh = _filter_known_urls(session, fetched, src["label"]) + alive = _filter_dead_domains(fresh, src["label"]) + unique = _dedup_semantic(session, alive, src["label"]) + scored = _score_articles(session, unique) + inserted = _insert_articles(session, scored) + _improve_titles(session, inserted) + with_content = _extract_full_content(session, inserted) + processed = _summarize_and_categorize(session, with_content) + _embed_articles(session, processed) + + elapsed = f"{time.time() - start:.1f}" + log(f"=== Pipeline complete in {elapsed}s ===") + + +# ─── Step 01 ──────────────────────────────────────────────────────────────── + + +def _fetch_source(fetcher, label: str) -> list[dict]: + articles = fetcher() + if not articles: + log(f"No articles from {label}") + return articles + + +# ─── Step 02 ──────────────────────────────────────────────────────────────── + + +def _filter_known_urls( + session: Session, articles: list[dict], label: str +) -> list[dict]: + if not articles: + return [] + all_urls = [a["url"] for a in articles] + + existing_urls = set( + session.exec(select(Article.url).where(Article.url.in_(all_urls))).all() + ) + scored_urls = set( + session.exec(select(ScoredUrl.url).where(ScoredUrl.url.in_(all_urls))).all() + ) + + fresh = [ + a + for a in articles + if a["url"] not in existing_urls and a["url"] not in scored_urls + ] + + if existing_urls or scored_urls: + log( + f" Filtered {len(existing_urls)} known + {len(scored_urls)} already-scored URLs" + ) + if not fresh: + log(f" No new articles from {label}") + return [] + log(f" {len(fresh)} new articles to process") + return fresh + + +# ─── Step 02b ─────────────────────────────────────────────────────────────── + + +def _filter_dead_domains(articles: list[dict], label: str) -> list[dict]: + if not articles: + return articles + + from urllib.parse import urlparse + + import dns.resolver + + def is_resolvable(url: str) -> bool: + try: + host = urlparse(url).hostname + if not host: + return False + dns.resolver.resolve(host, "A") + return True + except dns.resolver.NXDOMAIN: + return False + except Exception: + return True # Other errors (timeout, etc.) - assume alive + + results = [(a, is_resolvable(a["url"])) for a in articles] + alive = [a for a, ok in results if ok] + dead = [a for a, ok in results if not ok] + for a in dead: + log(f" Dropped dead URL: {a['url']}") + if dead: + log(f" Filtered {len(dead)} dead-domain URLs from {label}") + return alive + + +# ─── Step 03 ──────────────────────────────────────────────────────────────── + + +def _dedup_semantic(session: Session, articles: list[dict], label: str) -> list[dict]: + if not articles: + return articles + + cutoff = datetime.now(UTC) - timedelta(days=14) + recent_db = session.exec( + select(Article).where(Article.published_at >= cutoff) + ).all() + + if not recent_db: + return articles + + log(f" Deduplicating against {len(recent_db)} recent DB articles...") + + new_inputs = [_to_baml_input(a) for a in articles] + existing_inputs = [ + ExistingArticle(url=str(art.url), title=art.title, source=art.source) + for art in recent_db + ] + + try: + matches = b.SemanticDedup(new_inputs, existing_inputs) + merged_urls = {m.url for m in matches} + for m in matches: + log(f" Dropped duplicate: {m.url} (matches existing: {m.existingUrl})") + if merged_urls: + log(f" {len(merged_urls)} articles dropped as duplicates") + unique = [a for a in articles if a["url"] not in merged_urls] + if not unique: + log(f" No new unique articles from {label}") + return unique + except Exception as e: + log(f" Dedup failed, continuing without: {e}") + return articles + + +# ─── Step 04 ──────────────────────────────────────────────────────────────── + + +def _score_articles(session: Session, articles: list[dict]) -> list[dict]: + if not articles: + return [] + + log(f" Scoring {len(articles)} articles in batches of 5...") + + BATCH = 5 + all_scores: list[dict] = [] + for i in range(0, len(articles), BATCH): + batch_inputs = [_to_baml_input(a) for a in articles[i : i + BATCH]] + result = b.ScoreArticles(batch_inputs) + all_scores.extend({"url": r.url, "score": r.score} for r in result) + log(f" batch {i // BATCH + 1}/{(len(articles) + BATCH - 1) // BATCH} done") + wait_ms(1000) + + score_by_url = {s["url"]: s["score"] for s in all_scores} + scored = [] + for a in articles: + score = score_by_url.get(a["url"], 0) + if a["source"] == "Ben's Bites": + score = min(score + 10, 100) + scored.append({**a, "score": score}) + + scored.sort(key=lambda x: x["score"], reverse=True) + kept = [s for s in scored if s["score"] >= SCORE_THRESHOLD] + log(f" {len(kept)} articles pass scoring (threshold: {SCORE_THRESHOLD})") + + # Record ALL evaluated URLs (including sub-threshold) + for a in articles: + session.merge(ScoredUrl(url=a["url"])) + session.commit() + + return kept + + +# ─── Step 05 ──────────────────────────────────────────────────────────────── + + +def _insert_articles(session: Session, scored: list[dict]) -> list[dict]: + if not scored: + return [] + + by_url: dict[str, dict] = {} + for item in scored: + existing = by_url.get(item["url"]) + if not existing or item["score"] > existing["score"]: + by_url[item["url"]] = item + + inserted = [] + for item in by_url.values(): + item["title"] = sanitize_llm_text(item["title"]) + pub_at = None + if item.get("published_date"): + try: + from email.utils import parsedate_to_datetime + + try: + pub_at = parsedate_to_datetime(item["published_date"]) + except Exception: + pub_at = datetime.fromisoformat( + item["published_date"].replace("Z", "+00:00") + ) + except Exception: + pass + + article = Article( + title=item["title"], + source=item["source"], + source_type=item["source_type"], + url=item["url"], + published_at=pub_at, + score=item["score"], + content=item.get("content") or "", + ) + session.add(article) + session.flush() + assert article.id is not None + log(f" Inserted #{article.id}: [{item['score']}/100] {item['title']}") + inserted.append({**item, "id": article.id}) + + session.commit() + return inserted + + +# ─── Step 06 ──────────────────────────────────────────────────────────────── + + +def _improve_titles(session: Session, inserted: list[dict]) -> None: + if not inserted: + return + + log(f" Improving {len(inserted)} titles...") + + for item in inserted: + art_id = item["id"] + try: + fixes = b.ImproveTitles([_to_baml_input(item)]) + raw = fixes[0].title if fixes else None + if not raw: + continue + sanitized = sanitize_llm_text(strip_title_wrappers(raw)) + if not sanitized or sanitized == item["title"]: + continue + if item["source"].lower() in sanitized.lower(): + log(f' Skip rewrite #{art_id} (source name leaked): "{sanitized}"') + continue + old_title = item["title"] + article = session.get(Article, art_id) + if article: + article.title = sanitized + session.add(article) + session.commit() + item["title"] = sanitized + log(f' Improved title #{art_id}: "{old_title}" → "{sanitized}"') + except Exception as e: + log(f" Title improve failed for #{art_id}, continuing: {e}") + + +# ─── Step 07 ──────────────────────────────────────────────────────────────── + + +def _extract_full_content(session: Session, inserted: list[dict]) -> list[dict]: + if not inserted: + return [] + + to_extract = [a for a in inserted if a["source_type"] not in ("aiNews", "rss")] + content_map = re_extract_full_content([{"url": a["url"]} for a in to_extract]) + + for item in to_extract: + full = content_map.get(item["url"]) + if full: + article = session.get(Article, item["id"]) + if article: + article.content = full + session.add(article) + session.commit() + + result = [] + for item in inserted: + if item["source_type"] in ("aiNews", "rss"): + result.append({**item, "full_content": item.get("content", "")}) + else: + result.append({**item, "full_content": content_map.get(item["url"], "")}) + return result + + +# ─── Step 08 ──────────────────────────────────────────────────────────────── + + +def _summarize_and_categorize(session: Session, items: list[dict]) -> list[dict]: + if not items: + return [] + + log(f" Summarizing and categorizing {len(items)} articles...") + processed = [] + + for item in items: + art_id = item["id"] + full_content = item.get("full_content", "") + summary = "" + categories: list[str] = [] + kind: str | None = kind_from_url(item["url"]) + + try: + if full_content: + result = b.SummarizeAndCategorize( + item["title"], full_content[:PROMPT_CONTENT_CAP] + ) + summary = sanitize_llm_text(result.summary or "") + categories = [c.lower() for c in result.categories] + if not kind: + kind = result.kind.value.lower() + if kind == "blog" and github_repo_from_content(full_content): + kind = "repo" + else: + result = b.CategorizeOnly(item["title"]) + categories = [c.lower() for c in result.categories] + if not kind: + kr = b.ClassifyKind(item["title"], item["url"], None) + kind = kr.kind.value.lower() + except Exception as e: + log(f" Summarize/categorize failed for #{art_id}: {e}") + + article = session.get(Article, art_id) + if article: + article.summary = summary + article.categories = categories + if kind: + article.kind = kind + session.add(article) + + processed.append( + { + "id": art_id, + "url": item["url"], + "title": item["title"], + "score": item["score"], + "summary": summary, + "categories": categories, + } + ) + + session.commit() + log(" Done summarizing and categorizing") + return processed + + +# ─── Step 09 ──────────────────────────────────────────────────────────────── + + +def _embed_articles(session: Session, items: list[dict]) -> None: + if not items: + return + + log(f" Embedding {len(items)} articles...") + + for item in items: + art_id = item["id"] + try: + text = ( + f"{item['title']}\n\n{item['summary']}" + if item.get("summary") + else item["title"] + ) + vec = _embed(text) + article = session.get(Article, art_id) + if article: + article.embedding = vec + session.add(article) + except Exception as e: + log(f" Embed failed for #{art_id}: {e}") + + session.commit() + + +# ─── CLI ──────────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + try: + run_pipeline() + sys.exit(0) + except Exception as e: + print(f"Pipeline failed: {e}", file=sys.stderr) + raise + sys.exit(1) diff --git a/backend/pipeline/sources/__init__.py b/backend/pipeline/sources/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/pipeline/sources/ainews.py b/backend/pipeline/sources/ainews.py new file mode 100644 index 0000000000..91e845902e --- /dev/null +++ b/backend/pipeline/sources/ainews.py @@ -0,0 +1,264 @@ +"""AI News source - extracts sub-stories from the news.smol.ai RSS feed.""" + +from __future__ import annotations + +import re +from html import unescape + +from pipeline.sources.utils import fetch_with_timeout, is_within_window +from pipeline.utils import log + +FEED_URL = "https://news.smol.ai/rss.xml" +MAX_CONTENT_LENGTH = 1400 +MIN_REDDIT_BODY_LENGTH = 120 +MIN_TWITTER_TITLE_LENGTH = 10 + + +# --- minimal RSS parsing (no feedparser dep for this source) --- + + +def _pick_tag(block: str, tag: str) -> str: + match = re.search(rf"<{tag}>([\s\S]*?)", block) + if not match: + return "" + val = match.group(1) + cdata = re.match(r"^$", val) + return cdata.group(1) if cdata else val + + +def _parse_feed(xml: str) -> list[dict]: + items = [] + for chunk in xml.split("")[1:]: + body = chunk.split("")[0] + items.append( + { + "title": _pick_tag(body, "title"), + "link": _pick_tag(body, "link"), + "content_html": unescape(_pick_tag(body, "content:encoded")), + "pub_date": _pick_tag(body, "pubDate"), + } + ) + return items + + +# --- HTML helpers --- + + +def _strip_tags(html: str) -> str: + return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html)).strip() + + +def _first_bold(html: str) -> str: + m = re.search(r"]*>([\s\S]*?)", html) + return _strip_tags(m.group(1)) if m else "" + + +def _host_of(url: str) -> str: + try: + from urllib.parse import urlparse + + return urlparse(url).hostname.removeprefix("www.") + except Exception: + return "" + + +def _collect_links(html: str) -> list[dict]: + return [ + { + "url": m.group(1), + "host": _host_of(m.group(1)), + "text": _strip_tags(m.group(2)), + } + for m in re.finditer(r']+href="([^"]+)"[^>]*>([\s\S]*?)', html) + ] + + +def _slice_section(html: str, heading_rx: re.Pattern) -> str: + start = heading_rx.search(html) + if not start: + return "" + after = html[start.end() :] + next_h1 = re.search(r"]*>", after, re.IGNORECASE) + return after[: next_h1.start()] if next_h1 else after + + +BLOG_HOST_RX = re.compile( + r"\b(openai|anthropic|deepmind|google|meta|microsoft|nvidia|mistral|unsloth|" + r"databricks|cohere|stability|perplexity|vercel|modal|together|replicate|groq)\b" +) + +PRIMARY_PRIORITY = ["github", "huggingface", "arxiv", "blog", "reddit", "tweet"] + +_JUNK_HOSTS = { + "news.smol.ai", + "latent.space", + "support.substack.com", + "i.redd.it", + "preview.redd.it", +} + + +def _classify_host(host: str) -> str | None: + if host in ("github.com",) or host.endswith(".github.io"): + return "github" + if host in ("huggingface.co", "hf.co"): + return "huggingface" + if host in ("arxiv.org", "ar5iv.labs.arxiv.org"): + return "arxiv" + if ( + BLOG_HOST_RX.search(host) + or host.endswith(".ai") + or host.endswith(".dev") + or "substack.com" in host + ): + return "blog" + if host in ("reddit.com",) or host.endswith(".reddit.com"): + return "reddit" + if host in ("x.com", "twitter.com"): + return "tweet" + return None + + +def _is_junk(link: dict) -> bool: + if not link["url"]: + return True + if link["host"] in _JUNK_HOSTS: + return True + if "twitter.com/i/lists/" in link["url"] or "x.com/i/" in link["url"]: + return True + return False + + +def _pick_primary(links: list[dict]) -> dict: + clean = [lnk for lnk in links if not _is_junk(lnk)] + if not clean: + return {"url": "", "kind": "other"} + for kind in PRIMARY_PRIORITY: + hit = next((lnk for lnk in clean if _classify_host(lnk["host"]) == kind), None) + if hit: + return {"url": hit["url"], "kind": kind} + return {"url": clean[0]["url"], "kind": "other"} + + +# --- Twitter Recap --- + + +def _extract_twitter_recap(html: str) -> dict | None: + region = _slice_section( + html, re.compile(r"]*>\s*AI Twitter Recap\s*", re.IGNORECASE) + ) + if not region: + return None + title = _first_bold(region) + if not title or len(title) < MIN_TWITTER_TITLE_LENGTH: + return None + primary = _pick_primary(_collect_links(region)) + if not primary["url"]: + return None + bullets = [ + _strip_tags(m.group(1)) for m in re.finditer(r"
  • ([\s\S]*?)
    • ", region) + ][:3] + content = " ".join(bullets) or _strip_tags(region) + return { + "title": title, + "url": primary["url"], + "content": content[:MAX_CONTENT_LENGTH], + } + + +# --- Reddit Recap --- + +REDDIT_ANCHOR_RX = re.compile( + r']+href="(https?://(?:www\.)?reddit\.com/r/[^"]+/comments/[^"]+)"[^>]*>([\s\S]*?)' +) + + +def _extract_reddit_recap(html: str) -> list[dict]: + region = _slice_section( + html, re.compile(r"]*>\s*AI Reddit Recap\s*", re.IGNORECASE) + ) + if not region: + return [] + + anchors = [ + { + "url": m.group(1), + "title": _strip_tags(m.group(2)), + "start": m.start(), + "end": m.end(), + } + for m in REDDIT_ANCHOR_RX.finditer(region) + ] + + stories = [] + for i, anchor in enumerate(anchors): + next_start = anchors[i + 1]["start"] if i + 1 < len(anchors) else len(region) + after = region[anchor["end"] :] + heading_rel = re.search(r"]*>", after, re.IGNORECASE) + heading_abs = ( + anchor["end"] + heading_rel.start() if heading_rel else len(region) + ) + body_end = min(next_start, heading_abs) + body = region[anchor["start"] : body_end] + content = _strip_tags(body) + if len(content) < MIN_REDDIT_BODY_LENGTH: + continue + primary = _pick_primary(_collect_links(body)) + stories.append( + { + "title": anchor["title"] or "(untitled)", + "url": primary["url"] or anchor["url"], + "content": content[:MAX_CONTENT_LENGTH], + } + ) + return stories + + +# --- fetch --- + + +def fetch_ai_news() -> list[dict]: + log("Fetching AI News feed...") + try: + resp = fetch_with_timeout(FEED_URL) + if not resp.is_success: + raise ValueError(f"Status {resp.status_code}") + xml = resp.text + except Exception as e: + log(f" AI News fetch FAILED: {e}") + return [] + + items = [it for it in _parse_feed(xml) if is_within_window(it["pub_date"])] + log(f" AI News: {len(items)} issues within window") + + articles: list[dict] = [] + for item in items: + if re.match(r"^not much happened", item["title"], re.IGNORECASE): + continue + twitter = _extract_twitter_recap(item["content_html"]) + if twitter: + articles.append( + { + **twitter, + "published_date": item["pub_date"], + "source": "AI News", + "source_type": "aiNews", + } + ) + for reddit in _extract_reddit_recap(item["content_html"]): + articles.append( + { + **reddit, + "published_date": item["pub_date"], + "source": "AI News", + "source_type": "aiNews", + } + ) + + by_url: dict[str, dict] = {} + for a in articles: + if a["url"] not in by_url: + by_url[a["url"]] = a + + log(f"AI News: {len(by_url)} articles extracted") + return list(by_url.values()) diff --git a/backend/pipeline/sources/email.py b/backend/pipeline/sources/email.py new file mode 100644 index 0000000000..386702441d --- /dev/null +++ b/backend/pipeline/sources/email.py @@ -0,0 +1,290 @@ +"""Newsletter (IMAP) pipeline source. + +Reads unread emails from known newsletter senders in the "sub" mailbox, +extracts named products via BAML, and resolves each to a canonical URL via +web search. Newsletter links are tracking redirects, not trustworthy hrefs, +so product identity comes from the email text and the URL is rediscovered. +""" + +from __future__ import annotations + +import os +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import UTC, datetime +from html import unescape +from urllib.parse import urlparse + +from imap_tools import AND, MailBox, MailMessageFlags + +from baml_client.sync_client import b +from baml_client.types import NewsletterProduct, SearchCandidate +from pipeline.sources.utils import tavily_search +from pipeline.utils import log + +IMAP_PORT_DEFAULT = 993 +IMAP_FOLDER = "sub" +RESOLVE_CONCURRENCY = 5 + +# sender -> display name. "@x" matches any address ending in x; otherwise exact match. +# Senders with a working RSS feed have moved to pipeline/sources/rss.py (or, for +# Ben's Bites, are already covered by substack-sources.json) - IMAP stays as the +# fallback for newsletters that don't publish one. +NEWSLETTER_SOURCES: list[tuple[str, str]] = [ + ("@deeperlearning.producthunt.com", "The Frontier by Product Hunt"), + ("@changelog.com", "Changelog News"), + ("@deeplearning.ai", "The Batch"), + ("@pointer.io", "Pointer"), + ("@mail.theresanaiforthat.com", "There's An AI For That"), + ("@technews.therundown.ai", "The Rundown AI Tech"), + ("@mail.joinsuperhuman.ai", "Superhuman"), + ("agentai@mail.beehiiv.com", "AgentAI"), +] + +# Never a product's first-party source: social posts, video, aggregators/content farms. +DENY_DOMAINS = [ + "x.com", + "twitter.com", + "t.co", + "linkedin.com", + "reddit.com", + "threads.net", + "mastodon.social", + "facebook.com", + "instagram.com", + "youtube.com", + "youtu.be", + "digg.com", + "medium.com", + "news.ycombinator.com", +] + +_STYLE_RE = re.compile(r"]*>[\s\S]*?", re.IGNORECASE) +_SCRIPT_RE = re.compile(r"]*>[\s\S]*?", re.IGNORECASE) +_ANCHOR_RE = re.compile(r']*href="([^"]*)"[^>]*>([\s\S]*?)', re.IGNORECASE) +_TAG_RE = re.compile(r"<[^>]+>") +_WS_RE = re.compile(r"\s+") + + +def _match_sender(address: str) -> str | None: + addr = address.lower() + for pattern, name in NEWSLETTER_SOURCES: + p = pattern.lower() + if addr.endswith(p) if p.startswith("@") else addr == p: + return name + return None + + +def _html_to_text(html: str) -> str: + """Strip an email's HTML to text, keeping anchor text with hrefs as a weak + hint. Hrefs are usually tracking redirects, so product identity comes from + the surrounding words, not the link. + """ + cleaned = _STYLE_RE.sub("", html) + cleaned = _SCRIPT_RE.sub("", cleaned) + + def _anchor(m: re.Match[str]) -> str: + text = _TAG_RE.sub("", m.group(2)).strip() + return f"[{text}]({m.group(1)})" if text else "" + + cleaned = _ANCHOR_RE.sub(_anchor, cleaned) + cleaned = _TAG_RE.sub(" ", cleaned) + cleaned = unescape(cleaned) + cleaned = _WS_RE.sub(" ", cleaned).strip() + return cleaned[:50_000] + + +def _extract_products(html: str, newsletter_name: str, email_date: str) -> list[dict]: + text = _html_to_text(html) + if not text: + return [] + try: + products = b.ExtractProducts(text) + except Exception as e: + log(f" Failed to extract products from {newsletter_name}: {e}") + return [] + return [ + { + "name": p.name.strip(), + "description": (p.description or "").strip(), + "email_date": email_date, + "newsletter_name": newsletter_name, + } + for p in products + if p.name and p.name.strip() + ] + + +def _is_denied(url: str) -> bool: + try: + host = (urlparse(url).hostname or "").removeprefix("www.").lower() + except Exception: + return True + if not host: + return True + return any(host == d or host.endswith(f".{d}") for d in DENY_DOMAINS) + + +def _resolve_one(product: dict) -> dict | None: + """One product -> one canonical URL, or None if nothing is a clean first-party source.""" + name = product["name"] + description = product["description"] + query = f"{name} {description}".strip() + + try: + results = tavily_search(query) + except Exception as e: + log(f' Search failed for "{name}": {e}') + return None + + results = [r for r in results if not _is_denied(r["url"])] + if not results: + log(f' No usable search result for "{name}"') + return None + + try: + choice = b.SelectProductLink( + name, + description, + [ + SearchCandidate(title=r["title"], url=r["url"], snippet=r["description"]) + for r in results + ], + ) + except Exception as e: + log(f' Link selection failed for "{name}": {e}') + return None + + # index is 1-based; 0 means "no candidate is a clean first-party source". + picked = results[choice.index - 1] if choice.index >= 1 else None + if not picked: + log(f' Dropped "{name}": no first-party result among candidates') + return None + + return { + "title": name, + "url": picked["url"], + "content": description, + "published_date": product["email_date"], + "source": product["newsletter_name"], + "source_type": "newsletter", + } + + +def _resolve_products(raw: list[dict]) -> list[dict]: + """Dedupe products by name across all issues in this run, drop the + un-notable ones, then resolve each survivor to a URL. + """ + if not raw: + return [] + + by_name: dict[str, dict] = {} + for p in raw: + by_name.setdefault(p["name"].lower(), p) + unique = list(by_name.values()) + + products = unique + try: + keep = b.SelectNotableProducts( + [NewsletterProduct(name=p["name"], description=p["description"]) for p in unique] + ) + keep_idx = set(keep) + selected = [p for i, p in enumerate(unique) if (i + 1) in keep_idx] + if selected: # guard against a degenerate empty response dropping everything + products = selected + except Exception as e: + log(f" Notability filter failed, resolving all: {e}") + + log( + f" Resolving {len(products)} notable products via search " + f"({len(raw)} raw -> {len(unique)} unique -> {len(products)} notable)" + ) + + articles: list[dict] = [] + with ThreadPoolExecutor(max_workers=RESOLVE_CONCURRENCY) as ex: + futures = [ex.submit(_resolve_one, p) for p in products] + for f in as_completed(futures): + article = f.result() + if article: + articles.append(article) + + log(f" Resolved {len(articles)}/{len(products)} products to URLs") + return articles + + +def _imap_config() -> dict: + host = os.environ.get("IMAP_HOST") + user = os.environ.get("IMAP_USER") + password = os.environ.get("IMAP_PASSWORD") + if not host or not user or not password: + raise RuntimeError("Missing IMAP env vars: IMAP_HOST, IMAP_USER, IMAP_PASSWORD") + port = int(os.environ.get("IMAP_PORT", IMAP_PORT_DEFAULT)) + return {"host": host, "port": port, "user": user, "password": password} + + +def _run_imap_fetch(config: dict) -> list[dict]: + """One connect -> fetch -> mark-seen -> logout cycle. Two passes: headers + only to find matches (avoids downloading full bodies of unrelated mail), + then full source for matched UIDs only. + """ + raw: list[dict] = [] + + with MailBox(config["host"], port=config["port"]).login( + config["user"], config["password"], initial_folder=IMAP_FOLDER + ) as mb: + matches: dict[str, str] = {} + for msg in mb.fetch(AND(seen=False), mark_seen=False, headers_only=True): + name = _match_sender(msg.from_ or "") + if name: + matches[msg.uid] = name + + log(f" Found {len(matches)} unread newsletter emails") + if not matches: + return raw + + processed_uids: list[str] = [] + for msg in mb.fetch( + AND(uid=",".join(matches.keys())), mark_seen=False, headers_only=False + ): + name = matches.get(msg.uid) + if not name: + continue + + subject = msg.subject or "(no subject)" + log(f" Processing: {name} ({subject})") + + html = msg.html or msg.text or "" + if not html: + log(" No HTML content, skipping") + processed_uids.append(msg.uid) + continue + + email_date = (msg.date or datetime.now(UTC)).isoformat() + products = _extract_products(html, name, email_date) + log(f" Found {len(products)} products") + raw.extend(products) + processed_uids.append(msg.uid) + + if processed_uids: + mb.flag(processed_uids, MailMessageFlags.SEEN, True) + + return raw + + +def fetch_newsletter() -> list[dict]: + if not NEWSLETTER_SOURCES: + log("Newsletter: no sources configured, skipping") + return [] + + config = _imap_config() + log(f"Connecting to {config['host']} as {config['user']}...") + + try: + raw = _run_imap_fetch(config) + except Exception as e: + log(f"Newsletter fetch failed: {e}") + return [] + + articles = _resolve_products(raw) + log(f"Newsletter: {len(articles)} articles extracted") + return articles diff --git a/backend/pipeline/sources/extract_content.py b/backend/pipeline/sources/extract_content.py new file mode 100644 index 0000000000..ff2ce21657 --- /dev/null +++ b/backend/pipeline/sources/extract_content.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import re +from concurrent.futures import ThreadPoolExecutor, as_completed + +import trafilatura + +from pipeline.sources.utils import fetch_with_timeout +from pipeline.utils import log + +SKIP_DOMAINS: set[str] = {"x.com", "twitter.com"} +SNIPPET_MAX_LENGTH = 500 +EXTRACT_TIMEOUT_SECS = 5.0 +CONCURRENCY = 5 + +BLOCKER_PATTERNS: list[re.Pattern] = [ + re.compile(r"something went wrong.*don't fret", re.IGNORECASE | re.DOTALL), + re.compile(r"disable.*privacy\s+extensions", re.IGNORECASE), + re.compile(r"privacy\s+extensions.*and.*retry", re.IGNORECASE), + re.compile(r"please\s+(?:sign|log)\s*in", re.IGNORECASE), + re.compile(r"sign\s*in\s+to\s+(?:continue|read|access)", re.IGNORECASE), + re.compile(r"log\s*in\s+to\s+(?:continue|read|access)", re.IGNORECASE), + re.compile(r"subscribe\s+to\s+(?:continue|read|access|unlock)", re.IGNORECASE), + re.compile( + r"create\s+an?\s+account\s+to\s+(?:continue|read|access)", re.IGNORECASE + ), + re.compile(r"enable\s+javascript\s+to\s+(?:continue|view|use)", re.IGNORECASE), + re.compile(r"javascript\s+is\s+(?:disabled|required)", re.IGNORECASE), + re.compile(r"access\s+denied", re.IGNORECASE), + re.compile(r"403\s+forbidden", re.IGNORECASE), +] + + +def _should_skip(url: str) -> bool: + try: + from urllib.parse import urlparse + + host = urlparse(url).hostname or "" + return any(host == d or host.endswith(f".{d}") for d in SKIP_DOMAINS) + except Exception: + return True + + +def _is_blocker(text: str) -> bool: + if len(text) < 50: + return True + return any(p.search(text) for p in BLOCKER_PATTERNS) + + +def _fetch_html(url: str) -> str: + if _should_skip(url): + return "" + try: + resp = fetch_with_timeout(url, timeout=EXTRACT_TIMEOUT_SECS) + if not resp.is_success: + return "" + ct = resp.headers.get("content-type", "") + if "text/html" not in ct: + return "" + return resp.text + except Exception: + return "" + + +def _extract_text(html: str, max_length: int | None = None) -> str: + if not html: + return "" + text = trafilatura.extract(html, include_comments=False, include_tables=False) or "" + text = re.sub(r"\s+", " ", text).strip() + if not text or _is_blocker(text): + return "" + return text[:max_length] if max_length else text + + +def _extract_one_snippet(url: str) -> tuple[str, str]: + return url, _extract_text(_fetch_html(url), SNIPPET_MAX_LENGTH) + + +def _extract_one_full(url: str, idx: int, total: int) -> tuple[str, str]: + log(f" [{idx + 1}/{total}] Fetching: {url}") + html = _fetch_html(url) + text = _extract_text(html) + log( + f" [{idx + 1}/{total}] {'OK (' + str(len(text)) + ' chars)' if text else 'no content'}: {url}" + ) + return url, text + + +def extract_content(articles: list[dict]) -> list[dict]: + """Fill in missing content snippets for a list of article dicts.""" + needs = [a for a in articles if not a.get("content")] + if not needs: + return articles + + unique_urls = list(dict.fromkeys(a["url"] for a in needs)) + log(f" Extracting content for {len(unique_urls)} URLs...") + + snippet_map: dict[str, str] = {} + with ThreadPoolExecutor(max_workers=CONCURRENCY) as executor: + futures = { + executor.submit(_extract_one_snippet, url): url for url in unique_urls + } + for future in as_completed(futures): + try: + url, text = future.result() + if text: + snippet_map[url] = text + except Exception: + pass + + log(f" Extracted {len(snippet_map)}/{len(unique_urls)} snippets") + + result = [] + for a in articles: + if not a.get("content") and a["url"] in snippet_map: + result.append({**a, "content": snippet_map[a["url"]]}) + else: + result.append(a) + return result + + +def re_extract_full_content(articles: list[dict]) -> dict[str, str]: + """Re-fetch full article text for already-inserted articles. Returns url->text map.""" + if not articles: + return {} + + unique_urls = list(dict.fromkeys(a["url"] for a in articles)) + log(f" Re-extracting full content for {len(unique_urls)} URLs...") + + content_map: dict[str, str] = {} + total = len(unique_urls) + + with ThreadPoolExecutor(max_workers=CONCURRENCY) as executor: + futures = { + executor.submit(_extract_one_full, url, idx, total): url + for idx, url in enumerate(unique_urls) + } + for future in as_completed(futures): + try: + url, text = future.result() + if text: + content_map[url] = text + except Exception: + pass + + log(f" Re-extracted {len(content_map)}/{len(unique_urls)} full texts") + return content_map diff --git a/backend/pipeline/sources/hn.py b/backend/pipeline/sources/hn.py new file mode 100644 index 0000000000..039aea222b --- /dev/null +++ b/backend/pipeline/sources/hn.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import re +from concurrent.futures import ThreadPoolExecutor + +from pipeline.sources.extract_content import extract_content +from pipeline.sources.utils import clean_title, fetch_with_timeout, is_within_window +from pipeline.utils import log + +HN_TOP = "https://hacker-news.firebaseio.com/v0/topstories.json" +HN_ITEM = "https://hacker-news.firebaseio.com/v0/item" +HN_FETCH_LIMIT = 200 + +HN_AI_KEYWORDS = re.compile( + r"\b(ai|llm|gpt|claude|gemini|llama|openai|anthropic|deepseek|mistral|opencode|" + r"tokens|transformer|diffusion|machine.?learning|deep.?learning|neural.?net|" + r"language.?model|artificial.?intelligen|stable.?diffusion|midjourney|copilot|" + r"chatbot|rag|fine.?tun|embedding|token|lora|rlhf|gguf|ollama|hugging.?face)\b", + re.IGNORECASE, +) + + +def _fetch_item(item_id: int) -> dict | None: + try: + resp = fetch_with_timeout(f"{HN_ITEM}/{item_id}.json", timeout=10.0) + return resp.json() + except Exception: + return None + + +def fetch_hn() -> list[dict]: + log("Fetching Hacker News top stories...") + try: + resp = fetch_with_timeout(HN_TOP) + ids: list[int] = resp.json() + except Exception as e: + log(f" HN top stories fetch failed: {e}") + return [] + + top_ids = ids[:HN_FETCH_LIMIT] + + with ThreadPoolExecutor(max_workers=20) as executor: + results = list(executor.map(_fetch_item, top_ids)) + + articles: list[dict] = [] + for item in results: + if not item or item.get("type") != "story" or not item.get("title"): + continue + if not HN_AI_KEYWORDS.search(item["title"]): + continue + pub_date = ( + __import__("datetime") + .datetime.fromtimestamp( + item["time"], tz=__import__("datetime").timezone.utc + ) + .isoformat() + if item.get("time") + else None + ) + if not is_within_window(pub_date): + continue + articles.append( + { + "title": clean_title(item["title"]), + "url": item.get("url") + or f"https://news.ycombinator.com/item?id={item['id']}", + "content": "", + "published_date": pub_date + or __import__("datetime") + .datetime.now(__import__("datetime").timezone.utc) + .isoformat(), + "source": "Hacker News", + "source_type": "hackerNews", + } + ) + + log(f"Hacker News: {len(articles)} AI-related stories") + return extract_content(articles) diff --git a/backend/pipeline/sources/rss.py b/backend/pipeline/sources/rss.py new file mode 100644 index 0000000000..bfb2c071a6 --- /dev/null +++ b/backend/pipeline/sources/rss.py @@ -0,0 +1,112 @@ +"""Generic multi-feed RSS source. Not wired into SOURCES by default (see +run.py) - it wasn't active in the original TS pipeline either. +""" + +from __future__ import annotations + +import feedparser + +from pipeline.sources.utils import ( + clean_title, + fetch_with_timeout, + is_twitter_url, + is_within_window, + resolve_twitter_url, +) +from pipeline.utils import log + +SOURCES = [ + # Only ingest these RSS categories - others are noise (drama, events, business, etc.). + { + "name": "Aligned News", + "rss_url": "https://alignednews.com/feed.xml", + "allowed_categories": {"tips", "products"}, + }, + # Newsletters that publish an RSS feed - moved here from the IMAP-based + # Newsletter source (pipeline/sources/email.py) since reading the feed + # directly is simpler and doesn't need a mailbox. No category filter - + # take everything within the recency window, same as substack.py/hn.py. + {"name": "Console", "rss_url": "https://console.dev/rss.xml"}, + {"name": "The Neuron", "rss_url": "https://rss.beehiiv.com/feeds/N4eCstxvgX.xml"}, + {"name": "AI Hero", "rss_url": "https://www.aihero.dev/rss.xml"}, + {"name": "Import AI", "rss_url": "https://importai.substack.com/feed"}, + {"name": "TLDR AI", "rss_url": "https://tldr.tech/api/rss/ai"}, + {"name": "The Rundown AI", "rss_url": "https://rss.beehiiv.com/feeds/2R3C6Bt5wj.xml"}, + # Ben's Bites is deliberately not here: it's already covered by + # substack-sources.json / fetch_substack(), which is wired into SOURCES. + # Adding it here too would fetch the same feed twice per run. +] + +USER_AGENT = "Agentique/2.0 (+https://agentique.ch)" + + +def _entry_categories(entry: dict) -> list[str]: + tags = entry.get("tags") + if tags: + return [t.get("term", "") for t in tags if t.get("term")] + category = entry.get("category") + return [category] if category else [] + + +def fetch_rss() -> list[dict]: + articles: list[dict] = [] + + for source in SOURCES: + log(f"Fetching {source['name']}...") + try: + resp = fetch_with_timeout(source["rss_url"], headers={"User-Agent": USER_AGENT}) + if not resp.is_success: + raise ValueError(f"Status {resp.status_code}") + feed = feedparser.parse(resp.content) + except Exception as e: + log(f" FAILED: {e}") + continue + + within_window = [ + e for e in feed.entries if is_within_window(e.get("published")) + ] + + allowed_categories = source.get("allowed_categories") + if allowed_categories is None: + relevant = within_window + else: + relevant = [ + e + for e in within_window + if any(c.lower() in allowed_categories for c in _entry_categories(e)) + ] + + log( + f" {source['name']}: {len(within_window)} in window, " + f"{len(relevant)} after category filter" + ) + + for entry in relevant: + raw_url = entry.get("link", "") + description = entry.get("summary", "") + title = clean_title(entry.get("title", "(no title)")) + + url = raw_url + if is_twitter_url(raw_url): + log(f" Resolving tweet: {title}") + found = resolve_twitter_url(title, description) + if found: + url = found + + articles.append( + { + "title": title, + "url": url, + # RSS description is a quality AI summary, far better than + # what scraping x.com would return. + "content": description, + "published_date": entry.get("published"), + "source": source["name"], + "source_type": "rss", + } + ) + + log(f" {source['name']}: {len(articles)} articles ready") + + log(f"RSS: {len(articles)} articles") + return articles diff --git a/backend/pipeline/sources/substack-sources.json b/backend/pipeline/sources/substack-sources.json new file mode 100644 index 0000000000..c3fd0f6541 --- /dev/null +++ b/backend/pipeline/sources/substack-sources.json @@ -0,0 +1,430 @@ +[ + { + "name": "Nate's Substack", + "rssUrl": "https://natesnewsletter.substack.com/feed" + }, + { + "name": "Interconnects AI", + "rssUrl": "https://www.interconnects.ai/feed" + }, + { + "name": "AI Supremacy", + "rssUrl": "https://www.ai-supremacy.com/feed" + }, + { + "name": "The AI Maker", + "rssUrl": "https://aimaker.substack.com/feed" + }, + { + "name": "Understanding AI", + "rssUrl": "https://www.understandingai.org/feed" + }, + { + "name": "AI Tidbits", + "rssUrl": "https://www.aitidbits.ai/feed" + }, + { + "name": "Deep (Learning) Focus", + "rssUrl": "https://cameronrwolfe.substack.com/feed" + }, + { + "name": "AI Newsletter", + "rssUrl": "https://nlp.elvissaravia.com/feed" + }, + { + "name": "Artificial Ignorance", + "rssUrl": "https://www.ignorance.ai/feed" + }, + { + "name": "AI by Hand", + "rssUrl": "https://www.byhand.ai/feed" + }, + { + "name": "Practical AI for Solo Entrepreneurs", + "rssUrl": "https://soloailab.substack.com/feed" + }, + { + "name": "Last Week in AI", + "rssUrl": "https://lastweekin.ai/feed" + }, + { + "name": "The Generative Programmer", + "rssUrl": "https://generativeprogrammer.com/feed" + }, + { + "name": "Designing with AI", + "rssUrl": "https://newsletter.victordibia.com/feed" + }, + { + "name": "Andrej Karpathy", + "rssUrl": "https://karpathy.substack.com/feed" + }, + { + "name": "Ben's Bites", + "rssUrl": "https://www.bensbites.com/feed" + }, + { + "name": "Machine Learning Pills", + "rssUrl": "https://mlpills.substack.com/feed" + }, + { + "name": "Chipstrat", + "rssUrl": "https://www.chipstrat.com/feed" + }, + { + "name": "Making AI Accessible", + "rssUrl": "https://louisbouchard.substack.com/feed" + }, + { + "name": "Redwood Research", + "rssUrl": "https://blog.redwoodresearch.org/feed" + }, + { + "name": "Works on My Machine", + "rssUrl": "https://worksonmymachine.ai/feed" + }, + { + "name": "Hamel's Substack", + "rssUrl": "https://hamelhusain.substack.com/feed" + }, + { + "name": "The AI Engineer", + "rssUrl": "https://newsletter.owainlewis.com/feed" + }, + { + "name": "Trelis Research", + "rssUrl": "https://trelis.substack.com/feed" + }, + { + "name": "Gradient Flow", + "rssUrl": "https://gradientflow.substack.com/feed" + }, + { + "name": "Generally Intelligent", + "rssUrl": "https://generallyintelligent.substack.com/feed" + }, + { + "name": "Building AI Products From Scratch", + "rssUrl": "https://mikikobazeley.substack.com/feed" + }, + { + "name": "LLM Watch", + "rssUrl": "https://www.llmwatch.com/feed" + }, + { + "name": "Loop Engineering", + "rssUrl": "https://linas.substack.com/feed" + }, + { + "name": "Cobus Greyling on LLMs", + "rssUrl": "https://cobusgreyling.substack.com/feed" + }, + { + "name": "Elevate by Addy Osmani", + "rssUrl": "https://addyo.substack.com/feed" + }, + { + "name": "DiamantAI", + "rssUrl": "https://newsletter.diamant-ai.com/feed" + }, + { + "name": "MLWhiz", + "rssUrl": "https://www.mlwhiz.com/feed" + }, + { + "name": "Semantics & Systems", + "rssUrl": "https://khaledea.substack.com/feed" + }, + { + "name": "Token Contributions", + "rssUrl": "https://tokencontributions.substack.com/feed" + }, + { + "name": "Generative History", + "rssUrl": "https://generativehistory.substack.com/feed" + }, + { + "name": "Digital Thoughts", + "rssUrl": "https://thoughts.jock.pl/feed" + }, + { + "name": "Modern Mom Playbook", + "rssUrl": "https://modernmomplaybook.substack.com/feed" + }, + { + "name": "Prompt-Led Product | For PMs Building in the AI Era", + "rssUrl": "https://promptledproduct.substack.com/feed" + }, + { + "name": "AI Weekender", + "rssUrl": "https://aiweekender.substack.com/feed" + }, + { + "name": "Senior Data Science Lead", + "rssUrl": "https://joseparreogarcia.substack.com/feed" + }, + { + "name": "AI Espresso | your tech brew", + "rssUrl": "https://aiespressotechbrew.substack.com/feed" + }, + { + "name": "AI by Aakash", + "rssUrl": "https://www.aibyaakash.com/feed" + }, + { + "name": "Engincan Veske", + "rssUrl": "https://engincanveske.substack.com/feed" + }, + { + "name": "ModelCraft", + "rssUrl": "https://modelcraft.substack.com/feed" + }, + { + "name": "Vik's Newsletter", + "rssUrl": "https://www.viksnewsletter.com/feed" + }, + { + "name": "Standout Systems by Teodora", + "rssUrl": "https://teodoracoach.substack.com/feed" + }, + { + "name": "The Circuit", + "rssUrl": "https://metacircuits.substack.com/feed" + }, + { + "name": "Artificial Enigma", + "rssUrl": "https://idakaukonen.substack.com/feed" + }, + { + "name": "Travis Sparks - Sparkry.AI + Neurodivergence + Business", + "rssUrl": "https://sparkryai.substack.com/feed" + }, + { + "name": "AI Disruption", + "rssUrl": "https://aidisruption.ai/feed" + }, + { + "name": "Cash & Cache", + "rssUrl": "https://cashandcache.substack.com/feed" + }, + { + "name": "Prosper", + "rssUrl": "https://prosperinai.substack.com/feed" + }, + { + "name": "AI Superhero", + "rssUrl": "https://shmulc.substack.com/feed" + }, + { + "name": "AI blew my mind", + "rssUrl": "https://aiblewmymind.substack.com/feed" + }, + { + "name": "ToxSec - AI and Cybersecurity", + "rssUrl": "https://www.toxsec.com/feed" + }, + { + "name": "The Ai Rabbit Hole", + "rssUrl": "https://jtnovelo2131.substack.com/feed" + }, + { + "name": "The Zhang-ularity", + "rssUrl": "https://fzhang.substack.com/feed" + }, + { + "name": "Asymmetric Jump", + "rssUrl": "https://asymmetricjump.substack.com/feed" + }, + { + "name": "Automato: Smarter Work in the Age of AI", + "rssUrl": "https://automato.substack.com/feed" + }, + { + "name": "Empathetic AI Lab", + "rssUrl": "https://aicompanionlab.substack.com/feed" + }, + { + "name": "Leadership in Change", + "rssUrl": "https://leadershipinchange.com/feed" + }, + { + "name": "The FoLD", + "rssUrl": "https://liamdarmody.substack.com/feed" + }, + { + "name": "AI Echoes", + "rssUrl": "https://aiechoes.substack.com/feed" + }, + { + "name": "AI in public", + "rssUrl": "https://humzakhalid.substack.com/feed" + }, + { + "name": "Artificial Code", + "rssUrl": "https://artificialcode.substack.com/feed" + }, + { + "name": "Purposeful AI", + "rssUrl": "https://purposefulai.substack.com/feed" + }, + { + "name": "Shrivu's Substack", + "rssUrl": "https://blog.sshh.io/feed" + }, + { + "name": "The AI Architect", + "rssUrl": "https://mvidmar.substack.com/feed" + }, + { + "name": "The Humans in the Loop", + "rssUrl": "https://www.thehumansintheloop.ai/feed" + }, + { + "name": "AI Enhanced Engineer", + "rssUrl": "https://aienhancedengineer.substack.com/feed" + }, + { + "name": "Product with Attitude", + "rssUrl": "https://karozieminski.substack.com/feed" + }, + { + "name": "And Yet It Moves", + "rssUrl": "https://andyetitmoves.substack.com/feed" + }, + { + "name": "Department of Product", + "rssUrl": "https://departmentofproduct.substack.com/feed" + }, + { + "name": "Design with AI", + "rssUrl": "https://designwithai.substack.com/feed" + }, + { + "name": "Limited Edition Jonathan", + "rssUrl": "https://limitededitionjonathan.substack.com/feed" + }, + { + "name": "Product Growth", + "rssUrl": "https://www.news.aakashg.com/feed" + }, + { + "name": "The Outer Loop", + "rssUrl": "https://theouterloop.substack.com/feed" + }, + { + "name": "Co-Write with AI", + "rssUrl": "https://articles.cowritewithai.com/feed" + }, + { + "name": "Exploring ChatGPT", + "rssUrl": "https://exploringchatgpt.substack.com/feed" + }, + { + "name": "Hello AI World", + "rssUrl": "https://malihaarshad1.substack.com/feed" + }, + { + "name": "Savvy Sloth", + "rssUrl": "https://savvysloth.substack.com/feed" + }, + { + "name": "Turing Post", + "rssUrl": "https://turingpost.substack.com/feed" + }, + { + "name": "Marily’s AI Product Academy Newsletter", + "rssUrl": "https://marily.substack.com/feed" + }, + { + "name": "ByteByteGo Newsletter", + "rssUrl": "https://blog.bytebytego.com/feed" + }, + { + "name": "The Tech Buffet", + "rssUrl": "https://thetechbuffet.substack.com/feed" + }, + { + "name": "SwirlAI Newsletter", + "rssUrl": "https://www.newsletter.swirlai.com/feed" + }, + { + "name": "Simon Willison’s Newsletter", + "rssUrl": "https://simonw.substack.com/feed" + }, + { + "name": "Ahead of AI", + "rssUrl": "https://magazine.sebastianraschka.com/feed" + }, + { + "name": "The Pipe & The Line", + "rssUrl": "https://thepipeandtheline.substack.com/feed" + }, + { + "name": "Coding with Intelligence", + "rssUrl": "https://codingwithintelligence.com/feed" + }, + { + "name": "AI for Software Engineers", + "rssUrl": "https://www.aiforswes.com/feed" + }, + { + "name": "Thonk From First Principles", + "rssUrl": "https://www.thonking.ai/feed" + }, + { + "name": "The Kaitchup – AI on a Budget", + "rssUrl": "https://kaitchup.substack.com/feed" + }, + { + "name": "ThursdAI - Highest signal weekly AI news show", + "rssUrl": "https://sub.thursdai.news/feed" + }, + { + "name": "The AI Break", + "rssUrl": "https://theaibreak.substack.com/feed" + }, + { + "name": "Artificial Corner", + "rssUrl": "https://artificialcorner.com/feed" + }, + { + "name": "The Ontologist", + "rssUrl": "https://ontologist.substack.com/feed" + }, + { + "name": "The Technical Executive", + "rssUrl": "https://sderosiaux.substack.com/feed" + }, + { + "name": "AI Health Uncut", + "rssUrl": "https://www.fixhealth.ai/feed" + }, + { + "name": "Exploring Artificial Intelligence", + "rssUrl": "https://exploringartificialintelligence.substack.com/feed" + }, + { + "name": "All Agents Considered", + "rssUrl": "https://allagentsconsidered.substack.com/feed" + }, + { + "name": "ai.tism", + "rssUrl": "https://aitism.substack.com/feed" + }, + { + "name": "Practical AI Brief", + "rssUrl": "https://practicalaibrief.substack.com/feed" + }, + { + "name": "Matt Paige", + "rssUrl": "https://mattpaige68.substack.com/feed" + }, + { + "name": "Artificial Intelligence Learning 🤖🧠🦾", + "rssUrl": "https://offthegridxp.substack.com/feed" + }, + { + "name": "Machine Learning At Scale", + "rssUrl": "https://machinelearningatscale.substack.com/feed" + } +] diff --git a/backend/pipeline/sources/substack.py b/backend/pipeline/sources/substack.py new file mode 100644 index 0000000000..e4f36fff42 --- /dev/null +++ b/backend/pipeline/sources/substack.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import os +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +import feedparser +import httpx + +from pipeline.sources.utils import BROWSER_HEADERS, clean_title, is_within_window +from pipeline.utils import log + +_PROXY_URL = os.environ.get("RESIDENTIAL_PROXY_URL") + +if _PROXY_URL: + try: + from urllib.parse import urlparse + + _u = urlparse(_PROXY_URL) + log( + f"Substack proxy: {_u.scheme}://{_u.hostname}:{_u.port or '(default)'} (auth: {'yes' if _u.username else 'no'})" + ) + except Exception: + log(f"Substack proxy: set but unparseable (len {len(_PROXY_URL)})") +else: + log("Substack proxy: none (RESIDENTIAL_PROXY_URL unset)") + + +def _fetch_feed_xml(url: str, retries: int = 2, backoff: float = 2.0) -> str: + for attempt in range(retries + 1): + use_proxy = attempt > 0 and bool(_PROXY_URL) + kwargs: dict = { + "headers": BROWSER_HEADERS, + "timeout": 15.0, + "follow_redirects": True, + } + if use_proxy: + kwargs["proxy"] = _PROXY_URL + try: + with httpx.Client(**kwargs) as client: + resp = client.get(url) + except Exception as e: + raise RuntimeError( + f"fetch failed{'(via proxy)' if use_proxy else ''}: {e}" + ) from e + + if resp.is_success: + return resp.text + if resp.status_code in (403, 429) and attempt < retries and _PROXY_URL: + time.sleep(backoff * (2**attempt)) + continue + raise RuntimeError(f"Status code {resp.status_code}") + + raise RuntimeError("Exhausted retries") + + +def _fetch_source(source: dict) -> list[dict]: + name = source["name"] + rss_url = source["rssUrl"] + log(f"Fetching {name}...") + try: + xml = _fetch_feed_xml(rss_url) + feed = feedparser.parse(xml) + items = feed.get("entries", []) + within = [ + it + for it in items + if is_within_window(it.get("published") or it.get("updated")) + ] + log(f" {name}: {len(within)} in window") + return [ + { + "title": clean_title(it.get("title") or "(no title)"), + "url": it.get("link") or "", + "content": it.get("summary") or "", + "published_date": it.get("published") or it.get("updated") or "", + "source": name, + "source_type": "rss", + } + for it in within + ] + except Exception as e: + log(f" FAILED {name}: {e}") + return [] + + +def fetch_substack() -> list[dict]: + import json + import pathlib + + sources_path = pathlib.Path(__file__).parent / "substack-sources.json" + with open(sources_path) as f: + sources: list[dict] = json.load(f) + + articles: list[dict] = [] + with ThreadPoolExecutor(max_workers=10) as executor: + futures = {executor.submit(_fetch_source, src): src for src in sources} + for future in as_completed(futures): + try: + articles.extend(future.result()) + except Exception: + pass + + log(f"Substack: {len(articles)} articles") + return articles diff --git a/backend/pipeline/sources/utils.py b/backend/pipeline/sources/utils.py new file mode 100644 index 0000000000..7e9dfcbdb2 --- /dev/null +++ b/backend/pipeline/sources/utils.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import os +import re +from datetime import UTC, datetime + +import httpx + +from pipeline.utils import log + +HN_PREFIX_RE = re.compile(r"^(?:Show|Launch|Ask|Tell) HN:\s*", re.IGNORECASE) +WINDOW_HOURS = 168 +FETCH_TIMEOUT_SECS = 15.0 +TAVILY_SEARCH_URL = "https://api.tavily.com/search" +TAVILY_SEARCH_CANDIDATES = 5 +TWITTER_HOSTS = {"x.com", "twitter.com", "t.co"} +_TWITTER_HANDLE_PREFIX_RE = re.compile(r"^@\w+\s+[-–]\s+") + +BROWSER_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" + ), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", +} + + +def clean_title(title: str) -> str: + return HN_PREFIX_RE.sub("", title).strip() + + +def is_within_window(date_str: str | None, window_hours: int = WINDOW_HOURS) -> bool: + if not date_str: + return True + try: + from email.utils import parsedate_to_datetime + + try: + published = parsedate_to_datetime(date_str) + except Exception: + published = datetime.fromisoformat(date_str.replace("Z", "+00:00")) + now = datetime.now(UTC) + hours_ago = (now - published.astimezone(UTC)).total_seconds() / 3600 + return hours_ago <= window_hours + except Exception: + return True + + +def fetch_with_timeout( + url: str, + timeout: float = FETCH_TIMEOUT_SECS, + proxy: str | None = None, + headers: dict | None = None, +) -> httpx.Response: + kwargs: dict = {"timeout": timeout, "follow_redirects": True} + if proxy: + kwargs["proxy"] = proxy + if headers: + kwargs["headers"] = headers + with httpx.Client(**kwargs) as client: + return client.get(url) + + +def tavily_search(query: str, max_results: int = TAVILY_SEARCH_CANDIDATES) -> list[dict]: + api_key = os.environ["TAVILY_API_KEY"] + resp = httpx.post( + TAVILY_SEARCH_URL, + json={"api_key": api_key, "query": query, "max_results": max_results}, + timeout=15.0, + ) + resp.raise_for_status() + data = resp.json() + return [ + { + "title": r.get("title", ""), + "url": r["url"], + "description": r.get("content", ""), + } + for r in data.get("results", []) + ] + + +def is_twitter_url(url: str) -> bool: + try: + from urllib.parse import urlparse + + return urlparse(url).hostname in TWITTER_HOSTS + except Exception: + return False + + +def resolve_twitter_url(title: str, description: str) -> str | None: + """Given an x.com/twitter.com URL's title+description, search for the + underlying article and return the best non-Twitter result URL. + """ + clean = _TWITTER_HANDLE_PREFIX_RE.sub("", title).strip() + first_sentence = description.split(". ")[0].strip() if description else "" + query = f"{clean} {first_sentence}" if first_sentence else clean + + try: + results = tavily_search(query) + except Exception as e: + log(f" Search failed: {e}") + return None + + match = next((r for r in results if not is_twitter_url(r["url"])), None) + if match: + log(f" Resolved to: {match['url']}") + return match["url"] + return None diff --git a/backend/pipeline/steps.py b/backend/pipeline/steps.py new file mode 100644 index 0000000000..5e37a9ee64 --- /dev/null +++ b/backend/pipeline/steps.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import re + +SCORE_THRESHOLD = 76 +PROMPT_CONTENT_CAP = 1500 + +NON_REPO_OWNERS = { + "features", + "login", + "pricing", + "about", + "marketplace", + "explore", + "topics", + "collections", + "trending", + "sponsors", + "orgs", + "apps", + "contact", + "security", +} + +GITHUB_REPO_RE = re.compile( + r"https?://(?:www\.)?github\.com/([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)" +) + + +def github_repo_from_content(content: str) -> str | None: + match = GITHUB_REPO_RE.search(content) + if not match: + return None + owner, repo = match.group(1), match.group(2) + if owner.lower() in NON_REPO_OWNERS: + return None + return f"https://github.com/{owner}/{repo}" + + +def kind_from_url(url: str) -> str | None: + try: + from urllib.parse import urlparse + + host = urlparse(url).hostname or "" + host = host.removeprefix("www.") + if host in ("github.com", "gitlab.com"): + return "repo" + if host in ("huggingface.co", "hf.co"): + return "model" + if host in ("arxiv.org", "ar5iv.labs.arxiv.org"): + return "paper" + except Exception: + pass + return None + + +TRUST_BY_SOURCE: dict[str, str] = { + "Hacker News": "high", + "AI News": "high", +} diff --git a/backend/pipeline/utils.py b/backend/pipeline/utils.py new file mode 100644 index 0000000000..12e629a124 --- /dev/null +++ b/backend/pipeline/utils.py @@ -0,0 +1,48 @@ +from datetime import UTC, datetime + +import regex + + +def log(message: str) -> None: + ts = datetime.now(UTC).isoformat() + print(f"[{ts}] {message}") + + +def wait_ms(ms: int) -> None: + import time + + time.sleep(ms / 1000) + + +def strip_title_wrappers(text: str) -> str: + """Remove LLM echo artifacts: leading [Source] tags, numbering, surrounding quotes.""" + s = text.strip() + s = regex.sub(r"^\[[^\]]+\]\s*", "", s) + s = regex.sub(r"^\d+\.\s*", "", s) + pairs = [('"', '"'), ("'", "'"), ("“", "”"), ("‘", "’")] + for open_, close in pairs: + if s.startswith(open_) and s.endswith(close) and len(s) >= 2: + s = s[len(open_) : len(s) - len(close)].strip() + break + return s + + +def sanitize_llm_text(text: str) -> str: + """Strip smart quotes, arrows, emoji, and extra whitespace from LLM output.""" + s = text + s = regex.sub(r"[‘’‚]", "'", s) + s = regex.sub(r"[“”„]", '"', s) + s = regex.sub(r"→", "->", s) + s = regex.sub(r"←", "<-", s) + s = regex.sub(r"↔", "<->", s) + s = regex.sub(r"[–—]", "-", s) + s = regex.sub(r"…", "...", s) + s = regex.sub(r"[•‣◦▪▫]", "-", s) + s = regex.sub( + r"[\p{Emoji_Presentation}\p{Extended_Pictographic}]️?" + r"(‍[\p{Emoji_Presentation}\p{Extended_Pictographic}]️?)*", + "", + s, + ) + s = regex.sub(r" {2,}", " ", s) + return s.strip() diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 0e3921a1ca..e7b10a7d1d 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -19,6 +19,15 @@ dependencies = [ "sentry-sdk[fastapi]>=2.0.0,<3.0.0", "pyjwt<3.0.0,>=2.13.0", "pwdlib[argon2,bcrypt]>=0.3.0", + "pgvector>=0.3.0", + "model2vec>=0.4.0", + "baml-py>=0.223.0", + "trafilatura>=2.0.0", + "feedparser>=6.0.0", + "dnspython>=2.6.0", + "regex>=2024.0.0", + "resend>=2.0.0", + "imap-tools>=1.5.0", ] [dependency-groups] @@ -65,6 +74,10 @@ ignore = [ # Preserve types, even if a file imports `from __future__ import annotations`. keep-runtime-typing = true +[tool.ruff.lint.per-file-ignores] +# The pipeline logs to stdout via print() so supercronic captures it in container logs. +"pipeline/**" = ["T201"] + [tool.coverage.run] source = ["app"] dynamic_context = "test_function" @@ -72,6 +85,13 @@ dynamic_context = "test_function" [tool.coverage.report] show_missing = true sort = "-Cover" +omit = [ + "app/api/routes/items.py", + "app/api/routes/private.py", + "app/api/routes/utils.py", + "app/seed_articles.py", + "app/initial_data.py", +] [tool.coverage.html] show_contexts = true diff --git a/backend/scripts/import_articles.py b/backend/scripts/import_articles.py new file mode 100644 index 0000000000..8e3644a3fd --- /dev/null +++ b/backend/scripts/import_articles.py @@ -0,0 +1,107 @@ +""" +One-time migration: import articles from articles-export.json into Postgres, +re-embedding each article (title + summary) with model2vec potion-base-8M. + +Usage (from repo root, with .env sourced or DATABASE_URL set): + uv run python backend/scripts/import_articles.py [path/to/articles-export.json] + +Default input path: articles-export.json (repo root). +""" +import json +import sys +from datetime import datetime +from pathlib import Path + +import numpy as np +from model2vec import StaticModel +from sqlmodel import Session, create_engine + +# Allow running from repo root without installing the package +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from app.core.config import settings # noqa: E402 +from app.models_articles import Article # noqa: E402 + +EXPORT_PATH = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("articles-export.json") +BATCH_SIZE = 64 + + +def parse_dt(value: str | None) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def embed_batch(model: StaticModel, texts: list[str]) -> list[list[float]]: + vecs = model.encode(texts) + norms = np.linalg.norm(vecs, axis=1, keepdims=True) + norms = np.where(norms == 0, 1, norms) + return (vecs / norms).tolist() + + +def main() -> None: + if not EXPORT_PATH.exists(): + print(f"Export file not found: {EXPORT_PATH}") + sys.exit(1) + + raw = json.loads(EXPORT_PATH.read_text()) + print(f"Loaded {len(raw)} articles from {EXPORT_PATH}") + + print("Loading model2vec model (minishlab/potion-base-8M)…") + model = StaticModel.from_pretrained("minishlab/potion-base-8M") + + engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) + + inserted = skipped = embedded = 0 + + with Session(engine) as session: + # Process in batches for embedding efficiency + embeddable = [ + r for r in raw if r.get("score") is not None and r.get("summary") + ] + non_embeddable = [ + r for r in raw if not (r.get("score") is not None and r.get("summary")) + ] + + # Build embedding map: id → vector + embedding_map: dict[int, list[float]] = {} + for i in range(0, len(embeddable), BATCH_SIZE): + batch = embeddable[i : i + BATCH_SIZE] + texts = [f"{r['title']}\n{r['summary']}" for r in batch] + vecs = embed_batch(model, texts) + for r, vec in zip(batch, vecs): + embedding_map[r["id"]] = vec + embedded += len(batch) + print(f" embedded {embedded}/{len(embeddable)}", end="\r") + + print(f"\nEmbedded {embedded} articles, skipping {len(non_embeddable)} (no summary)") + + all_rows = embeddable + non_embeddable + for row in all_rows: + article = Article( + title=row["title"], + source=row["source"], + source_type=row["source_type"], + url=row.get("url"), + published_at=parse_dt(row.get("published_at")), + score=row.get("score"), + summary=row.get("summary"), + categories=row.get("categories") or [], + kind=row.get("kind"), + content=row.get("content") or "", + embedding=embedding_map.get(row["id"]), + created_at=parse_dt(row.get("created_at")), + ) + session.add(article) + inserted += 1 + + session.commit() + + print(f"Done. Inserted {inserted} articles ({embedded} with embeddings, {skipped} skipped).") + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/prestart.sh b/backend/scripts/prestart.sh index 1b395d513f..6326b7528c 100644 --- a/backend/scripts/prestart.sh +++ b/backend/scripts/prestart.sh @@ -11,3 +11,8 @@ alembic upgrade head # Create initial data in DB python app/initial_data.py + +# Seed sample articles for local/CI (module also self-guards on production) +if [ "$ENVIRONMENT" != "production" ]; then + python -m app.seed_articles +fi diff --git a/backend/tests/api/routes/test_articles.py b/backend/tests/api/routes/test_articles.py new file mode 100644 index 0000000000..ed1177e58a --- /dev/null +++ b/backend/tests/api/routes/test_articles.py @@ -0,0 +1,227 @@ +import uuid +from datetime import UTC, datetime, timedelta + +import pytest +from fastapi.testclient import TestClient +from pgvector.sqlalchemy import Vector +from sqlalchemy import cast +from sqlmodel import Session, col, select + +from app.api.routes import articles +from app.core.config import settings +from app.core.security import create_access_token +from app.models_agentique import Article +from tests.utils.article import create_random_article +from tests.utils.user import authentication_token_from_email +from tests.utils.utils import random_email + +ARTICLES_URL = f"{settings.API_V1_STR}/articles" + + +def test_read_articles_default(client: TestClient) -> None: + r = client.get(f"{ARTICLES_URL}/") + assert r.status_code == 200 + data = r.json() + assert data["count"] > 0 + assert len(data["data"]) > 0 + article = data["data"][0] + assert "title" in article + assert "score" in article + + +def test_read_articles_filter_category(client: TestClient) -> None: + r = client.get(f"{ARTICLES_URL}/", params={"category": "dev", "limit": 50}) + assert r.status_code == 200 + data = r.json() + assert data["count"] > 0 + assert all("dev" in a["categories"] for a in data["data"]) + + +def test_read_articles_filter_min_score(client: TestClient) -> None: + r = client.get(f"{ARTICLES_URL}/", params={"min_score": 8, "limit": 50}) + assert r.status_code == 200 + data = r.json() + assert all(a["score"] >= 8 for a in data["data"]) + + +def test_read_articles_filter_kind(client: TestClient) -> None: + r = client.get(f"{ARTICLES_URL}/", params={"kind": "repo", "limit": 50}) + assert r.status_code == 200 + data = r.json() + assert data["count"] > 0 + assert all(a["kind"] == "repo" for a in data["data"]) + + +def test_read_articles_filter_never_increases_count(client: TestClient) -> None: + base = client.get(f"{ARTICLES_URL}/").json()["count"] + filtered = client.get(f"{ARTICLES_URL}/", params={"min_score": 5}).json()["count"] + assert filtered <= base + + +def test_read_articles_sort_published_at_desc(client: TestClient) -> None: + r = client.get( + f"{ARTICLES_URL}/", params={"sort": "published_at-desc", "limit": 50} + ) + data = r.json()["data"] + dates = [datetime.fromisoformat(a["published_at"]) for a in data] + assert dates == sorted(dates, reverse=True) + + +def test_read_articles_sort_default_is_score_desc(client: TestClient) -> None: + r = client.get(f"{ARTICLES_URL}/", params={"limit": 50}) + scores = [a["score"] for a in r.json()["data"]] + assert scores == sorted(scores, reverse=True) + + +def test_read_articles_since_narrows_results(client: TestClient) -> None: + wide_since = (datetime.now(UTC) - timedelta(days=30)).isoformat() + narrow_since = (datetime.now(UTC) - timedelta(days=3)).isoformat() + + wide = client.get( + f"{ARTICLES_URL}/", params={"since": wide_since, "limit": 50} + ).json() + narrow = client.get( + f"{ARTICLES_URL}/", params={"since": narrow_since, "limit": 50} + ).json() + + assert narrow["count"] <= wide["count"] + narrow_ids = {a["id"] for a in narrow["data"]} + wide_ids = {a["id"] for a in wide["data"]} + assert narrow_ids <= wide_ids + + +def test_read_articles_malformed_since_falls_back_to_default_window( + client: TestClient, +) -> None: + default = client.get(f"{ARTICLES_URL}/").json() + malformed = client.get(f"{ARTICLES_URL}/", params={"since": "not-a-date"}).json() + + assert malformed["count"] == default["count"] + + +def test_search_articles( + client: TestClient, db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + fake_vec = [0.05] * 256 + monkeypatch.setattr(articles, "_embed", lambda text: fake_vec) + + r = client.get(f"{ARTICLES_URL}/search", params={"q": "agents", "limit": 5}) + assert r.status_code == 200 + data = r.json() + assert len(data["data"]) <= 5 + assert data["count"] == len(data["data"]) + + expected_ids = db.exec( + select(Article.id) + .where(Article.score.is_not(None)) # type: ignore[union-attr] + .where(Article.embedding.is_not(None)) # type: ignore[union-attr] + .order_by( + cast(Article.embedding, Vector(256)).cosine_distance(fake_vec), + col(Article.id).desc(), + ) + .limit(5) + ).all() + assert [a["id"] for a in data["data"]] == list(expected_ids) + + +def test_article_stats(client: TestClient) -> None: + r = client.get(f"{ARTICLES_URL}/stats") + assert r.status_code == 200 + data = r.json() + assert data["total"] >= 50 + datetime.fromisoformat(data["lastUpdated"]) + + +def test_read_articles_anonymous_has_like_count_and_liked_by_me( + client: TestClient, +) -> None: + r = client.get(f"{ARTICLES_URL}/", params={"limit": 50}) + data = r.json()["data"] + assert len(data) > 0 + for article in data: + assert article["like_count"] >= 0 + assert article["liked_by_me"] is False + + +def test_read_articles_liked_by_me_true_only_for_liked( + client: TestClient, db: Session, normal_user_token_headers: dict[str, str] +) -> None: + now = datetime.now(UTC) + liked = create_random_article(db, score=9, published_at=now) + unliked = create_random_article(db, score=9, published_at=now) + + r = client.put(f"{ARTICLES_URL}/{liked.id}/like", headers=normal_user_token_headers) + assert r.status_code == 200 + + r = client.get( + f"{ARTICLES_URL}/", params={"limit": 50}, headers=normal_user_token_headers + ) + data = {a["id"]: a for a in r.json()["data"]} + assert data[liked.id]["liked_by_me"] is True + assert data[liked.id]["like_count"] == 1 + assert data[unliked.id]["liked_by_me"] is False + assert data[unliked.id]["like_count"] == 0 + + +def test_read_articles_garbage_token_treated_as_anonymous(client: TestClient) -> None: + r = client.get( + f"{ARTICLES_URL}/", + params={"limit": 5}, + headers={"Authorization": "Bearer not-a-real-token"}, + ) + assert r.status_code == 200 + for article in r.json()["data"]: + assert article["liked_by_me"] is False + + +def test_read_articles_valid_token_unknown_user_treated_as_anonymous( + client: TestClient, +) -> None: + token = create_access_token(str(uuid.uuid4()), expires_delta=timedelta(minutes=5)) + r = client.get( + f"{ARTICLES_URL}/", + params={"limit": 5}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert r.status_code == 200 + for article in r.json()["data"]: + assert article["liked_by_me"] is False + + +def test_read_articles_sort_likes_desc(client: TestClient, db: Session) -> None: + now = datetime.now(UTC) + low_score_more_likes = create_random_article(db, score=3, published_at=now) + high_score_no_likes = create_random_article(db, score=9, published_at=now) + tied_score_a = create_random_article(db, score=5, published_at=now) + tied_score_b = create_random_article(db, score=5, published_at=now) + + headers_list = [ + authentication_token_from_email(client=client, email=random_email(), db=db) + for _ in range(2) + ] + for headers in headers_list: + client.put(f"{ARTICLES_URL}/{low_score_more_likes.id}/like", headers=headers) + client.put(f"{ARTICLES_URL}/{tied_score_a.id}/like", headers=headers_list[0]) + + r = client.get(f"{ARTICLES_URL}/", params={"sort": "likes-desc", "limit": 50}) + data = r.json()["data"] + ids = [a["id"] for a in data] + + assert ids.index(low_score_more_likes.id) < ids.index(high_score_no_likes.id) + # tied like counts (both 0) fall back to score desc + assert ids.index(high_score_no_likes.id) < ids.index(tied_score_b.id) + # tied_score_a has 1 like, tied_score_b has 0 -> a before b despite equal score + assert ids.index(tied_score_a.id) < ids.index(tied_score_b.id) + + +def test_search_articles_has_like_count_and_liked_by_me( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + fake_vec = [0.05] * 256 + monkeypatch.setattr(articles, "_embed", lambda text: fake_vec) + + r = client.get(f"{ARTICLES_URL}/search", params={"q": "agents", "limit": 5}) + assert r.status_code == 200 + for article in r.json()["data"]: + assert "like_count" in article + assert "liked_by_me" in article diff --git a/backend/tests/api/routes/test_items.py b/backend/tests/api/routes/test_items.py index 3e82cd0134..89635bba3e 100644 --- a/backend/tests/api/routes/test_items.py +++ b/backend/tests/api/routes/test_items.py @@ -1,11 +1,14 @@ import uuid +import pytest from fastapi.testclient import TestClient from sqlmodel import Session from app.core.config import settings from tests.utils.item import create_random_item +pytestmark = pytest.mark.skip(reason="auth unused in Agentique") + def test_create_item( client: TestClient, superuser_token_headers: dict[str, str] diff --git a/backend/tests/api/routes/test_likes.py b/backend/tests/api/routes/test_likes.py new file mode 100644 index 0000000000..ca2e6a4934 --- /dev/null +++ b/backend/tests/api/routes/test_likes.py @@ -0,0 +1,202 @@ +from datetime import datetime + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy.exc import IntegrityError +from sqlmodel import Session, select + +from app.core.config import settings +from app.models_agentique import ArticleLike +from tests.utils.article import create_random_article +from tests.utils.user import authentication_token_from_email, create_random_user +from tests.utils.utils import random_email + + +def test_article_like_duplicate_pk_raises(db: Session) -> None: + user = create_random_user(db) + article = create_random_article(db) + assert user.id is not None + assert article.id is not None + + db.add(ArticleLike(user_id=user.id, article_id=article.id)) + db.commit() + + db.add(ArticleLike(user_id=user.id, article_id=article.id)) + with pytest.raises(IntegrityError): + db.commit() + db.rollback() + + +def test_article_like_created_at_auto_populates(db: Session) -> None: + user = create_random_user(db) + article = create_random_article(db) + assert user.id is not None + assert article.id is not None + + like = ArticleLike(user_id=user.id, article_id=article.id) + db.add(like) + db.commit() + db.refresh(like) + + assert isinstance(like.created_at, datetime) + + +def test_like_without_token_returns_401(client: TestClient, db: Session) -> None: + article = create_random_article(db) + r = client.put(f"{settings.API_V1_STR}/articles/{article.id}/like") + assert r.status_code == 401 + + +def test_like_with_token_creates_row( + client: TestClient, db: Session, normal_user_token_headers: dict[str, str] +) -> None: + article = create_random_article(db) + r = client.put( + f"{settings.API_V1_STR}/articles/{article.id}/like", + headers=normal_user_token_headers, + ) + assert r.status_code == 200 + + rows = db.exec( + select(ArticleLike).where(ArticleLike.article_id == article.id) + ).all() + assert len(rows) == 1 + + +def test_like_is_idempotent( + client: TestClient, db: Session, normal_user_token_headers: dict[str, str] +) -> None: + article = create_random_article(db) + for _ in range(3): + r = client.put( + f"{settings.API_V1_STR}/articles/{article.id}/like", + headers=normal_user_token_headers, + ) + assert r.status_code == 200 + + rows = db.exec( + select(ArticleLike).where(ArticleLike.article_id == article.id) + ).all() + assert len(rows) == 1 + + +def test_like_nonexistent_article_404( + client: TestClient, db: Session, normal_user_token_headers: dict[str, str] +) -> None: + r = client.put( + f"{settings.API_V1_STR}/articles/999999999/like", + headers=normal_user_token_headers, + ) + assert r.status_code == 404 + + rows = db.exec(select(ArticleLike).where(ArticleLike.article_id == 999999999)).all() + assert len(rows) == 0 + + +def test_unlike_removes_row( + client: TestClient, db: Session, normal_user_token_headers: dict[str, str] +) -> None: + article = create_random_article(db) + client.put( + f"{settings.API_V1_STR}/articles/{article.id}/like", + headers=normal_user_token_headers, + ) + + r = client.delete( + f"{settings.API_V1_STR}/articles/{article.id}/like", + headers=normal_user_token_headers, + ) + assert r.status_code == 200 + + rows = db.exec( + select(ArticleLike).where(ArticleLike.article_id == article.id) + ).all() + assert len(rows) == 0 + + +def test_unlike_when_not_liked_is_idempotent( + client: TestClient, db: Session, normal_user_token_headers: dict[str, str] +) -> None: + article = create_random_article(db) + r = client.delete( + f"{settings.API_V1_STR}/articles/{article.id}/like", + headers=normal_user_token_headers, + ) + assert r.status_code == 200 + + +def test_liked_articles_without_token_returns_401(client: TestClient) -> None: + r = client.get(f"{settings.API_V1_STR}/me/liked-articles") + assert r.status_code == 401 + + +def test_liked_articles_ordered_newest_liked_first( + client: TestClient, db: Session, normal_user_token_headers: dict[str, str] +) -> None: + articles = [create_random_article(db) for _ in range(3)] + for article in articles: + r = client.put( + f"{settings.API_V1_STR}/articles/{article.id}/like", + headers=normal_user_token_headers, + ) + assert r.status_code == 200 + + r = client.get( + f"{settings.API_V1_STR}/me/liked-articles", headers=normal_user_token_headers + ) + assert r.status_code == 200 + data = r.json()["data"] + returned_ids = [a["id"] for a in data] + liked_ids = [a.id for a in reversed(articles)] + assert returned_ids[: len(liked_ids)] == liked_ids + + +def test_liked_articles_have_correct_like_count_and_liked_by_me( + client: TestClient, db: Session, normal_user_token_headers: dict[str, str] +) -> None: + article = create_random_article(db) + other_headers = authentication_token_from_email( + client=client, email=random_email(), db=db + ) + client.put( + f"{settings.API_V1_STR}/articles/{article.id}/like", headers=other_headers + ) + client.put( + f"{settings.API_V1_STR}/articles/{article.id}/like", + headers=normal_user_token_headers, + ) + + r = client.get( + f"{settings.API_V1_STR}/me/liked-articles", headers=normal_user_token_headers + ) + assert r.status_code == 200 + data = {a["id"]: a for a in r.json()["data"]} + assert data[article.id]["liked_by_me"] is True + assert data[article.id]["like_count"] == 2 + + +def test_liked_articles_does_not_leak_other_users_likes( + client: TestClient, db: Session, normal_user_token_headers: dict[str, str] +) -> None: + my_article = create_random_article(db) + other_article = create_random_article(db) + + client.put( + f"{settings.API_V1_STR}/articles/{my_article.id}/like", + headers=normal_user_token_headers, + ) + + other_headers = authentication_token_from_email( + client=client, email=random_email(), db=db + ) + client.put( + f"{settings.API_V1_STR}/articles/{other_article.id}/like", + headers=other_headers, + ) + + r = client.get( + f"{settings.API_V1_STR}/me/liked-articles", headers=normal_user_token_headers + ) + returned_ids = {a["id"] for a in r.json()["data"]} + assert my_article.id in returned_ids + assert other_article.id not in returned_ids diff --git a/backend/tests/api/routes/test_login.py b/backend/tests/api/routes/test_login.py index 96677a25f6..4663090dd7 100644 --- a/backend/tests/api/routes/test_login.py +++ b/backend/tests/api/routes/test_login.py @@ -1,5 +1,6 @@ from unittest.mock import patch +import resend from fastapi.testclient import TestClient from pwdlib.hashers.bcrypt import BcryptHasher from sqlmodel import Session @@ -47,11 +48,38 @@ def test_use_access_token( def test_recovery_password( + client: TestClient, normal_user_token_headers: dict[str, str], monkeypatch +) -> None: + monkeypatch.setattr( + "app.core.config.settings.RESEND_API_KEY", "test-key", raising=False + ) + monkeypatch.setattr( + "app.core.config.settings.EMAILS_FROM_EMAIL", "admin@example.com" + ) + calls = [] + monkeypatch.setattr(resend.Emails, "send", lambda payload: calls.append(payload)) + + email = "test@example.com" + r = client.post( + f"{settings.API_V1_STR}/password-recovery/{email}", + headers=normal_user_token_headers, + ) + assert r.status_code == 200 + assert r.json() == { + "message": "If that email is registered, we sent a password recovery link" + } + assert len(calls) == 1 + assert calls[0]["to"] == email + + +def test_recovery_password_smtp_fallback( client: TestClient, normal_user_token_headers: dict[str, str] ) -> None: with ( patch("app.core.config.settings.SMTP_HOST", "smtp.example.com"), patch("app.core.config.settings.SMTP_USER", "admin@example.com"), + patch("app.core.config.settings.EMAILS_FROM_EMAIL", "admin@example.com"), + patch("app.core.config.settings.RESEND_API_KEY", None), ): email = "test@example.com" r = client.post( diff --git a/backend/tests/api/routes/test_newsletter.py b/backend/tests/api/routes/test_newsletter.py new file mode 100644 index 0000000000..633265256d --- /dev/null +++ b/backend/tests/api/routes/test_newsletter.py @@ -0,0 +1,65 @@ +import resend +from fastapi.testclient import TestClient +from sqlmodel import Session + +from app.models_agentique import NewsletterSubscriber +from tests.utils.utils import random_email + +NEWSLETTER_URL = "/api/newsletter/subscribe" + + +def test_subscribe_valid_email_defaults_to_all( + client: TestClient, db: Session, monkeypatch +) -> None: + monkeypatch.setenv("RESEND_API_KEY", "test-key") + monkeypatch.setenv("RESEND_AUDIENCE_ID", "test-audience") + calls = [] + monkeypatch.setattr( + resend.Contacts, "create", lambda payload: calls.append(payload) + ) + email = random_email() + + r = client.post(NEWSLETTER_URL, json={"email": email}) + assert r.status_code == 200 + assert r.json() == {"ok": True} + + subscriber = db.get(NewsletterSubscriber, email) + assert subscriber is not None + assert subscriber.categories == ["all"] + assert calls == [{"email": email, "audience_id": "test-audience"}] + + db.delete(subscriber) + db.commit() + + +def test_subscribe_with_categories_and_custom_category( + client: TestClient, db: Session, monkeypatch +) -> None: + monkeypatch.setattr(resend.Contacts, "create", lambda *args, **kwargs: None) + email = random_email() + + r = client.post( + NEWSLETTER_URL, + json={ + "email": email, + "categories": ["dev", "research"], + "customCategory": "rust", + }, + ) + assert r.status_code == 200 + + subscriber = db.get(NewsletterSubscriber, email) + assert subscriber is not None + assert subscriber.categories == ["dev", "research"] + assert subscriber.custom_category == "rust" + + db.delete(subscriber) + db.commit() + + +def test_subscribe_invalid_email_returns_400(client: TestClient, monkeypatch) -> None: + monkeypatch.setattr(resend.Contacts, "create", lambda *args, **kwargs: None) + + r = client.post(NEWSLETTER_URL, json={"email": "not-an-email"}) + assert r.status_code == 400 + assert r.json()["detail"] == "Valid email is required" diff --git a/backend/tests/api/routes/test_private.py b/backend/tests/api/routes/test_private.py index 1e1f985021..0a995721ab 100644 --- a/backend/tests/api/routes/test_private.py +++ b/backend/tests/api/routes/test_private.py @@ -1,9 +1,12 @@ +import pytest from fastapi.testclient import TestClient from sqlmodel import Session, select from app.core.config import settings from app.models import User +pytestmark = pytest.mark.skip(reason="auth unused in Agentique") + def test_create_user(client: TestClient, db: Session) -> None: r = client.post( diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 7cdabf3c45..9cff4f3b10 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -8,6 +8,7 @@ from app.core.db import engine, init_db from app.main import app from app.models import Item, User +from app.models_agentique import ArticleLike from tests.utils.user import authentication_token_from_email from tests.utils.utils import get_superuser_token_headers @@ -17,6 +18,8 @@ def db() -> Generator[Session]: with Session(engine) as session: init_db(session) yield session + statement = delete(ArticleLike) + session.execute(statement) statement = delete(Item) session.execute(statement) statement = delete(User) diff --git a/backend/tests/crud/test_user.py b/backend/tests/crud/test_user.py index 3db77ef624..c6b6c63553 100644 --- a/backend/tests/crud/test_user.py +++ b/backend/tests/crud/test_user.py @@ -1,3 +1,4 @@ +import pytest from fastapi.encoders import jsonable_encoder from pwdlib.hashers.bcrypt import BcryptHasher from sqlmodel import Session @@ -7,6 +8,8 @@ from app.models import User, UserCreate, UserUpdate from tests.utils.utils import random_email, random_lower_string +pytestmark = pytest.mark.skip(reason="auth unused in Agentique") + def test_create_user(db: Session) -> None: email = random_email() diff --git a/backend/tests/utils/article.py b/backend/tests/utils/article.py new file mode 100644 index 0000000000..82a3bfc3a7 --- /dev/null +++ b/backend/tests/utils/article.py @@ -0,0 +1,26 @@ +import random + +from sqlmodel import Session + +from app.models_agentique import Article +from tests.utils.utils import random_lower_string + + +def create_random_article(db: Session, **overrides: object) -> Article: + defaults: dict[str, object] = { + "title": random_lower_string(), + "source": "Hacker News", + "source_type": "hackerNews", + "url": f"https://example.com/{random_lower_string()}", + "score": random.randint(1, 10), + "summary": random_lower_string(), + "categories": ["dev"], + "kind": "blog", + "content": random_lower_string(), + } + defaults.update(overrides) + article = Article(**defaults) # type: ignore[arg-type] + db.add(article) + db.commit() + db.refresh(article) + return article diff --git a/baml_src/clients.baml b/baml_src/clients.baml new file mode 100644 index 0000000000..a7cca05e9e --- /dev/null +++ b/baml_src/clients.baml @@ -0,0 +1,19 @@ +client Nvidia { + provider openai-generic + options { + base_url "https://integrate.api.nvidia.com/v1" + api_key env.NVIDIA_NIM_API_KEY + model "qwen/qwen3-next-80b-a3b-instruct" + timeout 60 + } +} + +client NvidiaGptOss { + provider openai-generic + options { + base_url "https://integrate.api.nvidia.com/v1" + api_key env.NVIDIA_NIM_API_KEY + model "openai/gpt-oss-120b" + timeout 60 + } +} diff --git a/baml_src/dedup.baml b/baml_src/dedup.baml new file mode 100644 index 0000000000..fd536bb051 --- /dev/null +++ b/baml_src/dedup.baml @@ -0,0 +1,58 @@ +class DedupMatch { + url string @description("URL of the new article") + existingUrl string @description("URL of the existing article it duplicates") +} + +function SemanticDedup( + newArticles: ArticleInput[], + existingArticles: ExistingArticle[] +) -> DedupMatch[] { + client NvidiaGptOss + prompt #" + {{ _.role("system") }} + You are a semantic similarity engine. You compare article pairs and determine whether they cover the same underlying story. You focus on substance, not surface-level keyword overlap. + + {{ _.role("user") }} + You are given two lists: + 1. NEW articles that were just fetched + 2. EXISTING articles already in our database from the last 14 days + + Your job: identify which NEW articles are about the same story as an EXISTING article. Two articles are "the same story" if they cover the same announcement, release, event, or topic - even if from different sources or with different angles. + + NEW articles: + {% for a in newArticles %} + {{ loop.index }}. [{{ a.source }}] "{{ a.title }}" - {{ a.url }}{% if a.trust %} [trust: {{ a.trust }}]{% endif %} + {% if a.snippet %} {{ a.snippet }}{% endif %} + {% endfor %} + + EXISTING articles: + {% for a in existingArticles %} + {{ loop.index }}. [{{ a.source }}] "{{ a.title }}" - {{ a.url }} + {% endfor %} + + For each match, return the new article's URL and the existing article's URL. + If there are no duplicates, return an empty list. + + {{ ctx.output_format }} + "# +} + +test dedup_no_matches { + functions [SemanticDedup] + args { + newArticles [ + { + url "https://a.com/1" + title "Claude 4 released" + source "HN" + } + ] + existingArticles [ + { + url "https://b.com/1" + title "The state of Rust in 2026" + source "RSS" + } + ] + } +} diff --git a/baml_src/discover.baml b/baml_src/discover.baml new file mode 100644 index 0000000000..30a55c1ad4 --- /dev/null +++ b/baml_src/discover.baml @@ -0,0 +1,42 @@ +// Batched relevance classifier for the Substack discovery crawler. +// One LLM call judges many candidate profiles at once (see ScoreArticles for +// the array-in/array-out batching pattern). Classification runs on cheap +// metadata (name + description) BEFORE any per-profile post fetch, and gates +// both DB insertion and whether the crawler recurses into that profile. + +class ProfileInput { + handle string @description("Substack owner handle, echoed back in the verdict") + name string @description("Publication name") + description string? @description("Publication hero text and/or author bio, if known") +} + +class ProfileVerdict { + handle string @description("Must exactly match the input handle") + isAiRelated bool + confidence int @description("0-100 confidence that AI/ML/LLMs is the PRIMARY focus") +} + +function ClassifyProfiles(profiles: ProfileInput[]) -> ProfileVerdict[] { + client Nvidia + prompt #" + {{ _.role("system") }} + You curate sources for a developer-focused AI news platform. For each Substack + publication, decide whether artificial intelligence is its PRIMARY subject: + AI/ML/LLMs, foundation models, AI engineering, AI products, or AI research. + + Mark isAiRelated = true ONLY when AI is clearly the main topic. Mark false for + general tech, productivity, marketing, business, crypto, politics, lifestyle, or + publications that merely mention AI in passing. When the signal is thin (sparse + description, generic name), lean false and assign low confidence - the crawler + prunes everything you reject, so false positives pollute the whole subtree. + + {{ _.role("user") }} + Classify every profile. Echo each handle back EXACTLY as given. + + {% for p in profiles %} + {{ loop.index }}. handle={{ p.handle }} | {{ p.name }}{% if p.description %} — {{ p.description }}{% endif %} + {% endfor %} + + {{ ctx.output_format }} + "# +} diff --git a/baml_src/extract_links.baml b/baml_src/extract_links.baml new file mode 100644 index 0000000000..579c0d8aa4 --- /dev/null +++ b/baml_src/extract_links.baml @@ -0,0 +1,42 @@ +class ExtractedLink { + title string + url string + snippet string @description("One-sentence description of the article") +} + +function ExtractLinks(emailHtml: string) -> ExtractedLink[] { + client NvidiaGptOss + prompt #" + {{ _.role("system") }} + You are a precise link extraction engine. You parse newsletter HTML and identify external article links while filtering out tracking, admin, and promotional URLs. + + {{ _.role("user") }} + You are given the HTML body of a newsletter email. Extract all article links that point to external news stories, blog posts, research papers, or product announcements. + + For each article, return its title, URL, and a one-sentence snippet describing it. + + IGNORE these types of links: + - Tracking/redirect URLs (e.g. links through email marketing platforms) + - Unsubscribe, manage preferences, or email settings links + - Social media profile links (twitter, linkedin, etc.) + - Links back to the newsletter's own homepage, archive index, or subscription page (but DO include article/post links hosted on the newsletter's domain - curated newsletters often host their own content) + - Sponsor/advertisement links + - Job posting links + + RESOLVE redirect URLs: if a link is clearly a tracking redirect (contains /click, /track, utm_ params, etc.), try to extract the final destination URL from the href. Strip all UTM parameters from URLs. + + Newsletter HTML: + {{ emailHtml }} + + If no relevant article links are found, return an empty list. + + {{ ctx.output_format }} + "# +} + +test extract_links_basic { + functions [ExtractLinks] + args { + emailHtml "Welcome to today's issue! [Read OpenAI's new paper](https://openai.com/papers/gpt-5). [Unsubscribe](https://newsletter.com/unsubscribe)." + } +} diff --git a/baml_src/extract_products.baml b/baml_src/extract_products.baml new file mode 100644 index 0000000000..9e5f5a531a --- /dev/null +++ b/baml_src/extract_products.baml @@ -0,0 +1,125 @@ +// Product-centric newsletter extraction. +// +// Replaces the per-newsletter tracking-URL resolvers: instead of decoding each +// provider's redirect scheme to recover the real link, we extract the *named +// products* a newsletter reports on (from anchor text + surrounding prose), +// keep only the notable ones, and rediscover their canonical URLs via web +// search (src/sources/email.ts). SelectProductLink then picks the best +// first-party result among the candidates. + +class NewsletterProduct { + name string @description("Exact product / tool / model / launch name") + description string @description("Concise phrase: what it is and does, usable verbatim as a web search query") +} + +function ExtractProducts(newsletterText: string) -> NewsletterProduct[] { + client NvidiaGptOss + prompt #" + {{ _.role("system") }} + You extract the AI/tech products, tools, models, and launches that a newsletter is reporting on. Work only from the text; never invent products that are not mentioned. + + {{ _.role("user") }} + Below is the text of a newsletter email. Links appear as [anchor text](url) but the URLs are often tracking redirects, so judge by the words, not the links. + + Identify every distinct product, tool, model, SDK, dataset, or launch the newsletter is reporting as news. For each, return: + - name: the exact product/tool/model name + - description: a concise phrase describing what it is and does, suitable as a web search query + + INCLUDE only real, named launches a reader could go use or read about. + + IGNORE: + - the author's intro letter, essay, or personal commentary + - sponsor blurbs, advertisements, "brought to you by", upgrade / subscribe / manage-preferences prompts + - social-media embeds (an @handle with like / repost / reply counts) and the newsletter's own social links + - section headers, navigation, and unsubscribe links + + Newsletter text: + {{ newsletterText }} + + If the text names no qualifying products, return an empty list. + + {{ ctx.output_format }} + "# +} + +// Notability gate: trims the extracted list to the launches actually worth +// resolving, so we don't spend a web search on every incremental SDK bump. +// Returns the 1-based indices of the products to KEEP. +function SelectNotableProducts(products: NewsletterProduct[]) -> int[] { + client Nvidia + prompt #" + {{ _.role("user") }} + Below is a numbered list of products/tools/models a newsletter mentioned. Return the numbers of the ones that are NOTABLE enough to feature in an AI-news aggregator. + + KEEP a product if it is a distinct, real, individually newsworthy launch a reader would want to click: new models, significant new products or platforms, notable agents/tools, major releases. + + DROP: + - minor or incremental items: small SDK/utility bumps, niche helper libraries, design skills, config tweaks + - vague or marketing-only entries with no concrete product behind them + - duplicates or sub-features of another item in the list + + Products: + {% for p in products %} + {{ loop.index }}. {{ p.name }} — {{ p.description }} + {% endfor %} + + Return only the numbers to keep, as a JSON array of integers. If none qualify, return []. + + {{ ctx.output_format }} + "# +} + +class SearchCandidate { + title string + url string + snippet string +} + +class ProductLinkChoice { + index int @description("1-based index of the best first-party result, or 0 if none qualifies") + confidence float @description("0.0 - 1.0") +} + +// Picks the best canonical URL for a product among search candidates, biased +// hard toward the official / first-party source and away from social posts and +// SEO/aggregator pages. Returns index 0 when nothing is clearly the product. +function SelectProductLink( + productName: string, + productDescription: string, + candidates: SearchCandidate[] +) -> ProductLinkChoice { + client Nvidia + prompt #" + {{ _.role("user") }} + A newsletter featured this item: + name: {{ productName }} + description: {{ productDescription }} + + Web search returned these candidate results: + {% for c in candidates %} + {{ loop.index }}. {{ c.title }} + url: {{ c.url }} + snippet: {{ c.snippet }} + {% endfor %} + + Pick the ONE candidate that is the canonical, first-party source for this exact product — its official site, product page, model card, repo, or the vendor's own announcement. + + PREFER the official / first-party domain that matches the product or its maker. + REJECT (do not pick) when it is the best available: + - social-media posts (x.com, twitter.com, linkedin.com, reddit.com, threads, mastodon) + - videos (youtube.com) + - content-farm / SEO / generic aggregator or listicle pages that merely mention the product + - a DIFFERENT product that happens to share the name + + Return the 1-based index of the best first-party result. If none of the candidates is clearly a first-party source for THIS product, return index 0. + + {{ ctx.output_format }} + "# +} + +test extract_products_basic { + functions [ExtractProducts] + args { + newsletterText "Headlines: [OpenAI ships GPT-5](https://sub.stack/redirect/2/abc), a new flagship model. Brought to you by Acme. [Unsubscribe](https://newsletter.com/unsub)." + } +} diff --git a/baml_src/fix_titles.baml b/baml_src/fix_titles.baml new file mode 100644 index 0000000000..9239b53d20 --- /dev/null +++ b/baml_src/fix_titles.baml @@ -0,0 +1,61 @@ +class TitleFix { + url string @description("The exact URL from the matching input article - copy verbatim") + title string @description("Plain title text only. No surrounding quotes. No leading source name or bracket tag. Max 12 words. Return the input title verbatim if it is already clear, specific, and informative.") +} + +function ImproveTitles(articles: ArticleInput[]) -> TitleFix[] { + client NvidiaGptOss + prompt #" + {{ _.role("system") }} + You are a technical editor. You write clear, informative article titles for an AI developer news platform. + + Output rules (strict): + - For each input article, return one TitleFix whose `url` exactly matches that article's URL. + - The `title` field contains plain title text ONLY. Do NOT wrap it in quotes. Do NOT prefix it with "[Source]" or any source name. Do NOT include the URL. + - Use only the snippet of the SAME article to inform the rewrite. Never blend information across articles. + + {{ _.role("user") }} + Decide for each article whether the existing title is already good. A good title is specific, informative, and technical/neutral in register - it tells a developer what the article is actually about. + + - If the existing title is already good, return it verbatim. + - Otherwise rewrite it using only that article's snippet: specific, informative, max 12 words, technical/neutral register, no hype or filler. + + {% for a in articles %} + --- Article {{ loop.index }} --- + url: {{ a.url }} + source: {{ a.source }} + current_title: {{ a.title }} + {% if a.snippet %}snippet: {{ a.snippet }}{% endif %} + {% endfor %} + + {{ ctx.output_format }} + "# +} + +test improve_titles_rewrite_short { + functions [ImproveTitles] + args { + articles [ + { + url "https://example.com/claude" + title "Claude 4" + source "HN" + snippet "Anthropic today released Claude 4, a new flagship model with improved reasoning and a 1M-token context window." + } + ] + } +} + +test improve_titles_keep_unchanged { + functions [ImproveTitles] + args { + articles [ + { + url "https://example.com/post" + title "Anthropic releases Claude 4 with 1M-token context window" + source "HN" + snippet "Anthropic today released Claude 4, a new flagship model with improved reasoning." + } + ] + } +} diff --git a/baml_src/generators.baml b/baml_src/generators.baml new file mode 100644 index 0000000000..c5ab04792f --- /dev/null +++ b/baml_src/generators.baml @@ -0,0 +1,5 @@ +generator python { + output_type "python/pydantic" + output_dir "../backend/" + version "0.223.0" +} diff --git a/baml_src/score.baml b/baml_src/score.baml new file mode 100644 index 0000000000..01d73713df --- /dev/null +++ b/baml_src/score.baml @@ -0,0 +1,84 @@ +class ScoredArticle { + url string + score int @description("1-100 developer-actionability rating") +} + +function ScoreArticles(articles: ArticleInput[]) -> ScoredArticle[] { + client Nvidia + prompt #" + {{ _.role("system") }} + You are a senior editorial analyst at a developer news platform. Your job is to evaluate articles for relevance and actionability for AI builders. You are precise, consistent, and ruthless about filtering noise. + + {{ _.role("user") }} + Score every article for developer-actionability on a 1-100 scale. + + STEP 1 - SCOPE CHECK (apply first, before scoring): + Is the article about AI, ML, LLMs, foundation models, or AI-adjacent developer tools/infrastructure? + - NO → score 1-20 and move on. Do NOT apply the scoring guide below. + - Out-of-scope examples: text editors, parsers, version control tools, data structures (CRDTs, B-trees), memory management, general programming tutorials, non-AI open source projects, codebase audits, legacy code. These are general software engineering - not AI. + + STEP 2 - SCORE (only for in-scope articles): + Audience: developers, startup founders, and tech leaders building with AI who want to ship faster. + + Score 91-100 is reserved for exceptional articles that are BOTH immediately actionable today AND fit one of these themes: local/on-device inference, privacy-preserving AI, efficiency breakthroughs (quantization, token optimization, running models on less hardware), or deep technical dives that explain how something actually works. A score in this range means "a developer could open a terminal and act on this within the hour, AND it reveals a non-obvious insight or technique." Zero is a valid outcome - do not give 91+ just because something is the best of a weak batch. + + Models (new releases, benchmarks, evals, model cards, comparisons): + - 81-90: New model release or major update, model system cards, open-weight models a developer can run today + - 71-80: Meaningful benchmark or eval, model comparison with actionable conclusions + + Dev (SDKs, frameworks, CLI tools, IDE integrations, open-source projects, tutorials, deployment): + - 81-90: New AI API or SDK, open-source AI tool/library a developer can use today + - 71-80: Fine-tuning guide, RAG pattern, significant framework update, AI infrastructure (training, inference, memory optimization), deployment recipes + + Research (papers, breakthroughs, novel techniques): + - 71-80: Research with immediate practical implications, AI security research + - 51-60: Research with near-term practical implications + + Industry (funding, platform changes, acquisitions, partnerships, market shifts): + - 51-60: Industry analysis with actionable takeaways, AI platform partnerships + - 31-40: Funding rounds without product details, opinion/commentary with no actionable steps, "AI is changing everything" think pieces, personal anecdotes about using AI + + Upgrade to 91-100: if an article would score 81-90 AND fits the top-tier criteria above (local inference, privacy-preserving, efficiency breakthrough, or deep how-it-works dive), give it 91-100. + + Bottom tier: + - 1-20: Non-AI topics (see step 1), regulation, politics, lawsuits, corporate drama + + Trust bonus: entries may include a [trust: high/medium/low] tag. Give +1 to high-trust sources when content quality is comparable. + + Articles: + {% for a in articles %} + {{ loop.index }}. [{{ a.source }}] "{{ a.title }}" - {{ a.url }}{% if a.trust %} [trust: {{ a.trust }}]{% endif %} + {% if a.snippet %} {{ a.snippet }}{% endif %} + {% endfor %} + + {{ ctx.output_format }} + "# +} + +test score_gpt5_release { + functions [ScoreArticles] + args { + articles [ + { + url "https://openai.com/index/introducing-gpt-5" + title "Introducing GPT-5" + source "Hacker News" + trust "medium" + } + ] + } +} + +test score_out_of_scope { + functions [ScoreArticles] + args { + articles [ + { + url "https://example.com/crdt-deep-dive" + title "A Deep Dive Into CRDTs and Conflict Resolution" + source "Some Blog" + trust "low" + } + ] + } +} diff --git a/baml_src/shared.baml b/baml_src/shared.baml new file mode 100644 index 0000000000..46d5b8ff35 --- /dev/null +++ b/baml_src/shared.baml @@ -0,0 +1,33 @@ +// Shared types used by multiple BAML functions. + +enum ArticleCategory { + Models + Dev + Research +} + +enum ArticleKind { + Repo // GitHub/GitLab repository + Paper // Academic paper, preprint, technical report + Model // Model card or model release page + Blog // Developer blog post, tutorial, how-to, opinion piece + Product // Product launch page or product website + Announcement // Official release post from a major AI lab +} + +// Used as input to ScoreArticles, ImproveTitles, SemanticDedup. +// Mirrors the fields the existing formatArticles() helper emits. +class ArticleInput { + url string + title string + source string + snippet string? @description("First ~200 chars of the article content, if any") + trust string? @description("\"high\" | \"medium\" | \"low\" - source trust tag") +} + +// Used as input to SemanticDedup for rows already in the DB. +class ExistingArticle { + url string + title string + source string +} diff --git a/baml_src/summarize.baml b/baml_src/summarize.baml new file mode 100644 index 0000000000..59fecb7e0b --- /dev/null +++ b/baml_src/summarize.baml @@ -0,0 +1,111 @@ +class SummarizeAndCategorizeResult { + summary string @description("2-3 short factual lines separated by \\n") + categories ArticleCategory[] @description("1-2 categories") + kind ArticleKind +} + +class CategorizeOnlyResult { + categories ArticleCategory[] @description("1-2 categories") +} + +class ClassifyKindResult { + kind ArticleKind +} + +function SummarizeAndCategorize(title: string, content: string) -> SummarizeAndCategorizeResult { + client NvidiaGptOss + prompt #" + {{ _.role("system") }} + You are a technical writer for a developer news digest. Your readers are tech-savvy but busy. Write in plain, direct language. + + {{ _.role("user") }} + Summarize this article. Be plain and brief. + + Title: {{ title }} + + Content: + {{ content }} + + 1. Write 2-3 lines. Keep each line to one short sentence. Rules: + - Only state facts from the article. + - No passive voice. No "enables", "allows", "provides". + - Separate lines with a newline. + 2. Assign 1-2 categories. + 3. Assign a kind. + + Category definitions: + - Models: New model releases, benchmarks, evals, model cards, model comparisons + - Dev: Repos, tools, product launches, developer blogs, tutorials, how-tos, fine-tuning guides, deployment recipes, infrastructure updates + - Research: Papers, breakthroughs, novel techniques with practical implications + + Kind definitions: + - Repo: GitHub or GitLab repository + - Paper: Academic paper, preprint, or long-form technical report + - Model: Model card or model release page (e.g. HuggingFace) + - Blog: Developer blog post, tutorial, how-to, or opinion piece + - Product: Product launch page or product website + - Announcement: Official release post from a major AI lab (OpenAI, Anthropic, Google, Meta, Mistral, etc.) + + {{ ctx.output_format }} + "# +} + +function ClassifyKind(title: string, url: string, summary: string?) -> ClassifyKindResult { + client NvidiaGptOss + prompt #" + {{ _.role("system") }} + You are a technical librarian who classifies AI developer content by format. + + {{ _.role("user") }} + Classify this article's kind. + + Title: {{ title }} + URL: {{ url }} + {% if summary %}Summary: {{ summary }}{% endif %} + + Kind definitions: + - Repo: GitHub or GitLab repository + - Paper: Academic paper, preprint, or long-form technical report + - Model: Model card or model release page (e.g. HuggingFace) + - Blog: Developer blog post, tutorial, how-to, or opinion piece + - Product: Product launch page or product website + - Announcement: Official release post from a major AI lab (OpenAI, Anthropic, Google, Meta, Mistral, etc.) + + {{ ctx.output_format }} + "# +} + +function CategorizeOnly(title: string) -> CategorizeOnlyResult { + client NvidiaGptOss + prompt #" + {{ _.role("system") }} + You are a technical librarian who classifies developer-focused AI articles. + + {{ _.role("user") }} + Assign 1-2 categories to this article title. + + Title: {{ title }} + + Category definitions: + - Models: New model releases, benchmarks, evals, model cards, model comparisons + - Dev: Repos, tools, product launches, developer blogs, tutorials, how-tos, fine-tuning guides, deployment recipes, infrastructure updates + - Research: Papers, breakthroughs, novel techniques with practical implications + + {{ ctx.output_format }} + "# +} + +test summarize_basic { + functions [SummarizeAndCategorize] + args { + title "Anthropic releases Claude 4" + content "Anthropic today launched Claude 4, a new flagship model. It features improved reasoning, a 1M-token context window, and native tool use. The model is available via API and Claude.ai starting today." + } +} + +test categorize_only_basic { + functions [CategorizeOnly] + args { + title "OpenAI releases GPT-5 with 2M context" + } +} diff --git a/biome.json b/biome.json new file mode 100644 index 0000000000..915c9f4e90 --- /dev/null +++ b/biome.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.3.14/schema.json", + "extends": ["./frontend/biome.json"] +} diff --git a/compose.yml b/compose.yml index 2488fc007b..5d9233e964 100644 --- a/compose.yml +++ b/compose.yml @@ -1,7 +1,7 @@ services: db: - image: postgres:18 + image: pgvector/pgvector:pg17 restart: always healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] @@ -58,6 +58,7 @@ services: env_file: - .env environment: + - PROJECT_NAME=${PROJECT_NAME} - DOMAIN=${DOMAIN} - FRONTEND_HOST=${FRONTEND_HOST?Variable not set} - ENVIRONMENT=${ENVIRONMENT} @@ -69,6 +70,8 @@ services: - SMTP_USER=${SMTP_USER} - SMTP_PASSWORD=${SMTP_PASSWORD} - EMAILS_FROM_EMAIL=${EMAILS_FROM_EMAIL} + - RESEND_API_KEY=${RESEND_API_KEY} + - RESEND_AUDIENCE_ID=${RESEND_AUDIENCE_ID} - POSTGRES_SERVER=db - POSTGRES_PORT=${POSTGRES_PORT} - POSTGRES_DB=${POSTGRES_DB} @@ -91,6 +94,7 @@ services: env_file: - .env environment: + - PROJECT_NAME=${PROJECT_NAME} - DOMAIN=${DOMAIN} - FRONTEND_HOST=${FRONTEND_HOST?Variable not set} - ENVIRONMENT=${ENVIRONMENT} @@ -102,6 +106,8 @@ services: - SMTP_USER=${SMTP_USER} - SMTP_PASSWORD=${SMTP_PASSWORD} - EMAILS_FROM_EMAIL=${EMAILS_FROM_EMAIL} + - RESEND_API_KEY=${RESEND_API_KEY} + - RESEND_AUDIENCE_ID=${RESEND_AUDIENCE_ID} - POSTGRES_SERVER=db - POSTGRES_PORT=${POSTGRES_PORT} - POSTGRES_DB=${POSTGRES_DB} @@ -155,16 +161,56 @@ services: - traefik.http.services.${STACK_NAME?Variable not set}-frontend.loadbalancer.server.port=80 - - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.rule=Host(`dashboard.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.rule=Host(`${DOMAIN?Variable not set}`) - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.entrypoints=http - - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-https.rule=Host(`dashboard.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-https.rule=Host(`${DOMAIN?Variable not set}`) - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-https.entrypoints=https - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-https.tls=true - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-https.tls.certresolver=le # Enable redirection for HTTP and HTTPS - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.middlewares=https-redirect + + # Redirect www.${DOMAIN} to the bare domain (separate routers so the "le" + # resolver issues a cert for the www host too, instead of falling back + # to Traefik's default self-signed cert) + - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-www-http.rule=Host(`www.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-www-http.entrypoints=http + - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-www-http.middlewares=https-redirect + + - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-www-https.rule=Host(`www.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-www-https.entrypoints=https + - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-www-https.tls=true + - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-www-https.tls.certresolver=le + - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-www-https.middlewares=${STACK_NAME?Variable not set}-www-redirect + + - traefik.http.middlewares.${STACK_NAME?Variable not set}-www-redirect.redirectregex.regex=^https://www\.(.+) + - traefik.http.middlewares.${STACK_NAME?Variable not set}-www-redirect.redirectregex.replacement=https://$${1} + - traefik.http.middlewares.${STACK_NAME?Variable not set}-www-redirect.redirectregex.permanent=true + pipeline: + image: '${DOCKER_IMAGE_BACKEND?Variable not set}:${TAG-latest}' + restart: always + networks: + - default + depends_on: + db: + condition: service_healthy + prestart: + condition: service_completed_successfully + env_file: + - .env + environment: + - SHELL=/bin/sh + - POSTGRES_SERVER=db + - POSTGRES_PORT=${POSTGRES_PORT} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER?Variable not set} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD?Variable not set} + - NVIDIA_NIM_API_KEY=${NVIDIA_NIM_API_KEY} + - RESIDENTIAL_PROXY_URL=${RESIDENTIAL_PROXY_URL} + command: supercronic -no-reap /app/backend/pipeline/crontab + volumes: app-db-data: diff --git a/docs/about_this_project.md b/docs/about_this_project.md new file mode 100644 index 0000000000..8897e20db2 --- /dev/null +++ b/docs/about_this_project.md @@ -0,0 +1,101 @@ +# Agentique — what this project is + +Agentique (agentique.ch) is an AI news feed for people building with AI: developers, +founders, and tech leads who want the signal without wading through hype. Instead of +scrolling ten newsletters and three subreddits, you get one feed of articles that have +already been filtered, scored, summarized, and tagged — and you can semantically search +across all of it. + +Think of it as an automated editorial desk: a robot intern that reads everything on the +internet about AI every night, throws out the noise, and hands you a short, ranked +digest by morning. + +## What it does today + +**A filtered, ranked article feed.** The homepage lists recent articles sorted by an +LLM-assigned "developer-actionability" score (or by recency). Each entry shows a +2–3 line factual summary, a category (Models / Dev / Research), and a "kind" (repo, +paper, model, blog, product, announcement). You can filter by category, kind, score, +and time window (last 3 days / week / month). + +**Semantic search.** Typing a query searches by meaning, not keyword — it embeds your +query and finds the nearest articles in vector space, so "how do I run a model locally" +surfaces relevant pieces even if none of them use that exact phrase. + +**Newsletter signup.** Visitors can subscribe with an email + category preferences; +subscribers are synced to a Resend audience for future digest emails (the sending side +isn't built yet — this is capture only, for now). + +**A public API.** Everything above is backed by a documented REST API (auto-generated +OpenAPI/Swagger) that anyone could build a client against — a Slack bot, a CLI, a +personal dashboard. + +## How the pipeline works + +Once a day, a scheduled job goes out, gathers candidate articles from a handful of +sources (Hacker News, an AI-news aggregator feed, and a curated list of Substack +newsletters), and runs each fresh batch through a chain of small, focused steps: + +1. **Skip anything already seen** — URLs already in the database, or already evaluated + and rejected before, are dropped immediately. +2. **Drop dead links** — a quick DNS check filters out URLs whose domains no longer + resolve. +3. **Deduplicate by meaning** — an LLM compares new articles against everything + published in the last two weeks and drops ones that are "the same story," even if + the wording or source differs. +4. **Score for relevance** — an LLM rates each surviving article 1–100 on how + actionable it is for a developer building with AI right now. Only the top scorers + (currently ≥76) make it into the database at all — this is the main noise filter. +5. **Insert & clean up the title** — the article is saved, then a second LLM pass + tightens up clickbait-y or vague titles into something plain and informative. +6. **Pull the full article text** — for sources that only gave us a link, the pipeline + fetches and extracts the actual article body (skipping paywalled junk, ads, nav). +7. **Summarize & categorize** — another LLM pass turns the full text into the 2–3 line + summary and assigns category + kind tags shown in the feed. +8. **Embed** — a small, fast local embedding model turns the title + summary into a + vector, which is what powers semantic search. + +All the LLM steps are defined declaratively as prompt functions (via BAML) rather than +hand-rolled prompt strings scattered through the code, so tweaking a prompt or swapping +a model is a config change, not a refactor. + +There are a few more pipeline building blocks already defined (a Substack discovery +crawler that finds new AI-focused publications to follow, and newsletter-email +ingestion that extracts article links/products out of HTML emails) that exist but +aren't wired into the nightly run yet — candidates for whoever picks up sourcing next. + +## Under the hood, briefly + +- **Backend**: FastAPI (Python), auto-generates the OpenAPI spec the frontend client is + built from. +- **Database**: Postgres with the pgvector extension — one table holds articles plus + their embedding vectors, so relevance search is just a SQL query. +- **Pipeline**: a separate scheduled Python process (cron-style, once a day) that does + the fetch → filter → score → summarize → embed work described above. +- **Embeddings**: model2vec — a tiny, fast, CPU-only static embedding model, no GPU or + external API call needed for search. +- **Frontend**: React + Vite + Tailwind, talking to the backend through a generated + TypeScript client. +- **Deploy**: Docker Compose on a single VPS — db, backend, pipeline, frontend, and an + Adminer UI for poking at the database directly. + +The whole thing is a fork of a well-known open-source FastAPI starter template, so a lot +of the plumbing (auth scaffolding, migrations, project layout, CI) is inherited rather +than built from scratch — Agentique-specific logic lives in a small number of clearly +separate files rather than scattered through upstream code, which keeps it easy to pull +in upstream improvements later. + +## Where things stand / what's next + +The product today is deliberately narrow: one public feed, one search endpoint, one +capture-only newsletter form, no user accounts. Things explicitly not built yet (and +worth knowing about if you're picking up work here): sending the actual newsletter +digest, user accounts / saved articles / personalization, an MCP server so AI agents can +query the feed directly, the Substack discovery crawler and email-newsletter ingestion +mentioned above, and a real load/scale test (current data volume is small — this has +been an architecture spike more than a performance one so far). + +If you want the ground-level detail — endpoints, models, exact commands — see +`backend/README.md`, `frontend/README.md`, and `development.md`. `CHANGES.md` tracks +every place this fork has diverged from the upstream template, which is the fastest way +to see what's actually Agentique-specific versus inherited scaffolding. diff --git a/frontend/public/assets/images/favicon.png b/frontend/public/assets/images/favicon.png index e5b7c3ada7..a42cc28daf 100644 Binary files a/frontend/public/assets/images/favicon.png and b/frontend/public/assets/images/favicon.png differ diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index ea7b4f5cff..6ffb461829 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -1,5 +1,133 @@ // This file is auto-generated by @hey-api/openapi-ts +export const ArticlePublicSchema = { + properties: { + title: { + type: 'string', + title: 'Title' + }, + source: { + type: 'string', + title: 'Source' + }, + source_type: { + type: 'string', + title: 'Source Type' + }, + url: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Url' + }, + published_at: { + anyOf: [ + { + type: 'string', + format: 'date-time' + }, + { + type: 'null' + } + ], + title: 'Published At' + }, + score: { + anyOf: [ + { + type: 'integer' + }, + { + type: 'null' + } + ], + title: 'Score' + }, + summary: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Summary' + }, + categories: { + items: { + type: 'string' + }, + type: 'array', + title: 'Categories' + }, + kind: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Kind' + }, + id: { + type: 'integer', + title: 'Id' + }, + created_at: { + anyOf: [ + { + type: 'string', + format: 'date-time' + }, + { + type: 'null' + } + ], + title: 'Created At' + }, + like_count: { + type: 'integer', + title: 'Like Count', + default: 0 + }, + liked_by_me: { + type: 'boolean', + title: 'Liked By Me', + default: false + } + }, + type: 'object', + required: ['title', 'source', 'source_type', 'id'], + title: 'ArticlePublic' +} as const; + +export const ArticlesPublicSchema = { + properties: { + data: { + items: { + '$ref': '#/components/schemas/ArticlePublic' + }, + type: 'array', + title: 'Data' + }, + count: { + type: 'integer', + title: 'Count' + } + }, + type: 'object', + required: ['data', 'count'], + title: 'ArticlesPublic' +} as const; + export const Body_login_login_access_tokenSchema = { properties: { grant_type: { @@ -226,6 +354,53 @@ export const NewPasswordSchema = { title: 'NewPassword' } as const; +export const NewsletterSubscribeRequestSchema = { + properties: { + email: { + type: 'string', + title: 'Email' + }, + categories: { + items: { + type: 'string' + }, + type: 'array', + title: 'Categories' + }, + customCategory: { + type: 'string', + title: 'Customcategory', + default: '' + }, + utm_source: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Utm Source' + } + }, + type: 'object', + required: ['email'], + title: 'NewsletterSubscribeRequest' +} as const; + +export const NewsletterSubscribeResponseSchema = { + properties: { + ok: { + type: 'boolean', + title: 'Ok', + default: true + } + }, + type: 'object', + title: 'NewsletterSubscribeResponse' +} as const; + export const PrivateUserCreateSchema = { properties: { email: { diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index ba79e3f726..48f3c93999 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -3,7 +3,73 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { ItemsReadItemsData, ItemsReadItemsResponse, ItemsCreateItemData, ItemsCreateItemResponse, ItemsReadItemData, ItemsReadItemResponse, ItemsUpdateItemData, ItemsUpdateItemResponse, ItemsDeleteItemData, ItemsDeleteItemResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, PrivateCreateUserData, PrivateCreateUserResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse } from './types.gen'; +import type { ArticlesReadArticlesData, ArticlesReadArticlesResponse, ArticlesSearchArticlesData, ArticlesSearchArticlesResponse, ArticlesArticleStatsResponse, ItemsReadItemsData, ItemsReadItemsResponse, ItemsCreateItemData, ItemsCreateItemResponse, ItemsReadItemData, ItemsReadItemResponse, ItemsUpdateItemData, ItemsUpdateItemResponse, ItemsDeleteItemData, ItemsDeleteItemResponse, LikesLikeArticleData, LikesLikeArticleResponse, LikesUnlikeArticleData, LikesUnlikeArticleResponse, LikesReadLikedArticlesResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, NewsletterSubscribeData, NewsletterSubscribeResponse2, PrivateCreateUserData, PrivateCreateUserResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse } from './types.gen'; + +export class ArticlesService { + /** + * Read Articles + * @param data The data for the request. + * @param data.limit + * @param data.since + * @param data.minScore + * @param data.category + * @param data.kind + * @param data.sort + * @returns ArticlesPublic Successful Response + * @throws ApiError + */ + public static readArticles(data: ArticlesReadArticlesData = {}): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/articles/', + query: { + limit: data.limit, + since: data.since, + min_score: data.minScore, + category: data.category, + kind: data.kind, + sort: data.sort + }, + errors: { + 422: 'Validation Error' + } + }); + } + + /** + * Search Articles + * @param data The data for the request. + * @param data.q + * @param data.limit + * @returns ArticlesPublic Successful Response + * @throws ApiError + */ + public static searchArticles(data: ArticlesSearchArticlesData): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/articles/search', + query: { + q: data.q, + limit: data.limit + }, + errors: { + 422: 'Validation Error' + } + }); + } + + /** + * Article Stats + * @returns unknown Successful Response + * @throws ApiError + */ + public static articleStats(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/articles/stats' + }); + } +} export class ItemsService { /** @@ -116,6 +182,60 @@ export class ItemsService { } } +export class LikesService { + /** + * Like Article + * @param data The data for the request. + * @param data.articleId + * @returns unknown Successful Response + * @throws ApiError + */ + public static likeArticle(data: LikesLikeArticleData): CancelablePromise { + return __request(OpenAPI, { + method: 'PUT', + url: '/api/v1/articles/{article_id}/like', + path: { + article_id: data.articleId + }, + errors: { + 422: 'Validation Error' + } + }); + } + + /** + * Unlike Article + * @param data The data for the request. + * @param data.articleId + * @returns unknown Successful Response + * @throws ApiError + */ + public static unlikeArticle(data: LikesUnlikeArticleData): CancelablePromise { + return __request(OpenAPI, { + method: 'DELETE', + url: '/api/v1/articles/{article_id}/like', + path: { + article_id: data.articleId + }, + errors: { + 422: 'Validation Error' + } + }); + } + + /** + * Read Liked Articles + * @returns ArticlesPublic Successful Response + * @throws ApiError + */ + public static readLikedArticles(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/me/liked-articles' + }); + } +} + export class LoginService { /** * Login Access Token @@ -213,6 +333,27 @@ export class LoginService { } } +export class NewsletterService { + /** + * Subscribe + * @param data The data for the request. + * @param data.requestBody + * @returns NewsletterSubscribeResponse Successful Response + * @throws ApiError + */ + public static subscribe(data: NewsletterSubscribeData): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/api/newsletter/subscribe', + body: data.requestBody, + mediaType: 'application/json', + errors: { + 422: 'Validation Error' + } + }); + } +} + export class PrivateService { /** * Create User diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 596b8fc476..e91fac3796 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -1,5 +1,26 @@ // This file is auto-generated by @hey-api/openapi-ts +export type ArticlePublic = { + title: string; + source: string; + source_type: string; + url?: (string | null); + published_at?: (string | null); + score?: (number | null); + summary?: (string | null); + categories?: Array<(string)>; + kind?: (string | null); + id: number; + created_at?: (string | null); + like_count?: number; + liked_by_me?: boolean; +}; + +export type ArticlesPublic = { + data: Array; + count: number; +}; + export type Body_login_login_access_token = { grant_type?: (string | null); username: string; @@ -45,6 +66,17 @@ export type NewPassword = { new_password: string; }; +export type NewsletterSubscribeRequest = { + email: string; + categories?: Array<(string)>; + customCategory?: string; + utm_source?: (string | null); +}; + +export type NewsletterSubscribeResponse = { + ok?: boolean; +}; + export type PrivateUserCreate = { email: string; password: string; @@ -113,6 +145,26 @@ export type ValidationError = { }; }; +export type ArticlesReadArticlesData = { + category?: (string | null); + kind?: (string | null); + limit?: number; + minScore?: (number | null); + since?: (string | null); + sort?: string; +}; + +export type ArticlesReadArticlesResponse = (ArticlesPublic); + +export type ArticlesSearchArticlesData = { + limit?: number; + q: string; +}; + +export type ArticlesSearchArticlesResponse = (ArticlesPublic); + +export type ArticlesArticleStatsResponse = (unknown); + export type ItemsReadItemsData = { limit?: number; skip?: number; @@ -145,6 +197,20 @@ export type ItemsDeleteItemData = { export type ItemsDeleteItemResponse = (Message); +export type LikesLikeArticleData = { + articleId: number; +}; + +export type LikesLikeArticleResponse = (unknown); + +export type LikesUnlikeArticleData = { + articleId: number; +}; + +export type LikesUnlikeArticleResponse = (unknown); + +export type LikesReadLikedArticlesResponse = (ArticlesPublic); + export type LoginLoginAccessTokenData = { formData: Body_login_login_access_token; }; @@ -171,6 +237,12 @@ export type LoginRecoverPasswordHtmlContentData = { export type LoginRecoverPasswordHtmlContentResponse = (string); +export type NewsletterSubscribeData = { + requestBody: NewsletterSubscribeRequest; +}; + +export type NewsletterSubscribeResponse2 = (NewsletterSubscribeResponse); + export type PrivateCreateUserData = { requestBody: PrivateUserCreate; }; diff --git a/frontend/src/components/Articles/ArticleRow.tsx b/frontend/src/components/Articles/ArticleRow.tsx new file mode 100644 index 0000000000..c8f63297bb --- /dev/null +++ b/frontend/src/components/Articles/ArticleRow.tsx @@ -0,0 +1,63 @@ +import type { ArticlePublic } from "@/client" +import { LikeButton } from "./LikeButton" + +export function ArticleRow({ article }: { article: ArticlePublic }) { + return ( +
  • +
    + {article.source} + {article.score != null && ( + <> + · + score {article.score} + + )} + {article.published_at && ( + <> + · + + {new Date(article.published_at).toLocaleDateString("en-US", { + month: "short", + day: "numeric", + })} + + + )} + {article.kind && ( + <> + · + {article.kind} + + )} +
    + + {article.title} + + {article.summary && ( +

    + {article.summary} +

    + )} +
    + {article.categories && article.categories.length > 0 && ( +
    + {article.categories.map((cat: string) => ( + + {cat} + + ))} +
    + )} + +
    +
    • + ) +} diff --git a/frontend/src/components/Articles/ArticlesList.tsx b/frontend/src/components/Articles/ArticlesList.tsx new file mode 100644 index 0000000000..a4ab4be701 --- /dev/null +++ b/frontend/src/components/Articles/ArticlesList.tsx @@ -0,0 +1,82 @@ +import { keepPreviousData, useQuery } from "@tanstack/react-query" +import { ArticlesService } from "@/client" +import { useFilters } from "@/context/filters" +import { cn } from "@/lib/utils" +import { ArticleRow } from "./ArticleRow" + +const PUBLISHED_DAYS: Record = { "3d": 3, "1w": 7, "1m": 30 } + +function cutoffIso(days: number): string { + const d = new Date() + d.setDate(d.getDate() - days) + return d.toISOString() +} + +export function ArticlesList() { + const { filters } = useFilters() + const { search, dateRange, sort, category, kind } = filters + + const since = cutoffIso(PUBLISHED_DAYS[dateRange] ?? 7) + + const { data, isLoading, isFetching, isError } = useQuery({ + queryKey: ["articles", search, dateRange, sort, category, kind], + queryFn: () => { + if (search) { + return ArticlesService.searchArticles({ q: search, limit: 50 }) + } + return ArticlesService.readArticles({ + limit: 50, + since, + sort, + category: category || undefined, + kind: kind || undefined, + }) + }, + placeholderData: keepPreviousData, + }) + + if (isLoading) { + return null + } + + if (isError || !data) { + return ( +
    + Failed to load articles. +
    + ) + } + + const articles = data.data + + return ( +
    + {articles.length === 0 ? ( +
    + No articles found. +
    + ) : ( +
      + {articles.map((article) => ( + + ))} +
    + )} + + {!isFetching && articles.length > 0 && ( +

    + {articles.length} article{articles.length !== 1 ? "s" : ""} +

    + )} +
    + ) +} diff --git a/frontend/src/components/Articles/LikeButton.tsx b/frontend/src/components/Articles/LikeButton.tsx new file mode 100644 index 0000000000..b3cea8ece2 --- /dev/null +++ b/frontend/src/components/Articles/LikeButton.tsx @@ -0,0 +1,120 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query" +import { useNavigate, useRouterState } from "@tanstack/react-router" +import { Flame } from "lucide-react" + +import { + type ApiError, + type ArticlePublic, + type ArticlesPublic, + LikesService, +} from "@/client" +import { isLoggedIn } from "@/hooks/useAuth" +import useCustomToast from "@/hooks/useCustomToast" +import { cn } from "@/lib/utils" +import { handleError } from "@/utils" + +function patchArticle( + data: ArticlesPublic | undefined, + articleId: number, + liked: boolean, + delta: number, +): ArticlesPublic | undefined { + if (!data) return data + return { + ...data, + data: data.data.map((a) => + a.id === articleId + ? { + ...a, + liked_by_me: liked, + like_count: Math.max(0, (a.like_count ?? 0) + delta), + } + : a, + ), + } +} + +export function LikeButton({ article }: { article: ArticlePublic }) { + const navigate = useNavigate() + const routerState = useRouterState() + const queryClient = useQueryClient() + const { showErrorToast } = useCustomToast() + + const liked = article.liked_by_me ?? false + const count = article.like_count ?? 0 + + const mutation = useMutation({ + mutationFn: () => + liked + ? LikesService.unlikeArticle({ articleId: article.id }) + : LikesService.likeArticle({ articleId: article.id }), + onMutate: async () => { + await queryClient.cancelQueries({ queryKey: ["articles"] }) + await queryClient.cancelQueries({ queryKey: ["liked-articles"] }) + const previous = [ + ...queryClient.getQueriesData({ + queryKey: ["articles"], + }), + ...queryClient.getQueriesData({ + queryKey: ["liked-articles"], + }), + ] + queryClient.setQueriesData( + { queryKey: ["articles"] }, + (old) => patchArticle(old, article.id, !liked, liked ? -1 : 1), + ) + if (liked) { + queryClient.setQueriesData( + { queryKey: ["liked-articles"] }, + (old) => + old + ? { + ...old, + data: old.data.filter((a) => a.id !== article.id), + count: Math.max(0, old.count - 1), + } + : old, + ) + } + return { previous } + }, + onError: (err, _vars, context) => { + context?.previous?.forEach(([key, data]) => { + queryClient.setQueryData(key, data) + }) + handleError.call(showErrorToast, err as ApiError) + }, + onSettled: () => { + queryClient.invalidateQueries({ queryKey: ["articles"] }) + queryClient.invalidateQueries({ queryKey: ["liked-articles"] }) + }, + }) + + function handleClick() { + if (!isLoggedIn()) { + const redirect = + routerState.location.pathname + routerState.location.searchStr + navigate({ to: "/login", search: { redirect } }) + return + } + mutation.mutate() + } + + return ( + + ) +} diff --git a/frontend/src/components/Common/Appearance.tsx b/frontend/src/components/Common/Appearance.tsx index 1c56f6c410..75de3aecbc 100644 --- a/frontend/src/components/Common/Appearance.tsx +++ b/frontend/src/components/Common/Appearance.tsx @@ -1,105 +1,49 @@ -import { Monitor, Moon, Sun } from "lucide-react" +import { Moon, Sun } from "lucide-react" -import { type Theme, useTheme } from "@/components/theme-provider" +import { useTheme } from "@/components/theme-provider" import { Button } from "@/components/ui/button" -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu" -import { - SidebarMenuButton, - SidebarMenuItem, - useSidebar, -} from "@/components/ui/sidebar" - -type LucideIcon = React.FC> - -const ICON_MAP: Record = { - system: Monitor, - light: Sun, - dark: Moon, -} +import { SidebarMenuButton, SidebarMenuItem } from "@/components/ui/sidebar" export const SidebarAppearance = () => { - const { isMobile } = useSidebar() - const { setTheme, theme } = useTheme() - const Icon = ICON_MAP[theme] + const { resolvedTheme, setTheme } = useTheme() return ( - - - - - Appearance - Toggle theme - - - - setTheme("light")} - > - - Light - - setTheme("dark")} - > - - Dark - - setTheme("system")}> - - System - - - + setTheme(resolvedTheme === "dark" ? "light" : "dark")} + > + {resolvedTheme === "dark" ? ( + + ) : ( + + )} + {resolvedTheme === "dark" ? "Dark" : "Light"} + Toggle theme + ) } export const Appearance = () => { - const { setTheme } = useTheme() + const { resolvedTheme, setTheme } = useTheme() return (
    - - - - - - setTheme("light")} - > - - Light - - setTheme("dark")} - > - - Dark - - setTheme("system")}> - - System - - - +
    ) } diff --git a/frontend/src/components/Common/AuthLayout.tsx b/frontend/src/components/Common/AuthLayout.tsx index 4551610267..0b894146b9 100644 --- a/frontend/src/components/Common/AuthLayout.tsx +++ b/frontend/src/components/Common/AuthLayout.tsx @@ -10,7 +10,7 @@ export function AuthLayout({ children }: AuthLayoutProps) { return (
    - +
    diff --git a/frontend/src/components/Common/Footer.tsx b/frontend/src/components/Common/Footer.tsx index 279e1e7628..b4452a5afa 100644 --- a/frontend/src/components/Common/Footer.tsx +++ b/frontend/src/components/Common/Footer.tsx @@ -1,42 +1,59 @@ -import { FaGithub, FaLinkedinIn } from "react-icons/fa" -import { FaXTwitter } from "react-icons/fa6" +import { useQuery } from "@tanstack/react-query" -const socialLinks = [ - { - icon: FaGithub, - href: "https://github.com/fastapi/fastapi", - label: "GitHub", - }, - { icon: FaXTwitter, href: "https://x.com/fastapi", label: "X" }, - { - icon: FaLinkedinIn, - href: "https://linkedin.com/company/fastapi", - label: "LinkedIn", - }, -] +async function fetchStats(): Promise<{ + total: number + lastUpdated: string | null +}> { + const res = await fetch( + `${import.meta.env.VITE_API_URL}/api/v1/articles/stats`, + ) + if (!res.ok) return { total: 0, lastUpdated: null } + return res.json() +} + +function formatLastUpdated(iso: string | null): string { + if (!iso) return "" + try { + const hasTimezone = /[+-]\d{2}:\d{2}$|Z$/.test(iso) + const d = new Date(hasTimezone ? iso : `${iso}Z`) + const diffMs = Date.now() - d.getTime() + const diffH = Math.floor(diffMs / 3_600_000) + if (diffH < 1) return "Updated just now" + if (diffH < 24) return `Updated ${diffH}h ago` + const diffD = Math.floor(diffH / 24) + return `Updated ${diffD}d ago` + } catch { + return "" + } +} export function Footer() { - const currentYear = new Date().getFullYear() + const { data } = useQuery({ + queryKey: ["stats"], + queryFn: fetchStats, + staleTime: 5 * 60_000, + }) + + const total = data?.total ?? 0 + const updatedLabel = formatLastUpdated(data?.lastUpdated ?? null) return ( -