From 715bdb9e3672331b376bfcab62485f4ad3707326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Conejo?= Date: Wed, 9 Sep 2026 16:18:07 +0200 Subject: [PATCH 1/3] Move LLM Gateway docs to their canonical namespace --- autofix.mdx | 17 +- deploy.mdx | 25 +- deploy/aws.mdx | 3 +- deploy/coolify.mdx | 1 + deploy/digitalocean.mdx | 3 +- deploy/easypanel.mdx | 3 +- deploy/fly.mdx | 3 +- deploy/gcp.mdx | 3 +- deploy/heroku.mdx | 5 +- deploy/koyeb.mdx | 3 +- deploy/railway.mdx | 5 +- deploy/render.mdx | 3 +- docs.json | 150 +++++-- errors.mdx | 39 +- errors/M001.mdx | 13 +- errors/M002.mdx | 11 +- errors/M003.mdx | 13 +- errors/M004.mdx | 9 +- errors/M005.mdx | 13 +- errors/M100.mdx | 17 +- errors/M101.mdx | 17 +- errors/M102.mdx | 19 +- errors/M200.mdx | 21 +- errors/M201.mdx | 13 +- errors/M202.mdx | 13 +- errors/M203.mdx | 13 +- errors/M204.mdx | 7 +- errors/M300.mdx | 9 +- errors/M302.mdx | 13 +- errors/M303.mdx | 19 +- errors/M500.mdx | 9 +- integrations/n8n.mdx | 11 +- introduction.mdx | 19 +- llm-gateway.mdx | 33 +- llm-gateway/docs/autofix.mdx | 188 +++++++++ llm-gateway/docs/deploy.mdx | 98 +++++ llm-gateway/docs/deploy/aws.mdx | 94 +++++ llm-gateway/docs/deploy/coolify.mdx | 82 ++++ llm-gateway/docs/deploy/digitalocean.mdx | 77 ++++ llm-gateway/docs/deploy/easypanel.mdx | 112 ++++++ llm-gateway/docs/deploy/fly.mdx | 85 ++++ llm-gateway/docs/deploy/gcp.mdx | 95 +++++ llm-gateway/docs/deploy/heroku.mdx | 91 +++++ llm-gateway/docs/deploy/koyeb.mdx | 93 +++++ llm-gateway/docs/deploy/railway.mdx | 66 ++++ llm-gateway/docs/deploy/render.mdx | 72 ++++ llm-gateway/docs/errors.mdx | 78 ++++ llm-gateway/docs/errors/M001.mdx | 34 ++ llm-gateway/docs/errors/M002.mdx | 33 ++ llm-gateway/docs/errors/M003.mdx | 34 ++ llm-gateway/docs/errors/M004.mdx | 32 ++ llm-gateway/docs/errors/M005.mdx | 34 ++ llm-gateway/docs/errors/M100.mdx | 38 ++ llm-gateway/docs/errors/M101.mdx | 36 ++ llm-gateway/docs/errors/M102.mdx | 58 +++ llm-gateway/docs/errors/M200.mdx | 42 ++ llm-gateway/docs/errors/M201.mdx | 36 ++ llm-gateway/docs/errors/M202.mdx | 34 ++ llm-gateway/docs/errors/M203.mdx | 34 ++ llm-gateway/docs/errors/M204.mdx | 34 ++ llm-gateway/docs/errors/M300.mdx | 41 ++ llm-gateway/docs/errors/M302.mdx | 44 +++ llm-gateway/docs/errors/M303.mdx | 51 +++ llm-gateway/docs/errors/M500.mdx | 32 ++ llm-gateway/docs/integrations/n8n.mdx | 127 ++++++ llm-gateway/docs/introduction.mdx | 84 ++++ llm-gateway/docs/llm-gateway.mdx | 167 ++++++++ llm-gateway/docs/observability.mdx | 145 +++++++ .../docs/providers/api-key-providers.mdx | 108 ++++++ .../docs/providers/custom-providers.mdx | 66 ++++ llm-gateway/docs/providers/local-models.mdx | 110 ++++++ .../subscription-based-providers.mdx | 107 +++++ llm-gateway/docs/reference/api.mdx | 221 +++++++++++ .../docs/reference/environment-variables.mdx | 180 +++++++++ llm-gateway/docs/reference/glossary.mdx | 71 ++++ llm-gateway/docs/reference/headers.mdx | 76 ++++ llm-gateway/docs/reference/telemetry.mdx | 97 +++++ llm-gateway/docs/request-logs.mdx | 146 +++++++ llm-gateway/docs/self-hosted.mdx | 367 ++++++++++++++++++ observability.mdx | 25 +- providers/api-key-providers.mdx | 3 +- providers/custom-providers.mdx | 3 +- providers/local-models.mdx | 3 +- providers/subscription-based-providers.mdx | 5 +- reference/api.mdx | 17 +- reference/environment-variables.mdx | 21 +- reference/glossary.mdx | 19 +- reference/headers.mdx | 5 +- reference/telemetry.mdx | 11 +- request-logs.mdx | 25 +- self-hosted.mdx | 17 +- 91 files changed, 4359 insertions(+), 300 deletions(-) create mode 100644 llm-gateway/docs/autofix.mdx create mode 100644 llm-gateway/docs/deploy.mdx create mode 100644 llm-gateway/docs/deploy/aws.mdx create mode 100644 llm-gateway/docs/deploy/coolify.mdx create mode 100644 llm-gateway/docs/deploy/digitalocean.mdx create mode 100644 llm-gateway/docs/deploy/easypanel.mdx create mode 100644 llm-gateway/docs/deploy/fly.mdx create mode 100644 llm-gateway/docs/deploy/gcp.mdx create mode 100644 llm-gateway/docs/deploy/heroku.mdx create mode 100644 llm-gateway/docs/deploy/koyeb.mdx create mode 100644 llm-gateway/docs/deploy/railway.mdx create mode 100644 llm-gateway/docs/deploy/render.mdx create mode 100644 llm-gateway/docs/errors.mdx create mode 100644 llm-gateway/docs/errors/M001.mdx create mode 100644 llm-gateway/docs/errors/M002.mdx create mode 100644 llm-gateway/docs/errors/M003.mdx create mode 100644 llm-gateway/docs/errors/M004.mdx create mode 100644 llm-gateway/docs/errors/M005.mdx create mode 100644 llm-gateway/docs/errors/M100.mdx create mode 100644 llm-gateway/docs/errors/M101.mdx create mode 100644 llm-gateway/docs/errors/M102.mdx create mode 100644 llm-gateway/docs/errors/M200.mdx create mode 100644 llm-gateway/docs/errors/M201.mdx create mode 100644 llm-gateway/docs/errors/M202.mdx create mode 100644 llm-gateway/docs/errors/M203.mdx create mode 100644 llm-gateway/docs/errors/M204.mdx create mode 100644 llm-gateway/docs/errors/M300.mdx create mode 100644 llm-gateway/docs/errors/M302.mdx create mode 100644 llm-gateway/docs/errors/M303.mdx create mode 100644 llm-gateway/docs/errors/M500.mdx create mode 100644 llm-gateway/docs/integrations/n8n.mdx create mode 100644 llm-gateway/docs/introduction.mdx create mode 100644 llm-gateway/docs/llm-gateway.mdx create mode 100644 llm-gateway/docs/observability.mdx create mode 100644 llm-gateway/docs/providers/api-key-providers.mdx create mode 100644 llm-gateway/docs/providers/custom-providers.mdx create mode 100644 llm-gateway/docs/providers/local-models.mdx create mode 100644 llm-gateway/docs/providers/subscription-based-providers.mdx create mode 100644 llm-gateway/docs/reference/api.mdx create mode 100644 llm-gateway/docs/reference/environment-variables.mdx create mode 100644 llm-gateway/docs/reference/glossary.mdx create mode 100644 llm-gateway/docs/reference/headers.mdx create mode 100644 llm-gateway/docs/reference/telemetry.mdx create mode 100644 llm-gateway/docs/request-logs.mdx create mode 100644 llm-gateway/docs/self-hosted.mdx diff --git a/autofix.mdx b/autofix.mdx index baeae6f..1a34e7e 100644 --- a/autofix.mdx +++ b/autofix.mdx @@ -4,6 +4,7 @@ description: "Repair a malformed request and resend it once, before the fallback icon: "wrench" keywords: ["auto-fix", "autofix", "self-healing", "LLM request repair", "malformed request", "400 bad request", "422 unprocessable", "model not found", "agent debugging", "harness debugging"] +canonical: "https://manifest.build/llm-gateway/docs/autofix" ---
@@ -25,7 +26,7 @@ Autofix only touches request-side 4xx errors, where the provider is telling you | `404` | Not found, usually a model name that moved | | `422` | Unprocessable, usually a schema the API rejects | -It deliberately skips `401`, `403`, `429`, and every `5xx`. A bad key, a rate limit, or a provider outage isn't something a rewritten body can fix, so those go straight to [fallback](/llm-gateway#fallback). +It deliberately skips `401`, `403`, `429`, and every `5xx`. A bad key, a rate limit, or a provider outage isn't something a rewritten body can fix, so those go straight to [fallback](/llm-gateway/docs/llm-gateway#fallback). Renamed models are covered through the provider itself. When a request names a model Manifest hasn't catalogued but the provider and credentials are unambiguous, Manifest forwards it anyway. The provider answers with its real error, typically a `404` for a model that no longer exists, and that error goes through the normal Autofix path. A patch that corrects the model name serves the request. @@ -108,7 +109,7 @@ Here is exactly what Autofix sends to the healing service, and when nothing is s Self-hosting is often a privacy choice, so here is exactly what leaves your machine, and when nothing does. -The first time you enable Autofix, your install records a one-time consent: enabling it means you agree that failing requests are sent to the hosted healing service, as set out in the [terms](https://manifest.build/terms/). Any enable path records it. Your install identifies itself with an anonymous install id, the same id [telemetry](/reference/telemetry) uses. +The first time you enable Autofix, your install records a one-time consent: enabling it means you agree that failing requests are sent to the hosted healing service, as set out in the [terms](https://manifest.build/terms/). Any enable path records it. Your install identifies itself with an anonymous install id, the same id [telemetry](/llm-gateway/docs/reference/telemetry) uses.
@@ -144,7 +145,7 @@ Open any request in the dashboard's request log. Requests that went through Auto The request drawer on a repaired request, with the failed and patched attempts listed and the Autofix panel showing what changed -The failed original and its patched retry are separate [provider attempts](/observability#requests-and-provider-attempts), so with [request logs](/request-logs) on you can read both bodies side by side and see exactly which field was rewritten. +The failed original and its patched retry are separate [provider attempts](/llm-gateway/docs/observability#requests-and-provider-attempts), so with [request logs](/llm-gateway/docs/request-logs) on you can read both bodies side by side and see exactly which field was rewritten. The **Overview** page aggregates the same data across a time range: @@ -175,13 +176,13 @@ Autofix needs no setup. Every variable below is optional and adjusts behavior: | `AUTOFIX_HEALING_API_KEY` | unset | Sent as `x-api-key` to the healing service. Only relevant if Manifest issues you one | | `AUTOFIX_REPORT_ALL_4XX` | `false` | Opt-in. Also report request-side 4xx errors (except `401`/`402`/`403`/`429`) from harnesses with Autofix on, as diagnostic evidence with the secret-scrubbed request body. Bodies over 256 KB are dropped, not truncated. No fix comes back from it | -You don't need `AUTOFIX_HEALING_API_KEY`. With no key set, the install announces its anonymous install id instead. The full list of variables lives in [Environment variables](/reference/environment-variables). +You don't need `AUTOFIX_HEALING_API_KEY`. With no key set, the install announces its anonymous install id instead. The full list of variables lives in [Environment variables](/llm-gateway/docs/reference/environment-variables). ## Related -- [LLM Gateway](/llm-gateway): where Autofix sits in the request flow -- [Observability](/observability): finding failing requests in the log -- [Request logs](/request-logs): reading the original and patched bodies -- [Error codes](/errors) +- [LLM Gateway](/llm-gateway/docs/llm-gateway): where Autofix sits in the request flow +- [Observability](/llm-gateway/docs/observability): finding failing requests in the log +- [Request logs](/llm-gateway/docs/request-logs): reading the original and patched bodies +- [Error codes](/llm-gateway/docs/errors) diff --git a/deploy.mdx b/deploy.mdx index e17f6f1..93cf485 100644 --- a/deploy.mdx +++ b/deploy.mdx @@ -16,13 +16,14 @@ keywords: "Easypanel", "Coolify", ] +canonical: "https://manifest.build/llm-gateway/docs/deploy" --- Everything on this page runs the same open-source Manifest image on infrastructure you control, with your own PostgreSQL database. Pick the path that matches where you want it to live. If you'd rather not run anything yourself, [Manifest Cloud](https://app.manifest.build) is the hosted alternative. ## Your own machine - + The most direct path. One install command, or docker-compose if you want to edit the config first. Also covers upgrades, backups, and bringing your own PostgreSQL. @@ -33,27 +34,27 @@ Everything on this page runs the same open-source Manifest image on infrastructu Someone else runs the infrastructure. Usually a one-click template or deploy button. - + Best one-click path for most users. The template provisions Manifest and PostgreSQL. - + Blueprint-based deploy using the public Docker image, generated secrets, and Render PostgreSQL. - + App Platform deploy from the public repository with Manifest and a Dev PostgreSQL database. - + CLI-based deploy using the public Docker image, Fly Postgres, and generated Manifest secrets. - + Deploy-button guide with Heroku Postgres, generated secrets, and one required BETTER_AUTH_URL value. - + Deploy-button guide for Koyeb using the public Docker image, manual Postgres URL, and runtime secrets. @@ -64,11 +65,11 @@ Someone else runs the infrastructure. Usually a one-click template or deploy but Your own cloud account, provisioned with the provider's own infrastructure tooling. - + CloudFormation quick-create for ECS Fargate, RDS PostgreSQL, Secrets Manager, and ALB. - + Cloud Shell guided deploy for Cloud Run, Cloud SQL PostgreSQL, and Secret Manager. @@ -79,11 +80,11 @@ Your own cloud account, provisioned with the provider's own infrastructure tooli A PaaS you run on your own server, so you get deploy buttons on hardware you own. - + Docker Compose stack using Coolify magic variables and a private PostgreSQL container. - + Easypanel app service using PostgreSQL and the Manifest Docker image. @@ -94,4 +95,4 @@ Most hosted deployments create paid resources. Review each provider's pricing be For public deployments, make sure `BETTER_AUTH_URL` matches the exact browser URL. If it does not, Better Auth can reject login with an invalid-origin error. -Whichever path you pick, [Environment variables](/reference/environment-variables) is the full configuration list, and [Data and telemetry](/reference/telemetry) covers what your instance stores and what it reports back. +Whichever path you pick, [Environment variables](/llm-gateway/docs/reference/environment-variables) is the full configuration list, and [Data and telemetry](/llm-gateway/docs/reference/telemetry) covers what your instance stores and what it reports back. diff --git a/deploy/aws.mdx b/deploy/aws.mdx index dfe5dff..9ae9176 100644 --- a/deploy/aws.mdx +++ b/deploy/aws.mdx @@ -5,6 +5,7 @@ description: "Deploy Manifest on AWS with CloudFormation, ECS Fargate, RDS Postg icon: "/icons/deploy/aws.svg" keywords: ["Manifest deployment", "AWS", "self-hosted Manifest", "PostgreSQL", "Docker"] +canonical: "https://manifest.build/llm-gateway/docs/deploy/aws" --- /api/v1/health ## Notes -- App Platform has no persistent volumes, so [request logs](/request-logs) write to your DigitalOcean Space over S3. +- App Platform has no persistent volumes, so [request logs](/llm-gateway/docs/request-logs) write to your DigitalOcean Space over S3. - The template appends `uselibpqcompat=true` to DigitalOcean's PostgreSQL URL so Node `pg` handles the platform's `sslmode=require` connection string correctly. - Upgrade the Dev Database to a managed database for production data. - Use a custom domain before configuring production OAuth callbacks. diff --git a/deploy/easypanel.mdx b/deploy/easypanel.mdx index 031eced..065366e 100644 --- a/deploy/easypanel.mdx +++ b/deploy/easypanel.mdx @@ -11,6 +11,7 @@ keywords: "PostgreSQL", "Docker", ] +canonical: "https://manifest.build/llm-gateway/docs/deploy/easypanel" --- - The template keeps one Machine running so Manifest is always available for agents. - For production, choose a larger Postgres configuration or Fly Managed Postgres instead of the small script default. - Add a custom domain before configuring OAuth callback URLs. -- The deploy script creates a private Tigris bucket, stores its credentials as Fly secrets, and sets `REQUEST_RECORDING_STORAGE=s3` with the Tigris endpoint. [Request logs](/request-logs) persist across deploys and across scaled Machines. Nothing to attach manually. +- The deploy script creates a private Tigris bucket, stores its credentials as Fly secrets, and sets `REQUEST_RECORDING_STORAGE=s3` with the Tigris endpoint. [Request logs](/llm-gateway/docs/request-logs) persist across deploys and across scaled Machines. Nothing to attach manually. - Set `MANIFEST_TELEMETRY_DISABLED=1` as a Fly secret if you want to disable anonymous self-hosted telemetry. - Destroy both the app and database when testing is done. diff --git a/deploy/gcp.mdx b/deploy/gcp.mdx index 3119791..8c3390a 100644 --- a/deploy/gcp.mdx +++ b/deploy/gcp.mdx @@ -5,6 +5,7 @@ description: "Deploy Manifest on Google Cloud with Cloud Run, Cloud SQL for Post icon: "/icons/deploy/gcp.svg" keywords: ["Manifest deployment", "GCP", "self-hosted Manifest", "PostgreSQL", "Docker"] +canonical: "https://manifest.build/llm-gateway/docs/deploy/gcp" --- - `PGSSLMODE=no-verify` enables TLS for Heroku Postgres without editing the managed `DATABASE_URL`. - `DB_POOL_MAX=8` and `AUTH_DB_POOL_MAX=4` leave headroom under the Essential-0 connection limit. - Use a custom domain before configuring production OAuth callbacks. -- Heroku's filesystem is ephemeral, which is why the deploy collects S3-compatible storage for [request logs](/request-logs) up front. Recordings live in your bucket, not on the dyno. +- Heroku's filesystem is ephemeral, which is why the deploy collects S3-compatible storage for [request logs](/llm-gateway/docs/request-logs) up front. Recordings live in your bucket, not on the dyno. ## Tearing it down diff --git a/deploy/koyeb.mdx b/deploy/koyeb.mdx index bee11fe..34ad2ba 100644 --- a/deploy/koyeb.mdx +++ b/deploy/koyeb.mdx @@ -11,6 +11,7 @@ keywords: "PostgreSQL", "Docker", ] +canonical: "https://manifest.build/llm-gateway/docs/deploy/koyeb" --- /api/v1/health ## Notes -- Koyeb instances have an ephemeral filesystem, which is why the deploy link collects S3-compatible storage for [request logs](/request-logs). Recordings live in your bucket, not on the instance. +- Koyeb instances have an ephemeral filesystem, which is why the deploy link collects S3-compatible storage for [request logs](/llm-gateway/docs/request-logs). Recordings live in your bucket, not on the instance. ## Tearing it down diff --git a/deploy/railway.mdx b/deploy/railway.mdx index fa41828..8e8aaa7 100644 --- a/deploy/railway.mdx +++ b/deploy/railway.mdx @@ -11,6 +11,7 @@ keywords: "PostgreSQL", "Docker", ] +canonical: "https://manifest.build/llm-gateway/docs/deploy/railway" --- /api/v1/health - Use a custom domain before configuring production OAuth callbacks. - Keep PostgreSQL backups enabled for production data. -- The template provisions a `messages-recordings` bucket for [request logs](/request-logs), so they survive redeploys without extra setup. +- The template provisions a `messages-recordings` bucket for [request logs](/llm-gateway/docs/request-logs), so they survive redeploys without extra setup. - Use the Railway service logs when debugging failed boots or database connection issues. Relevant Railway links: diff --git a/deploy/render.mdx b/deploy/render.mdx index 7671ada..bcbd4a8 100644 --- a/deploy/render.mdx +++ b/deploy/render.mdx @@ -11,6 +11,7 @@ keywords: "PostgreSQL", "Docker", ] +canonical: "https://manifest.build/llm-gateway/docs/deploy/render" --- .onrender.com/api/v1/health - Render PostgreSQL connection strings are wired by the Blueprint. - Use a custom domain before configuring production OAuth callbacks. -- The Blueprint mounts a 1 GB persistent disk `manifest-recordings` at `/data/request-recordings` and sets `REQUEST_RECORDING_STORAGE=filesystem`, so [request logs](/request-logs) survive redeploys. This disk is single-instance. Point recording at S3-compatible storage before you scale to more than one instance. +- The Blueprint mounts a 1 GB persistent disk `manifest-recordings` at `/data/request-recordings` and sets `REQUEST_RECORDING_STORAGE=filesystem`, so [request logs](/llm-gateway/docs/request-logs) survive redeploys. This disk is single-instance. Point recording at S3-compatible storage before you scale to more than one instance. Relevant Render docs: diff --git a/docs.json b/docs.json index dce8f2c..7e8aef7 100644 --- a/docs.json +++ b/docs.json @@ -41,7 +41,7 @@ }, "seo": { "metatags": { - "canonical": "https://manifest.build/docs", + "canonical": "https://manifest.build", "og:site_name": "Manifest", "og:type": "website", "twitter:card": "summary_large_image", @@ -105,26 +105,26 @@ "group": "Getting Started", "boost": 3, "pages": [ - "introduction" + "llm-gateway/docs/introduction" ] }, { "group": "Self-hosting", "boost": 2, "pages": [ - "deploy", - "self-hosted", + "llm-gateway/docs/deploy", + "llm-gateway/docs/self-hosted", { "group": "Managed platforms", "icon": "cloud", "expanded": false, "pages": [ - "deploy/railway", - "deploy/render", - "deploy/digitalocean", - "deploy/fly", - "deploy/heroku", - "deploy/koyeb" + "llm-gateway/docs/deploy/railway", + "llm-gateway/docs/deploy/render", + "llm-gateway/docs/deploy/digitalocean", + "llm-gateway/docs/deploy/fly", + "llm-gateway/docs/deploy/heroku", + "llm-gateway/docs/deploy/koyeb" ] }, { @@ -132,8 +132,8 @@ "icon": "server", "expanded": false, "pages": [ - "deploy/aws", - "deploy/gcp" + "llm-gateway/docs/deploy/aws", + "llm-gateway/docs/deploy/gcp" ] }, { @@ -141,8 +141,8 @@ "icon": "panel-top", "expanded": false, "pages": [ - "deploy/coolify", - "deploy/easypanel" + "llm-gateway/docs/deploy/coolify", + "llm-gateway/docs/deploy/easypanel" ] } ] @@ -151,38 +151,38 @@ "group": "Features", "boost": 2, "pages": [ - "llm-gateway", - "autofix", - "observability", - "request-logs" + "llm-gateway/docs/llm-gateway", + "llm-gateway/docs/autofix", + "llm-gateway/docs/observability", + "llm-gateway/docs/request-logs" ] }, { "group": "Providers", "boost": 2, "pages": [ - "providers/api-key-providers", - "providers/subscription-based-providers", - "providers/custom-providers", - "providers/local-models" + "llm-gateway/docs/providers/api-key-providers", + "llm-gateway/docs/providers/subscription-based-providers", + "llm-gateway/docs/providers/custom-providers", + "llm-gateway/docs/providers/local-models" ] }, { "group": "Integrations", "boost": 2, "pages": [ - "integrations/n8n" + "llm-gateway/docs/integrations/n8n" ] }, { "group": "Reference", "pages": [ - "reference/api", - "reference/headers", - "reference/environment-variables", - "reference/telemetry", - "errors", - "reference/glossary" + "llm-gateway/docs/reference/api", + "llm-gateway/docs/reference/headers", + "llm-gateway/docs/reference/environment-variables", + "llm-gateway/docs/reference/telemetry", + "llm-gateway/docs/errors", + "llm-gateway/docs/reference/glossary" ] } ] @@ -268,55 +268,55 @@ }, { "source": "/install", - "destination": "/self-hosted" + "destination": "/llm-gateway/docs/self-hosted" }, { "source": "/docker", - "destination": "/self-hosted" + "destination": "/llm-gateway/docs/self-hosted" }, { "source": "/telemetry", - "destination": "/reference/telemetry" + "destination": "/llm-gateway/docs/reference/telemetry" }, { "source": "/privacy", - "destination": "/reference/telemetry" + "destination": "/llm-gateway/docs/reference/telemetry" }, { "source": "/cloud-vs-local", - "destination": "/introduction" + "destination": "/llm-gateway/docs/introduction" }, { "source": "/cloud-vs-self-hosted", - "destination": "/introduction" + "destination": "/llm-gateway/docs/introduction" }, { "source": "/get-started", - "destination": "/introduction" + "destination": "/llm-gateway/docs/introduction" }, { "source": "/track-usage", - "destination": "/observability" + "destination": "/llm-gateway/docs/observability" }, { "source": "/routing", - "destination": "/llm-gateway" + "destination": "/llm-gateway/docs/llm-gateway" }, { "source": "/fallback", - "destination": "/llm-gateway" + "destination": "/llm-gateway/docs/llm-gateway" }, { "source": "/set-limits", - "destination": "/observability" + "destination": "/llm-gateway/docs/observability" }, { "source": "/errors/M301", - "destination": "/errors" + "destination": "/llm-gateway/docs/errors" }, { "source": "/configuration", - "destination": "/self-hosted" + "destination": "/llm-gateway/docs/self-hosted" }, { "source": "/contributing", @@ -324,7 +324,71 @@ }, { "source": "/message-recording", - "destination": "/request-logs" + "destination": "/llm-gateway/docs/request-logs" + }, + { + "source": "/llm-gateway/docs", + "destination": "/llm-gateway/docs/introduction" + }, + { + "source": "/llm-gateway/docs/install", + "destination": "/llm-gateway/docs/self-hosted" + }, + { + "source": "/llm-gateway/docs/docker", + "destination": "/llm-gateway/docs/self-hosted" + }, + { + "source": "/llm-gateway/docs/telemetry", + "destination": "/llm-gateway/docs/reference/telemetry" + }, + { + "source": "/llm-gateway/docs/privacy", + "destination": "/llm-gateway/docs/reference/telemetry" + }, + { + "source": "/llm-gateway/docs/cloud-vs-local", + "destination": "/llm-gateway/docs/introduction" + }, + { + "source": "/llm-gateway/docs/cloud-vs-self-hosted", + "destination": "/llm-gateway/docs/introduction" + }, + { + "source": "/llm-gateway/docs/get-started", + "destination": "/llm-gateway/docs/introduction" + }, + { + "source": "/llm-gateway/docs/track-usage", + "destination": "/llm-gateway/docs/observability" + }, + { + "source": "/llm-gateway/docs/routing", + "destination": "/llm-gateway/docs/llm-gateway" + }, + { + "source": "/llm-gateway/docs/fallback", + "destination": "/llm-gateway/docs/llm-gateway" + }, + { + "source": "/llm-gateway/docs/set-limits", + "destination": "/llm-gateway/docs/observability" + }, + { + "source": "/llm-gateway/docs/errors/M301", + "destination": "/llm-gateway/docs/errors" + }, + { + "source": "/llm-gateway/docs/configuration", + "destination": "/llm-gateway/docs/self-hosted" + }, + { + "source": "/llm-gateway/docs/contributing", + "destination": "https://github.com/mnfst/manifest" + }, + { + "source": "/llm-gateway/docs/message-recording", + "destination": "/llm-gateway/docs/request-logs" } ] } diff --git a/errors.mdx b/errors.mdx index 49d808c..05e5ef9 100644 --- a/errors.mdx +++ b/errors.mdx @@ -5,13 +5,14 @@ description: "Reference for every Manifest proxy error code (M001-M500). What yo icon: "circle-alert" keywords: ["Manifest error codes", "M001", "M100", "M102", "M200", "M204", "M302", "M500", "OpenAI compatible errors", "401 Unauthorized", "402 Payment Required", "429 Too Many Requests", "Bearer token error", "chat completions error", "model not available", "troubleshooting", "proxy errors"] +canonical: "https://manifest.build/llm-gateway/docs/errors" --- When Manifest blocks or rejects a request, the response message starts with a code in square brackets: ```text [🦚 Manifest M100] No anthropic API key yet. Add one here: https://app.manifest.build/... -See https://manifest.build/docs/errors/M100 +See https://manifest.build/llm-gateway/docs/errors/M100 ``` Look up the code below to see what it means and how to fix it. @@ -24,21 +25,21 @@ These fire when the bearer token on `/v1/chat/completions` is missing or wrong. | Code | What | |------|------| -| [M001: Missing Authorization header](/errors/M001) | No `Authorization` header on the request | -| [M002: Empty Bearer token](/errors/M002) | Header present, token after `Bearer ` is blank | -| [M003: Invalid key format](/errors/M003) | Token doesn't start with `mnfst_` | -| [M004: Key expired](/errors/M004) | Key past its expiration date | -| [M005: Key not recognized](/errors/M005) | No matching harness for this key | +| [M001: Missing Authorization header](/llm-gateway/docs/errors/M001) | No `Authorization` header on the request | +| [M002: Empty Bearer token](/llm-gateway/docs/errors/M002) | Header present, token after `Bearer ` is blank | +| [M003: Invalid key format](/llm-gateway/docs/errors/M003) | Token doesn't start with `mnfst_` | +| [M004: Key expired](/llm-gateway/docs/errors/M004) | Key past its expiration date | +| [M005: Key not recognized](/llm-gateway/docs/errors/M005) | No matching harness for this key | ## Providers (M100–M102) -Your Manifest key is fine, but provider credentials are missing or unusable. See [Routing](/llm-gateway), [API key providers](/providers/api-key-providers), and [Subscription-based providers](/providers/subscription-based-providers). +Your Manifest key is fine, but provider credentials are missing or unusable. See [Routing](/llm-gateway/docs/llm-gateway), [API key providers](/llm-gateway/docs/providers/api-key-providers), and [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers). | Code | What | |------|------| -| [M100: Provider API key missing](/errors/M100) | Routing picked a provider you haven't connected | -| [M101: No providers configured](/errors/M101) | Harness has zero providers connected | -| [M102: Provider subscription credentials unusable](/errors/M102) | Subscription OAuth/token exists but cannot be refreshed | +| [M100: Provider API key missing](/llm-gateway/docs/errors/M100) | Routing picked a provider you haven't connected | +| [M101: No providers configured](/llm-gateway/docs/errors/M101) | Harness has zero providers connected | +| [M102: Provider subscription credentials unusable](/llm-gateway/docs/errors/M102) | Subscription OAuth/token exists but cannot be refreshed | ## Limits (M200–M204) @@ -46,11 +47,11 @@ You hit a usage cap, rate limit, or Free plan request quota. M200 comes back as | Code | What | |------|------| -| [M200: Usage limit exceeded](/errors/M200) | Cost or token limit you set was crossed | -| [M201: Per-user rate limit exceeded](/errors/M201) | More than 200 requests/minute from one user | -| [M202: Per-IP rate limit exceeded](/errors/M202) | More than 500 requests/minute from one IP | -| [M203: Concurrency limit exceeded](/errors/M203) | More than 10 in-flight requests at once | -| [M204: Monthly request limit reached](/errors/M204) | Free plan monthly request quota was exhausted | +| [M200: Usage limit exceeded](/llm-gateway/docs/errors/M200) | Cost or token limit you set was crossed | +| [M201: Per-user rate limit exceeded](/llm-gateway/docs/errors/M201) | More than 200 requests/minute from one user | +| [M202: Per-IP rate limit exceeded](/llm-gateway/docs/errors/M202) | More than 500 requests/minute from one IP | +| [M203: Concurrency limit exceeded](/llm-gateway/docs/errors/M203) | More than 10 in-flight requests at once | +| [M204: Monthly request limit reached](/llm-gateway/docs/errors/M204) | Free plan monthly request quota was exhausted | ## Request validation and model selection (M300–M303) @@ -58,9 +59,9 @@ M300 means the request body is malformed and surfaces as HTTP 400. M302 and M303 | Code | What | |------|------| -| [M300: Missing messages array](/errors/M300) | Body has no `messages` array, or it's empty | -| [M302: Model not available](/errors/M302) | Explicit model ID is not available for this harness | -| [M303: Local provider unavailable on Manifest Cloud](/errors/M303) | Routed to Ollama, LM Studio, or llama.cpp on Cloud | +| [M300: Missing messages array](/llm-gateway/docs/errors/M300) | Body has no `messages` array, or it's empty | +| [M302: Model not available](/llm-gateway/docs/errors/M302) | Explicit model ID is not available for this harness | +| [M303: Local provider unavailable on Manifest Cloud](/llm-gateway/docs/errors/M303) | Routed to Ollama, LM Studio, or llama.cpp on Cloud | M301 is missing from that list on purpose. It capped requests at 1000 @@ -74,4 +75,4 @@ Manifest itself broke. Surfaces as HTTP 500. | Code | What | |------|------| -| [M500: Internal server error](/errors/M500) | Unhandled error in the proxy pipeline | +| [M500: Internal server error](/llm-gateway/docs/errors/M500) | Unhandled error in the proxy pipeline | diff --git a/errors/M001.mdx b/errors/M001.mdx index 158093b..67d828e 100644 --- a/errors/M001.mdx +++ b/errors/M001.mdx @@ -5,13 +5,14 @@ description: "Manifest error M001 fires when /v1/chat/completions is called with icon: "key-round" keywords: ["M001", "Manifest M001", "Authorization header missing", "Bearer token", "401 Unauthorized", "OpenAI compatible 401", "Manifest auth error", "chat completions authorization"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M001" --- ## What you saw ```text [🦚 Manifest M001] Missing the Authorization header. Set it to "Bearer mnfst_". -See https://manifest.build/docs/errors/M001 +See https://manifest.build/llm-gateway/docs/errors/M001 ``` ## Why it happened @@ -26,8 +27,8 @@ Your client called `/v1/chat/completions` without an `Authorization` header. Man ## Related -- [M002: Empty Bearer token](/errors/M002) -- [M003: Invalid key format](/errors/M003) -- [M004: Key expired](/errors/M004) -- [M005: Key not recognized](/errors/M005) -- [All error codes](/errors) +- [M002: Empty Bearer token](/llm-gateway/docs/errors/M002) +- [M003: Invalid key format](/llm-gateway/docs/errors/M003) +- [M004: Key expired](/llm-gateway/docs/errors/M004) +- [M005: Key not recognized](/llm-gateway/docs/errors/M005) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M002.mdx b/errors/M002.mdx index 5c61133..72d2239 100644 --- a/errors/M002.mdx +++ b/errors/M002.mdx @@ -5,13 +5,14 @@ description: "Manifest error M002 fires when the Authorization header is set but icon: "key-round" keywords: ["M002", "Manifest M002", "empty Bearer token", "blank token", "401 Unauthorized", "MNFST_KEY env var", "Manifest auth error", "OpenAI compatible 401"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M002" --- ## What you saw ```text [🦚 Manifest M002] The Bearer token is empty. Paste your Manifest key into it. -See https://manifest.build/docs/errors/M002 +See https://manifest.build/llm-gateway/docs/errors/M002 ``` ## Why it happened @@ -26,7 +27,7 @@ The `Authorization` header is there, but the token after `Bearer ` is blank. The ## Related -- [M001: Missing Authorization header](/errors/M001) -- [M003: Invalid key format](/errors/M003) -- [M005: Key not recognized](/errors/M005) -- [All error codes](/errors) +- [M001: Missing Authorization header](/llm-gateway/docs/errors/M001) +- [M003: Invalid key format](/llm-gateway/docs/errors/M003) +- [M005: Key not recognized](/llm-gateway/docs/errors/M005) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M003.mdx b/errors/M003.mdx index 8f586e9..e938a89 100644 --- a/errors/M003.mdx +++ b/errors/M003.mdx @@ -5,13 +5,14 @@ description: "Manifest error M003 fires when the Bearer token doesn't start with icon: "key-round" keywords: ["M003", "Manifest M003", "invalid API key format", "mnfst_ prefix", "wrong API key", "OpenAI sk- key", "401 Unauthorized", "Manifest auth error"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M003" --- ## What you saw ```text [🦚 Manifest M003] That doesn't look right. Manifest keys start with "mnfst_". Grab yours from the dashboard. -See https://manifest.build/docs/errors/M003 +See https://manifest.build/llm-gateway/docs/errors/M003 ``` ## Why it happened @@ -26,8 +27,8 @@ Manifest keys always start with `mnfst_`. Your token has a different prefix, whi ## Related -- [M001: Missing Authorization header](/errors/M001) -- [M005: Key not recognized](/errors/M005) -- [M100: Provider API key missing](/errors/M100) -- [API key providers](/providers/api-key-providers) -- [All error codes](/errors) +- [M001: Missing Authorization header](/llm-gateway/docs/errors/M001) +- [M005: Key not recognized](/llm-gateway/docs/errors/M005) +- [M100: Provider API key missing](/llm-gateway/docs/errors/M100) +- [API key providers](/llm-gateway/docs/providers/api-key-providers) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M004.mdx b/errors/M004.mdx index 4cad522..7734989 100644 --- a/errors/M004.mdx +++ b/errors/M004.mdx @@ -5,13 +5,14 @@ description: "Manifest error M004 fires when the API key you sent is past its ex icon: "key-round" keywords: ["M004", "Manifest M004", "API key expired", "key rotation", "401 Unauthorized", "Manifest auth error", "expired token"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M004" --- ## What you saw ```text [🦚 Manifest M004] This key has expired. Generate a new one here -See https://manifest.build/docs/errors/M004 +See https://manifest.build/llm-gateway/docs/errors/M004 ``` ## Why it happened @@ -26,6 +27,6 @@ Manifest keys can carry an expiration date. Yours is past that date, so it no lo ## Related -- [M005: Key not recognized](/errors/M005) -- [M001: Missing Authorization header](/errors/M001) -- [All error codes](/errors) +- [M005: Key not recognized](/llm-gateway/docs/errors/M005) +- [M001: Missing Authorization header](/llm-gateway/docs/errors/M001) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M005.mdx b/errors/M005.mdx index 7c03e2a..36811ef 100644 --- a/errors/M005.mdx +++ b/errors/M005.mdx @@ -5,13 +5,14 @@ description: "Manifest error M005 fires when the mnfst_ key has the right shape icon: "key-round" keywords: ["M005", "Manifest M005", "invalid API key", "rotated key", "deleted agent", "deleted harness", "401 Unauthorized", "key not found", "Manifest auth error"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M005" --- ## What you saw ```text [🦚 Manifest M005] I don't recognize this key. It might have been rotated or deleted. Grab the current one from the dashboard. -See https://manifest.build/docs/errors/M005 +See https://manifest.build/llm-gateway/docs/errors/M005 ``` ## Why it happened @@ -26,8 +27,8 @@ The token shape is right (`mnfst_...`) but no matching harness exists. Usually t ## Related -- [M003: Invalid key format](/errors/M003) -- [M004: Key expired](/errors/M004) -- [M001: Missing Authorization header](/errors/M001) -- [Self-hosted setup](/self-hosted) -- [All error codes](/errors) +- [M003: Invalid key format](/llm-gateway/docs/errors/M003) +- [M004: Key expired](/llm-gateway/docs/errors/M004) +- [M001: Missing Authorization header](/llm-gateway/docs/errors/M001) +- [Self-hosted setup](/llm-gateway/docs/self-hosted) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M100.mdx b/errors/M100.mdx index 58a2d67..e299f13 100644 --- a/errors/M100.mdx +++ b/errors/M100.mdx @@ -5,13 +5,14 @@ description: "Manifest error M100 fires when routing picks a provider that has n icon: "plug-zap" keywords: ["M100", "Manifest M100", "provider API key missing", "no OpenAI key", "no Anthropic key", "no Gemini key", "OpenRouter key", "Manifest routing", "provider credentials"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M100" --- ## What you saw ```text [🦚 Manifest M100] No anthropic API key yet. Add one here: https://app.manifest.build/... -See https://manifest.build/docs/errors/M100 +See https://manifest.build/llm-gateway/docs/errors/M100 ``` The provider name (`anthropic` in the example) varies based on which provider Manifest selected. @@ -28,10 +29,10 @@ Routing picked a provider that has no key on file for this harness. Manifest won ## Related -- [M101: No providers configured](/errors/M101) -- [M102: Provider subscription credentials unusable](/errors/M102) -- [M003: Invalid key format](/errors/M003) -- [Routing](/llm-gateway) -- [API key providers](/providers/api-key-providers) -- [Subscription-based providers](/providers/subscription-based-providers) -- [All error codes](/errors) +- [M101: No providers configured](/llm-gateway/docs/errors/M101) +- [M102: Provider subscription credentials unusable](/llm-gateway/docs/errors/M102) +- [M003: Invalid key format](/llm-gateway/docs/errors/M003) +- [Routing](/llm-gateway/docs/llm-gateway) +- [API key providers](/llm-gateway/docs/providers/api-key-providers) +- [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M101.mdx b/errors/M101.mdx index 057ffca..98d238e 100644 --- a/errors/M101.mdx +++ b/errors/M101.mdx @@ -5,13 +5,14 @@ description: "Manifest error M101 fires when an authenticated harness has zero p icon: "plug-zap" keywords: ["M101", "Manifest M101", "no providers configured", "first-time setup", "empty agent", "empty harness", "Manifest routing", "connect provider", "OpenRouter setup"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M101" --- ## What you saw ```text [🦚 Manifest M101] You're connected, but no providers are set up yet. Add one here: https://app.manifest.build/... -See https://manifest.build/docs/errors/M101 +See https://manifest.build/llm-gateway/docs/errors/M101 ``` ## Why it happened @@ -26,10 +27,10 @@ You'll usually see this on a fresh harness. Your key works, but no provider cred ## Related -- [M100: Provider API key missing](/errors/M100) -- [M102: Provider subscription credentials unusable](/errors/M102) -- [Routing](/llm-gateway) -- [API key providers](/providers/api-key-providers) -- [Subscription-based providers](/providers/subscription-based-providers) -- [Local models (Ollama)](/providers/local-models) -- [All error codes](/errors) +- [M100: Provider API key missing](/llm-gateway/docs/errors/M100) +- [M102: Provider subscription credentials unusable](/llm-gateway/docs/errors/M102) +- [Routing](/llm-gateway/docs/llm-gateway) +- [API key providers](/llm-gateway/docs/providers/api-key-providers) +- [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers) +- [Local models (Ollama)](/llm-gateway/docs/providers/local-models) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M102.mdx b/errors/M102.mdx index eb8911d..675af13 100644 --- a/errors/M102.mdx +++ b/errors/M102.mdx @@ -15,13 +15,14 @@ keywords: "Manifest routing", "reconnect OAuth", ] +canonical: "https://manifest.build/llm-gateway/docs/errors/M102" --- ## What you saw ```text [🦚 Manifest M102] openai subscription credentials could not be refreshed. Reconnect OAuth here: https://app.manifest.build/... -See https://manifest.build/docs/errors/M102 +See https://manifest.build/llm-gateway/docs/errors/M102 ``` The provider name (`openai` in the example) varies based on which subscription route Manifest selected. @@ -36,7 +37,7 @@ Typical causes: - A pasted subscription token is no longer valid - Provider-side session invalidation after a password change or security reset -This is different from [M100](/errors/M100): M100 means there is no key on file. M102 means a subscription row exists and was elected, but the unwrap/refresh step failed. +This is different from [M100](/llm-gateway/docs/errors/M100): M100 means there is no key on file. M102 means a subscription row exists and was elected, but the unwrap/refresh step failed. When fallback routes are configured, Manifest records the M102 hop and continues the chain. You only see a terminal M102 response when no usable fallback remains. @@ -44,14 +45,14 @@ When fallback routes are configured, Manifest records the M102 hop and continues 1. Open the dashboard link in the error message. It goes straight to the harness's **Routing** page. 2. Open **Providers → Subscriptions** in the sidebar, click **Connect** on the provider, and redo the auth: OAuth, device code, or a fresh token. -3. Optionally add a working [API key](/providers/api-key-providers) or another [subscription](/providers/subscription-based-providers) as a fallback so a dead primary does not block traffic. +3. Optionally add a working [API key](/llm-gateway/docs/providers/api-key-providers) or another [subscription](/llm-gateway/docs/providers/subscription-based-providers) as a fallback so a dead primary does not block traffic. 4. Retry the request. ## Related -- [M100: Provider API key missing](/errors/M100) -- [M101: No providers configured](/errors/M101) -- [Subscription-based providers](/providers/subscription-based-providers) -- [API key providers](/providers/api-key-providers) -- [Routing](/llm-gateway) -- [All error codes](/errors) +- [M100: Provider API key missing](/llm-gateway/docs/errors/M100) +- [M101: No providers configured](/llm-gateway/docs/errors/M101) +- [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers) +- [API key providers](/llm-gateway/docs/providers/api-key-providers) +- [Routing](/llm-gateway/docs/llm-gateway) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M200.mdx b/errors/M200.mdx index 20138b4..8af7aa1 100644 --- a/errors/M200.mdx +++ b/errors/M200.mdx @@ -5,13 +5,14 @@ description: "Manifest error M200 fires when a harness crosses a cost or token c icon: "gauge" keywords: ["M200", "Manifest M200", "usage limit exceeded", "cost limit", "token limit", "spending cap", "budget cap", "monthly budget LLM", "Manifest limits", "HTTP 200 assistant message"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M200" --- ## What you saw ```text [🦚 Manifest M200] You hit your cost limit: $12.50 used, $10.00/day allowed. Adjust it here: https://app.manifest.build/... -See https://manifest.build/docs/errors/M200 +See https://manifest.build/llm-gateway/docs/errors/M200 ``` The metric (cost or tokens), used amount, threshold, and period vary based on the rule you tripped. @@ -20,9 +21,9 @@ The response arrives as HTTP 200 with this text in the assistant message content ## Why it happened -You set a limit on the harness's [Limits](/observability#spend-alerts) page (say, "$10/day" or "500k tokens/hour"), and the current period's usage just crossed it. Manifest blocks every following request until the period resets or you raise the cap. +You set a limit on the harness's [Limits](/llm-gateway/docs/observability#spend-alerts) page (say, "$10/day" or "500k tokens/hour"), and the current period's usage just crossed it. Manifest blocks every following request until the period resets or you raise the cap. -This is different from [M204](/errors/M204), which is the Manifest Cloud Free plan monthly request quota, and different from provider 402 errors such as "insufficient credits" from OpenRouter. +This is different from [M204](/llm-gateway/docs/errors/M204), which is the Manifest Cloud Free plan monthly request quota, and different from provider 402 errors such as "insufficient credits" from OpenRouter. ## How to fix it @@ -32,10 +33,10 @@ This is different from [M204](/errors/M204), which is the Manifest Cloud Free pl ## Related -- [Spend alerts](/observability#spend-alerts) -- [Hard limits](/llm-gateway#hard-limits) -- [M201: Per-user rate limit](/errors/M201) -- [M202: Per-IP rate limit](/errors/M202) -- [M203: Concurrency limit](/errors/M203) -- [M204: Monthly request limit reached](/errors/M204) -- [All error codes](/errors) +- [Spend alerts](/llm-gateway/docs/observability#spend-alerts) +- [Hard limits](/llm-gateway/docs/llm-gateway#hard-limits) +- [M201: Per-user rate limit](/llm-gateway/docs/errors/M201) +- [M202: Per-IP rate limit](/llm-gateway/docs/errors/M202) +- [M203: Concurrency limit](/llm-gateway/docs/errors/M203) +- [M204: Monthly request limit reached](/llm-gateway/docs/errors/M204) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M201.mdx b/errors/M201.mdx index 0378cab..ccf9184 100644 --- a/errors/M201.mdx +++ b/errors/M201.mdx @@ -5,13 +5,14 @@ description: "Manifest error M201 fires when one user sends more than 200 reques icon: "timer" keywords: ["M201", "Manifest M201", "rate limit", "429 Too Many Requests", "per-user rate limit", "throttling", "exponential backoff", "Manifest limits"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M201" --- ## What you saw ```text [🦚 Manifest M201] Too many requests — wait a few seconds and retry. -See https://manifest.build/docs/errors/M201 +See https://manifest.build/llm-gateway/docs/errors/M201 ``` The HTTP status is `429`, which most SDKs treat as retryable and back off from on their own. @@ -28,8 +29,8 @@ Manifest caps each user at 200 requests per minute. The cap is there mostly to k ## Related -- [M202: Per-IP rate limit](/errors/M202) -- [M203: Concurrency limit](/errors/M203) -- [M200: Usage limit exceeded](/errors/M200) -- [Self-hosted setup](/self-hosted) -- [All error codes](/errors) +- [M202: Per-IP rate limit](/llm-gateway/docs/errors/M202) +- [M203: Concurrency limit](/llm-gateway/docs/errors/M203) +- [M200: Usage limit exceeded](/llm-gateway/docs/errors/M200) +- [Self-hosted setup](/llm-gateway/docs/self-hosted) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M202.mdx b/errors/M202.mdx index 5a7b743..e5a7e00 100644 --- a/errors/M202.mdx +++ b/errors/M202.mdx @@ -5,13 +5,14 @@ description: "Manifest error M202 fires when one IP sends more than 500 requests icon: "timer" keywords: ["M202", "Manifest M202", "per-IP rate limit", "shared IP", "429 Too Many Requests", "Manifest limits", "abuse protection"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M202" --- ## What you saw ```text [🦚 Manifest M202] Too many requests from this IP — wait a few seconds and retry. -See https://manifest.build/docs/errors/M202 +See https://manifest.build/llm-gateway/docs/errors/M202 ``` ## Why it happened @@ -26,8 +27,8 @@ There's a separate per-IP cap (500 requests per minute) sitting on top of the pe ## Related -- [M201: Per-user rate limit](/errors/M201) -- [M203: Concurrency limit](/errors/M203) -- [M200: Usage limit exceeded](/errors/M200) -- [Self-hosted setup](/self-hosted) -- [All error codes](/errors) +- [M201: Per-user rate limit](/llm-gateway/docs/errors/M201) +- [M203: Concurrency limit](/llm-gateway/docs/errors/M203) +- [M200: Usage limit exceeded](/llm-gateway/docs/errors/M200) +- [Self-hosted setup](/llm-gateway/docs/self-hosted) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M203.mdx b/errors/M203.mdx index 4d93848..08aced8 100644 --- a/errors/M203.mdx +++ b/errors/M203.mdx @@ -5,13 +5,14 @@ description: "Manifest error M203 fires when a workspace has more than 10 in-fli icon: "timer" keywords: ["M203", "Manifest M203", "concurrency limit", "in-flight requests", "429 Too Many Requests", "parallel streams", "Manifest limits"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M203" --- ## What you saw ```text [🦚 Manifest M203] Too many concurrent requests. Give it a moment. -See https://manifest.build/docs/errors/M203 +See https://manifest.build/llm-gateway/docs/errors/M203 ``` ## Why it happened @@ -26,8 +27,8 @@ Manifest caps in-flight requests at 10 per workspace, shared by all its harnesse ## Related -- [M201: Per-user rate limit](/errors/M201) -- [M202: Per-IP rate limit](/errors/M202) -- [M200: Usage limit exceeded](/errors/M200) -- [Self-hosted setup](/self-hosted) -- [All error codes](/errors) +- [M201: Per-user rate limit](/llm-gateway/docs/errors/M201) +- [M202: Per-IP rate limit](/llm-gateway/docs/errors/M202) +- [M200: Usage limit exceeded](/llm-gateway/docs/errors/M200) +- [Self-hosted setup](/llm-gateway/docs/self-hosted) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M204.mdx b/errors/M204.mdx index d9f0b10..78e2b1e 100644 --- a/errors/M204.mdx +++ b/errors/M204.mdx @@ -5,13 +5,14 @@ description: "Manifest error M204 fires when a Manifest Cloud Free workspace use icon: "badge-alert" keywords: ["M204", "Manifest M204", "PLAN_LIMIT_REQUESTS", "monthly request limit", "free plan request limit", "Manifest billing", "402 Payment Required", "upgrade to Pro"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M204" --- ## What you saw ```text [🦚 Manifest M204] You've used all 10000 requests included this month on the Free plan. Upgrade to Pro for unlimited requests: https://app.manifest.build/upgrade?reason=requests -See https://manifest.build/docs/errors/M204 +See https://manifest.build/llm-gateway/docs/errors/M204 ``` SDK and tool callers receive an HTTP 402 JSON error with `error.code` set to `PLAN_LIMIT_REQUESTS`. @@ -29,5 +30,5 @@ Your Manifest Cloud workspace reached the Free plan monthly request quota. This ## Related - [Upgrade dashboard](https://app.manifest.build/upgrade?reason=requests) -- [M200: Usage limit exceeded](/errors/M200) -- [All error codes](/errors) +- [M200: Usage limit exceeded](/llm-gateway/docs/errors/M200) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M300.mdx b/errors/M300.mdx index 92e0501..b5705a3 100644 --- a/errors/M300.mdx +++ b/errors/M300.mdx @@ -5,13 +5,14 @@ description: "Manifest error M300 fires when the /v1/chat/completions body has n icon: "message-square-warning" keywords: ["M300", "Manifest M300", "messages array required", "empty messages", "400 Bad Request", "OpenAI compatible API", "chat completions validation", "Manifest validation"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M300" --- ## What you saw ```text [🦚 Manifest M300] `messages` array is required. -See https://manifest.build/docs/errors/M300 +See https://manifest.build/llm-gateway/docs/errors/M300 ``` ## Why it happened @@ -35,6 +36,6 @@ The OpenAI SDK handles this for you. If you're hand-rolling the request, eyeball ## Related -- [M303: Local provider unavailable on Manifest Cloud](/errors/M303) -- [API reference](/reference/api) -- [All error codes](/errors) +- [M303: Local provider unavailable on Manifest Cloud](/llm-gateway/docs/errors/M303) +- [API reference](/llm-gateway/docs/reference/api) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M302.mdx b/errors/M302.mdx index 7168022..9e3ac97 100644 --- a/errors/M302.mdx +++ b/errors/M302.mdx @@ -5,13 +5,14 @@ description: "Manifest error M302 fires when an explicit model ID is not availab icon: "message-square-warning" keywords: ["M302", "Manifest M302", "model not available", "model not found", "GET /v1/models", "direct model routing", "provider not connected", "provider not enabled", "agent-visible models", "harness-visible models", "Manifest validation"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M302" --- ## What you saw ```text [🦚 Manifest M302] Model "openai/gpt-4o" is not available for this agent. Use GET /v1/models to list available model IDs, or make the provider available for this agent here: https://app.manifest.build/... -See https://manifest.build/docs/errors/M302 +See https://manifest.build/llm-gateway/docs/errors/M302 ``` The model name varies based on the `model` value your client sent. @@ -36,8 +37,8 @@ This can happen when: ## Related -- [Routing: Route a specific model](/llm-gateway#route-a-specific-model) -- [API reference: Listing models](/reference/api#listing-models) -- [M300: Missing messages array](/errors/M300) -- [M303: Local provider unavailable on Manifest Cloud](/errors/M303) -- [All error codes](/errors) +- [Routing: Route a specific model](/llm-gateway/docs/llm-gateway#route-a-specific-model) +- [API reference: Listing models](/llm-gateway/docs/reference/api#listing-models) +- [M300: Missing messages array](/llm-gateway/docs/errors/M300) +- [M303: Local provider unavailable on Manifest Cloud](/llm-gateway/docs/errors/M303) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M303.mdx b/errors/M303.mdx index cde9a5f..b1cf4df 100644 --- a/errors/M303.mdx +++ b/errors/M303.mdx @@ -5,18 +5,19 @@ description: "Manifest error M303 fires when a Manifest Cloud request routes to icon: "server-off" keywords: ["M303", "Manifest M303", "local provider cloud", "Ollama Manifest Cloud", "LM Studio", "llama.cpp", "400 Bad Request", "custom provider", "tunnel"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M303" --- ## What you saw ```text [🦚 Manifest M303] Built-in local providers are only available in self-hosted Manifest. On Manifest Cloud, expose the runtime through a public URL or tunnel and connect it as a custom provider. -See https://manifest.build/docs/errors/M303 +See https://manifest.build/llm-gateway/docs/errors/M303 ``` ## Why it happened -The built-in [local provider](/providers/local-models) tiles (Ollama, LM Studio, llama.cpp) route to `localhost`. On a self-hosted install that's your machine, which is the whole point. On Manifest Cloud, `localhost` is our server, which has no model running on it and no route back to your laptop. +The built-in [local provider](/llm-gateway/docs/providers/local-models) tiles (Ollama, LM Studio, llama.cpp) route to `localhost`. On a self-hosted install that's your machine, which is the whole point. On Manifest Cloud, `localhost` is our server, which has no model running on it and no route back to your laptop. Rather than dial an address that can't be yours, Manifest stops the request here. @@ -27,13 +28,13 @@ Pick whichever fits: Local providers work as designed on a self-hosted install, and your prompts - never leave your network. See [Self-hosted](/self-hosted) for the one-line + never leave your network. See [Self-hosted](/llm-gateway/docs/self-hosted) for the one-line Docker install. Keep using Cloud and give your runtime a public address. A tunnel (Cloudflare Tunnel, ngrok, Tailscale Funnel) is the usual way. Then add it - under [Custom providers](/providers/custom-providers) using the public URL + under [Custom providers](/llm-gateway/docs/providers/custom-providers) using the public URL instead of the built-in tile. Put authentication in front of it. A public URL with no key is an open @@ -43,8 +44,8 @@ Pick whichever fits: ## Related -- [Local models](/providers/local-models) -- [Custom providers](/providers/custom-providers) -- [Self-hosted](/self-hosted) -- [M302: Model not available](/errors/M302) -- [All error codes](/errors) +- [Local models](/llm-gateway/docs/providers/local-models) +- [Custom providers](/llm-gateway/docs/providers/custom-providers) +- [Self-hosted](/llm-gateway/docs/self-hosted) +- [M302: Model not available](/llm-gateway/docs/errors/M302) +- [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M500.mdx b/errors/M500.mdx index 8a74600..9a42763 100644 --- a/errors/M500.mdx +++ b/errors/M500.mdx @@ -5,13 +5,14 @@ description: "Manifest error M500 is a generic 500 fallback for unhandled errors icon: "server-crash" keywords: ["M500", "Manifest M500", "internal server error", "500 Internal Server Error", "5xx", "proxy crash", "Manifest server error", "self-hosted debugging"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M500" --- ## What you saw ```text [🦚 Manifest M500] Something broke on our end. Try again in a moment. -See https://manifest.build/docs/errors/M500 +See https://manifest.build/llm-gateway/docs/errors/M500 ``` ## Why it happened @@ -26,6 +27,6 @@ Manifest hit an unexpected error while handling your request. Usually that's a d ## Related -- [Self-hosted setup](/self-hosted) -- [Fallback](/llm-gateway#fallback) -- [All error codes](/errors) +- [Self-hosted setup](/llm-gateway/docs/self-hosted) +- [Fallback](/llm-gateway/docs/llm-gateway#fallback) +- [All error codes](/llm-gateway/docs/errors) diff --git a/integrations/n8n.mdx b/integrations/n8n.mdx index 337ffd3..21148f0 100644 --- a/integrations/n8n.mdx +++ b/integrations/n8n.mdx @@ -4,6 +4,7 @@ description: "Use Manifest as the chat model behind n8n AI Agents, or call it di icon: "workflow" keywords: ["n8n", "n8n community node", "n8n-nodes-manifest", "n8n AI Agent", "Manifest Chat Model", "n8n self-hosted", "n8n LLM router", "n8n OpenAI alternative"] +canonical: "https://manifest.build/llm-gateway/docs/integrations/n8n" ---
@@ -22,7 +23,7 @@ Manifest ships as a verified n8n community node, [`n8n-nodes-manifest`](https:// -Both nodes are plain HTTP clients for the [gateway](/llm-gateway). Nothing about routing, limits, or [Autofix](/autofix) changes because the caller is n8n. +Both nodes are plain HTTP clients for the [gateway](/llm-gateway/docs/llm-gateway). Nothing about routing, limits, or [Autofix](/llm-gateway/docs/autofix) changes because the caller is n8n. ## Install the node @@ -73,7 +74,7 @@ Select **Test** on the credential. It calls `GET /v1/models`, so a passing test The credential test fails with a connection error when n8n cannot reach Manifest. Where the two run decides the address: - **n8n and Manifest in Docker on the same host.** `localhost` inside the n8n container is n8n itself. Put both containers on a shared Docker network and use the Manifest service name as the host, or use `http://host.docker.internal:2099` on Docker Desktop. -- **n8n on another machine.** The bundled compose file binds Manifest to `127.0.0.1` only. Set `HOST_BIND_ADDRESS=0.0.0.0` in Manifest's `.env`, as described in [Exposing on the LAN](/self-hosted#exposing-on-the-lan), so the port answers on the network. +- **n8n on another machine.** The bundled compose file binds Manifest to `127.0.0.1` only. Set `HOST_BIND_ADDRESS=0.0.0.0` in Manifest's `.env`, as described in [Exposing on the LAN](/llm-gateway/docs/self-hosted#exposing-on-the-lan), so the port answers on the network. - **Plain `http://` is fine.** The calls are server to server. The dashboard's browser security rules do not apply to them. The keyless loopback shortcut that a development build of Manifest accepts does not exist in the production image. n8n always needs a real `mnfst_` key. @@ -87,14 +88,14 @@ The keyless loopback shortcut that a development build of Manifest accepts does On an **AI Agent** or **Basic LLM Chain** node, select the **Model** connector and choose **Manifest Chat Model**. - Select the Manifest credential. The model list is loaded from your instance. Leave it on `auto` to let your [routing rules](/llm-gateway#routing) pick the model, or choose a model ID to pin one. + Select the Manifest credential. The model list is loaded from your instance. Leave it on `auto` to let your [routing rules](/llm-gateway/docs/llm-gateway#routing) pick the model, or choose a model ID to pin one. Streaming and tool calling work as they do with the OpenAI Chat Model, so agent tools run unchanged. Each call appears on the harness's **Requests** page with the model that served it and its cost. -An option on the sub-node switches it from Chat Completions to the [Responses API](/reference/api#endpoints). Leave it off unless a model you route to needs Responses. +An option on the sub-node switches it from Chat Completions to the [Responses API](/llm-gateway/docs/reference/api#endpoints). Leave it off unless a model you route to needs Responses. ## Call Manifest from a workflow @@ -123,4 +124,4 @@ Your routing decides whether the route answers with buffered JSON or a stream. T | Credential test fails with a connection error | n8n cannot reach the Base URL. See [Reaching a self-hosted instance](#reaching-a-self-hosted-instance-from-n8n). | | Credential test fails with `401` | The key is not a valid `mnfst_` harness key. Copy it again from the harness **Settings** page. | | Every call returns `404` | The Base URL ends in `/v1`. Remove it. | -| The model list shows only `auto`, or calls fail with [M100](/errors/M100) or [M101](/errors/M101) | No provider is connected to the harness yet. Connect one on the harness's **Providers** tab. | +| The model list shows only `auto`, or calls fail with [M100](/llm-gateway/docs/errors/M100) or [M101](/llm-gateway/docs/errors/M101) | No provider is connected to the harness yet. Connect one on the harness's **Providers** tab. | diff --git a/introduction.mdx b/introduction.mdx index 9c84419..216e522 100644 --- a/introduction.mdx +++ b/introduction.mdx @@ -5,6 +5,7 @@ description: "Open-source LLM router for agents. Connect your subscriptions, API icon: "house" keywords: ["LLM router", "AI subscriptions", "model routing", "open source", "LLM gateway", "quickstart", "first request", "create a harness", "gateway base URL"] +canonical: "https://manifest.build/llm-gateway/docs/introduction" ---
@@ -19,19 +20,19 @@ Send `auto` as the model and Manifest routes each request by the rules you set, ## Key features - + Connect API keys, subscriptions you already pay for, local models, or any custom endpoint. - + Send `auto` and your routing rules pick the model, with fallbacks when one fails. Limits cap what a harness can spend. - + Repair a malformed request and resend it, instead of failing on something a retry can't fix. - + See what every harness spent, which models served it, and what failed. - + Read back the full request and response body of any provider attempt, as a conversation. @@ -45,15 +46,15 @@ Manifest is open source and runs two ways. Both speak the same API, so pointing Runs on our servers. Sign up, connect a provider, and you're routing. Recommended if you're starting out. - + Runs on infrastructure you control: Docker on your own machine, a managed platform like Railway, your AWS or GCP account, or a server panel. -On the cloud there is nothing to install and nothing to configure on a server. You sign up, connect your providers, point your harness at the gateway URL, and everything else — routing, limits, [Autofix](/autofix), alerts — is a setting in the dashboard. The [environment variables](/reference/environment-variables) throughout these docs apply only to a Manifest instance you run yourself. +On the cloud there is nothing to install and nothing to configure on a server. You sign up, connect your providers, point your harness at the gateway URL, and everything else — routing, limits, [Autofix](/llm-gateway/docs/autofix), alerts — is a setting in the dashboard. The [environment variables](/llm-gateway/docs/reference/environment-variables) throughout these docs apply only to a Manifest instance you run yourself. -Cloud is also subject to [plan limits](/errors/M204): the free plan includes 10,000 routed requests per calendar month, shown by the usage meter in the dashboard sidebar and reset at 00:00 UTC on the 1st. And it can't reach [local models](/providers/local-models) running on your machine. A self-hosted instance has neither restriction. +Cloud is also subject to [plan limits](/llm-gateway/docs/errors/M204): the free plan includes 10,000 routed requests per calendar month, shown by the usage meter in the dashboard sidebar and reset at 00:00 UTC on the 1st. And it can't reach [local models](/llm-gateway/docs/providers/local-models) running on your machine. A self-hosted instance has neither restriction. ## Your first request @@ -78,6 +79,6 @@ curl -X POST https://app.manifest.build/v1/chat/completions \
-Install Manifest first: [Self-hosted](/self-hosted) covers Docker and the managed platforms. Your instance starts empty. [First request](/self-hosted#first-request) takes you from there to a routed request, against `http://localhost:2099/v1`. +Install Manifest first: [Self-hosted](/llm-gateway/docs/self-hosted) covers Docker and the managed platforms. Your instance starts empty. [First request](/llm-gateway/docs/self-hosted#first-request) takes you from there to a routed request, against `http://localhost:2099/v1`.
diff --git a/llm-gateway.mdx b/llm-gateway.mdx index db2ec17..6fa8c30 100644 --- a/llm-gateway.mdx +++ b/llm-gateway.mdx @@ -4,6 +4,7 @@ description: "Every request passes through the gateway: limits check it, routing icon: "split" keywords: ["LLM gateway", "model routing", "LLM router", "fallback chain", "hard limit", "custom tier", "default model", "fallback exhausted", "429 too many requests"] +canonical: "https://manifest.build/llm-gateway/docs/llm-gateway" --- Point your client at Manifest instead of a provider, and every request goes through the gateway. Three things happen there: limits decide whether the request runs at all, routing picks the model, and fallback steps in when that model fails. @@ -12,10 +13,10 @@ Point your client at Manifest instead of a provider, and every request goes thro - The `Authorization: Bearer mnfst_...` header resolves to one of your harnesses. A bad or missing key stops here with a [401](/errors). + The `Authorization: Bearer mnfst_...` header resolves to one of your harnesses. A bad or missing key stops here with a [401](/llm-gateway/docs/errors). - If a hard limit for this harness is already over its threshold, the request is blocked with [M200](/errors/M200) before any provider is called. Nothing is spent. + If a hard limit for this harness is already over its threshold, the request is blocked with [M200](/llm-gateway/docs/errors/M200) before any provider is called. Nothing is spent. A custom tier matches on a request header, or the request goes to your default tier. Send a real model ID instead of `auto` to skip routing. A matching custom tier header still wins. @@ -24,7 +25,7 @@ Point your client at Manifest instead of a provider, and every request goes thro Manifest calls the resolved provider with your credentials and streams the response back. - If the provider returns an error, [Autofix](/autofix) may repair and resend the request once. Anything still failing moves down the fallback chain. + If the provider returns an error, [Autofix](/llm-gateway/docs/autofix) may repair and resend the request once. Anything still failing moves down the fallback chain. @@ -72,7 +73,7 @@ Create as many tiers as you need, each with its own model and parameters. This i ### Route a specific model -To skip routing for a single request, send a real model ID instead of `auto`. Manifest forwards it straight to that model's provider, with no tier lookup and no fallbacks. A matching [custom tier](#custom-tiers) header is the exception. The request then runs on the tier's model and the tier's fallbacks. Call [`GET /v1/models`](/reference/api#listing-models) to list the model IDs your harness can reach. +To skip routing for a single request, send a real model ID instead of `auto`. Manifest forwards it straight to that model's provider, with no tier lookup and no fallbacks. A matching [custom tier](#custom-tiers) header is the exception. The request then runs on the tier's model and the tier's fallbacks. Call [`GET /v1/models`](/llm-gateway/docs/reference/api#listing-models) to list the model IDs your harness can reach. ```bash curl -X POST https://app.manifest.build/v1/chat/completions \ @@ -86,7 +87,7 @@ curl -X POST https://app.manifest.build/v1/chat/completions \ The response comes back with `X-Manifest-Tier: direct`, so you can tell a direct call from a routed one. A direct model ID works on all three endpoints: `/v1/chat/completions`, `/v1/responses`, and the Anthropic `/v1/messages`. -The model list is scoped to the harness key. A model may be missing because its provider is not connected yet, or because the provider exists in your workspace but is not enabled for this harness. If you send a model ID that has no connected provider route, Manifest returns [M302: Model not available](/errors/M302). Send `auto` to use routing. +The model list is scoped to the harness key. A model may be missing because its provider is not connected yet, or because the provider exists in your workspace but is not enabled for this harness. If you send a model ID that has no connected provider route, Manifest returns [M302: Model not available](/llm-gateway/docs/errors/M302). Send `auto` to use routing. ## Fallback @@ -107,7 +108,7 @@ Any HTTP status code **>= 400** triggers a fallback. | **503** | Service unavailable | | **529** | Provider overloaded | -A tier also falls back when its pinned model leaves your model list. When the tier has no fallback left, the request returns [M101](/errors/M101). +A tier also falls back when its pinned model leaves your model list. When the tier has no fallback left, the request returns [M101](/llm-gateway/docs/errors/M101). ### Configuration @@ -133,15 +134,15 @@ Fallback models are configured **per tier**. Each tier holds up to **5 fallback A provider that opens a connection but never returns eventually triggers a fallback via Manifest's per-attempt timeout (default 180 seconds), which surfaces as a synthetic `504 Gateway Timeout` and moves to the next model in the chain. -If your upstream client has its own timeout that fires at the same time, the client may disconnect first and Manifest will give up before reaching a healthy fallback. On self-hosted installs, lower [`PROVIDER_TIMEOUT_MS`](/reference/environment-variables) strictly below your client's timeout so the chain has room to run inside the client's window. +If your upstream client has its own timeout that fires at the same time, the client may disconnect first and Manifest will give up before reaching a healthy fallback. On self-hosted installs, lower [`PROVIDER_TIMEOUT_MS`](/llm-gateway/docs/reference/environment-variables) strictly below your client's timeout so the chain has room to run inside the client's window. ### Response headers -When a fallback succeeds, the response carries `X-Manifest-Fallback-From` (the primary that failed) and `X-Manifest-Fallback-Index` (its position in the chain) on top of the standard routing headers. When the chain is exhausted, `X-Manifest-Fallback-Exhausted: true` is set, the body carries the provider's own error code (or `fallback_exhausted` when there is none), and the response keeps the primary model's real error status (for example 500 or 529). Full table: [Headers reference](/reference/headers). +When a fallback succeeds, the response carries `X-Manifest-Fallback-From` (the primary that failed) and `X-Manifest-Fallback-Index` (its position in the chain) on top of the standard routing headers. When the chain is exhausted, `X-Manifest-Fallback-Exhausted: true` is set, the body carries the provider's own error code (or `fallback_exhausted` when there is none), and the response keeps the primary model's real error status (for example 500 or 529). Full table: [Headers reference](/llm-gateway/docs/reference/headers). ## Hard limits -A limit rule can block requests as well as email you. When a rule's action includes blocking and the harness is over its threshold for the current period, the gateway returns [M200](/errors/M200) as an HTTP `200` response whose assistant message carries the block text, so your agent keeps flowing instead of throwing: +A limit rule can block requests as well as email you. When a rule's action includes blocking and the harness is over its threshold for the current period, the gateway returns [M200](/llm-gateway/docs/errors/M200) as an HTTP `200` response whose assistant message carries the block text, so your agent keeps flowing instead of throwing: ```text [🦚 Manifest M200] You hit your cost limit: $12.50 used, $10.00/day allowed. Adjust it here: https://app.manifest.build/... @@ -151,16 +152,16 @@ The check runs before any provider is contacted, so a blocked request costs noth On Manifest Cloud, running out of the Free plan's monthly requests is a - different thing entirely: that returns [M204](/errors/M204) with HTTP `402` + different thing entirely: that returns [M204](/llm-gateway/docs/errors/M204) with HTTP `402` and isn't something you configured. -Rules are defined per harness, with a metric, a threshold, and a period. See [Spend alerts](/observability#spend-alerts) for how to create one and how the alerting half works. +Rules are defined per harness, with a metric, a threshold, and a period. See [Spend alerts](/llm-gateway/docs/observability#spend-alerts) for how to create one and how the alerting half works. ## Related -- [Autofix](/autofix) — repair a failing request before fallback runs -- [Observability](/observability) — what the gateway records, and spend alerts -- [API reference](/reference/api) -- [Headers reference](/reference/headers) -- [Error codes](/errors) +- [Autofix](/llm-gateway/docs/autofix) — repair a failing request before fallback runs +- [Observability](/llm-gateway/docs/observability) — what the gateway records, and spend alerts +- [API reference](/llm-gateway/docs/reference/api) +- [Headers reference](/llm-gateway/docs/reference/headers) +- [Error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/autofix.mdx b/llm-gateway/docs/autofix.mdx new file mode 100644 index 0000000..1a34e7e --- /dev/null +++ b/llm-gateway/docs/autofix.mdx @@ -0,0 +1,188 @@ +--- +title: "Autofix" +description: "Repair a malformed request and resend it once, before the fallback chain runs. Turn it on per harness." +icon: "wrench" +keywords: + ["auto-fix", "autofix", "self-healing", "LLM request repair", "malformed request", "400 bad request", "422 unprocessable", "model not found", "agent debugging", "harness debugging"] +canonical: "https://manifest.build/llm-gateway/docs/autofix" +--- + +
+ + +
+ +Some requests fail because of the request, not the model. A parameter the provider doesn't accept, a tool schema in the wrong shape, a model name that moved. Switching to a fallback model doesn't help, because the next model rejects the same body. + +Autofix handles that case. When a request fails with a repairable error, Manifest sends the failed request and the provider's error to a healing service, gets a corrected request back, and sends it once more. It runs **before** the fallback chain, so fallback is still the safety net if the repair doesn't land. + +## When it runs + +Autofix only touches request-side 4xx errors, where the provider is telling you the request was wrong: + +| Status | Meaning | +| ------ | ---------------------------------------------- | +| `400` | Bad request, usually a malformed parameter | +| `404` | Not found, usually a model name that moved | +| `422` | Unprocessable, usually a schema the API rejects | + +It deliberately skips `401`, `403`, `429`, and every `5xx`. A bad key, a rate limit, or a provider outage isn't something a rewritten body can fix, so those go straight to [fallback](/llm-gateway/docs/llm-gateway#fallback). + +Renamed models are covered through the provider itself. When a request names a model Manifest hasn't catalogued but the provider and credentials are unambiguous, Manifest forwards it anyway. The provider answers with its real error, typically a `404` for a model that no longer exists, and that error goes through the normal Autofix path. A patch that corrects the model name serves the request. + +## How it works + + + + A provider returns a repairable 4xx. Manifest normalizes the error into a + message, type, and parameter. + + + The failed request and the normalized error go to the hosted healing + service, which returns a corrected request body. + + + Manifest sends the patched request. There is no retry budget: it tries + exactly once. + + + If the patched request fails too, Manifest reports the outcome and hands + off to your fallback chain as normal. + + + +Autofix covers non-streaming responses, and streaming responses that fail before the first byte reaches your client. Once bytes are on the wire there's nothing to rewrite. + +## Turning it on + +Autofix is a per-harness toggle, and you meet it at two moments. + +
+ +Nothing to deploy or configure: your install talks to Manifest's hosted healing service directly. + +
+ +### When you create a harness + +The Connect Harness modal shows an Autofix toggle under the name field, pre-set to on. You decide there, and the choice is stored on the harness. This works the same on Cloud and self-hosted. + + + The Connect Harness modal with the Autofix toggle switched on under the name field + + +### Harnesses you already have + +A harness that existed before Autofix keeps its state: Manifest never flips the toggle for you. + +
+ +On Manifest Cloud, those harnesses are already on. You can switch any of them off on its **Settings** page. + +
+ +
+ +Enable each one from its **Settings** page or from the sidebar card. + +
+ +The dashboard shows a sidebar card while any harness has Autofix off. It opens a modal listing those harnesses with individual toggles, and you can dismiss it for the browser session. + + + A harness's Settings page with the Autofix failing requests toggle switched on + + + + The Enable Autofix modal listing harnesses that have Autofix off, each with its own toggle + + +## Your data + +
+ +Here is exactly what Autofix sends to the healing service, and when nothing is sent. + +
+ +
+ +Self-hosting is often a privacy choice, so here is exactly what leaves your machine, and when nothing does. + +The first time you enable Autofix, your install records a one-time consent: enabling it means you agree that failing requests are sent to the hosted healing service, as set out in the [terms](https://manifest.build/terms/). Any enable path records it. Your install identifies itself with an anonymous install id, the same id [telemetry](/llm-gateway/docs/reference/telemetry) uses. + +
+ +When Autofix repairs a request, Manifest sends the failed request, including its message content, plus the provider's error response to the healing service. + +Provider API keys and OAuth tokens are never sent: credentials travel in request headers, which are not part of the payload sent to the healing service. The provider's error response is scrubbed for secrets before it is sent. + +Nothing is sent for requests that succeed. No request data is ever sent while the toggle is off; the only call that can still happen is the boot health check, which carries no request content. + +
+ +Self-hosted deployments can also switch Autofix off globally, which stops every call to the healing service, including the health check at boot. See [Environment variables](#environment-variables). + +
+ +See the [privacy policy](https://manifest.build/privacy/) and the [terms](https://manifest.build/terms/). + +## Guardrails + +Autofix is built to stay out of the way when it isn't working: + +- **One attempt.** A patched request is sent once. It never loops. +- **Failures degrade quietly.** If the healing service errors or times out, your agent gets the original provider error. Autofix never turns a provider's 400 into a Manifest 500. +- **A slow healer stops being asked.** After three consecutive transport failures, Manifest stops calling it for 30 seconds so a down service doesn't add latency to every failing request. One successful call clears the streak. +- **Startup never waits.** In production, Manifest pings the healing service once at boot, fire-and-forget, and logs a warning if it's unreachable. It never blocks startup, and it's skipped entirely when Autofix is disabled globally. +- **Fallback still runs.** Autofix sits in front of the chain, it doesn't replace it. + +## Seeing what it did + +Open any request in the dashboard's request log. Requests that went through Autofix carry a panel showing what changed and whether the retry succeeded. Only a successful patched retry counts as recovered by Autofix. + + + The request drawer on a repaired request, with the failed and patched attempts listed and the Autofix panel showing what changed + + +The failed original and its patched retry are separate [provider attempts](/llm-gateway/docs/observability#requests-and-provider-attempts), so with [request logs](/llm-gateway/docs/request-logs) on you can read both bodies side by side and see exactly which field was rewritten. + +The **Overview** page aggregates the same data across a time range: + +| Card | What it measures | +|---|---| +| **Success rate** | Share of requests that ended successfully, however many attempts it took | +| **Recovered requests** | Share of requests that failed at least one attempt and still succeeded | +| **Recovered by Autofix** | Count of requests where a patched retry is what saved it | + +Both **Recovered** cards link through to the matching filter on the request log. + +
+ +## Environment variables + + + Self-hosted only. On [Manifest Cloud](https://app.manifest.build) there is no + server to configure: everything Autofix related lives in the dashboard. + + +Autofix needs no setup. Every variable below is optional and adjusts behavior: + +| Variable | Default | Description | +| ----------------------------- | ------------- | ------------------------------------------------------ | +| `AUTOFIX_GLOBAL_ENABLED` | `true` | Set `false` to turn Autofix off for the whole deployment. No call reaches the healing service, including the boot health check | +| `AUTOFIX_TIMEOUT_MS` | `10000` | Timeout per healing call, in milliseconds | +| `AUTOFIX_REPAIRABLE_STATUSES` | `400,404,422` | Which provider statuses are eligible for a repair | +| `AUTOFIX_HEALING_API_KEY` | unset | Sent as `x-api-key` to the healing service. Only relevant if Manifest issues you one | +| `AUTOFIX_REPORT_ALL_4XX` | `false` | Opt-in. Also report request-side 4xx errors (except `401`/`402`/`403`/`429`) from harnesses with Autofix on, as diagnostic evidence with the secret-scrubbed request body. Bodies over 256 KB are dropped, not truncated. No fix comes back from it | + +You don't need `AUTOFIX_HEALING_API_KEY`. With no key set, the install announces its anonymous install id instead. The full list of variables lives in [Environment variables](/llm-gateway/docs/reference/environment-variables). + +
+ +## Related + +- [LLM Gateway](/llm-gateway/docs/llm-gateway): where Autofix sits in the request flow +- [Observability](/llm-gateway/docs/observability): finding failing requests in the log +- [Request logs](/llm-gateway/docs/request-logs): reading the original and patched bodies +- [Error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/deploy.mdx b/llm-gateway/docs/deploy.mdx new file mode 100644 index 0000000..93cf485 --- /dev/null +++ b/llm-gateway/docs/deploy.mdx @@ -0,0 +1,98 @@ +--- +title: "Self-hosting Manifest" +sidebarTitle: "Overview" +description: "Every way to run your own Manifest instance: Docker on your machine, a managed platform like Railway, your own AWS or GCP account, or a server panel like Coolify." +icon: "rocket" +keywords: + [ + "self-hosted Manifest", + "Manifest deployment", + "Docker", + "Railway", + "Render", + "AWS", + "GCP", + "Fly.io", + "Easypanel", + "Coolify", + ] +canonical: "https://manifest.build/llm-gateway/docs/deploy" +--- + +Everything on this page runs the same open-source Manifest image on infrastructure you control, with your own PostgreSQL database. Pick the path that matches where you want it to live. If you'd rather not run anything yourself, [Manifest Cloud](https://app.manifest.build) is the hosted alternative. + +## Your own machine + + + The most direct path. One install command, or docker-compose if you want to + edit the config first. Also covers upgrades, backups, and bringing your own + PostgreSQL. + + +## Managed platforms + +Someone else runs the infrastructure. Usually a one-click template or deploy button. + + + + Best one-click path for most users. The template provisions Manifest and + PostgreSQL. + + + Blueprint-based deploy using the public Docker image, generated secrets, and + Render PostgreSQL. + + + App Platform deploy from the public repository with Manifest and a Dev + PostgreSQL database. + + + CLI-based deploy using the public Docker image, Fly Postgres, and generated + Manifest secrets. + + + Deploy-button guide with Heroku Postgres, generated secrets, and one + required BETTER_AUTH_URL value. + + + Deploy-button guide for Koyeb using the public Docker image, manual Postgres + URL, and runtime secrets. + + + +## Cloud providers + +Your own cloud account, provisioned with the provider's own infrastructure tooling. + + + + CloudFormation quick-create for ECS Fargate, RDS PostgreSQL, Secrets + Manager, and ALB. + + + Cloud Shell guided deploy for Cloud Run, Cloud SQL PostgreSQL, and Secret + Manager. + + + +## Server panels + +A PaaS you run on your own server, so you get deploy buttons on hardware you own. + + + + Docker Compose stack using Coolify magic variables and a private PostgreSQL + container. + + + Easypanel app service using PostgreSQL and the Manifest Docker image. + + + +## Before you deploy + +Most hosted deployments create paid resources. Review each provider's pricing before leaving test apps running. + +For public deployments, make sure `BETTER_AUTH_URL` matches the exact browser URL. If it does not, Better Auth can reject login with an invalid-origin error. + +Whichever path you pick, [Environment variables](/llm-gateway/docs/reference/environment-variables) is the full configuration list, and [Data and telemetry](/llm-gateway/docs/reference/telemetry) covers what your instance stores and what it reports back. diff --git a/llm-gateway/docs/deploy/aws.mdx b/llm-gateway/docs/deploy/aws.mdx new file mode 100644 index 0000000..9ae9176 --- /dev/null +++ b/llm-gateway/docs/deploy/aws.mdx @@ -0,0 +1,94 @@ +--- +title: "Deploy Manifest on AWS" +sidebarTitle: "AWS" +description: "Deploy Manifest on AWS with CloudFormation, ECS Fargate, RDS PostgreSQL, Secrets Manager, and an Application Load Balancer." +icon: "/icons/deploy/aws.svg" +keywords: + ["Manifest deployment", "AWS", "self-hosted Manifest", "PostgreSQL", "Docker"] +canonical: "https://manifest.build/llm-gateway/docs/deploy/aws" +--- + + + CloudFormation quick-create for ECS Fargate, RDS PostgreSQL, Secrets Manager, + and ALB. + + +## Overview + +The AWS template deploys Manifest with ECS Fargate, RDS PostgreSQL, Secrets Manager, CloudWatch Logs, and an Application Load Balancer. It is a good fit when you want Manifest in your own AWS account. + +## Prerequisites + +- An AWS account with billing enabled. +- Permission to create CloudFormation stacks, IAM roles, VPC resources, ECS, RDS, Elastic Load Balancing, CloudWatch Logs, and Secrets Manager secrets. +- AWS CloudShell or a local shell with `aws` and `git` installed. + +This stack creates paid resources, including an Application Load Balancer, ECS Fargate tasks, and RDS PostgreSQL. + +## Deploy + +Open [AWS CloudShell](https://console.aws.amazon.com/cloudshell/home), choose the region you want to deploy into, then run: + +```bash +git clone https://github.com/mnfst/manifest.git +cd manifest +AWS_REGION=us-east-1 ./deploy/aws/deploy.sh +``` + +Change `AWS_REGION` if you deploy outside `us-east-1`. + +You can also use the [CloudFormation quick-create link](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?stackName=manifest&templateURL=https%3A%2F%2Fmnfst-manifest-deploy-templates.s3.us-east-1.amazonaws.com%2Fmanifest.yaml). + +The first deploy usually takes 10-15 minutes because RDS needs time to provision. + +## Configuration + +The deploy script exposes common settings as environment variables: + +```bash +STACK_NAME=manifest \ +SERVICE_NAME=manifest \ +AWS_REGION=us-east-1 \ +IMAGE_URL=docker.io/manifestdotbuild/manifest:6 \ +DATABASE_INSTANCE_CLASS=db.t4g.micro \ +DESIRED_COUNT=1 \ +./deploy/aws/deploy.sh +``` + +The CloudFormation template generates: + +- `DATABASE_URL`. +- `BETTER_AUTH_SECRET`. +- `MANIFEST_ENCRYPTION_KEY`. + +Manifest runs with `PORT=2099`, `BIND_ADDRESS=0.0.0.0`, and `MANIFEST_MODE=selfhosted`. + +The template also creates an S3 bucket for [request logs](/llm-gateway/docs/request-logs). It is retained by design, so deleting the CloudFormation stack leaves the bucket in place. Delete it by hand when you tear the deployment down. + +## Open Manifest + +After deployment, the script prints `ServiceUrl` and `HealthCheckUrl`. + +Verify the deployment: + +```bash +curl -fsS http:///api/v1/health +``` + +Open `ServiceUrl` and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. + +## Production notes + +- The default template exposes Manifest over HTTP on the generated load balancer DNS name. +- Configure TLS with your own domain and ACM certificate before using the deployment for production authentication traffic. +- Set database deletion protection if you want to avoid accidental RDS deletion. + +Relevant AWS docs: + +- [AWS CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/Welcome.html) +- [Amazon ECS on Fargate](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/AWS_Fargate.html) +- [Amazon RDS for PostgreSQL](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_PostgreSQL.html) diff --git a/llm-gateway/docs/deploy/coolify.mdx b/llm-gateway/docs/deploy/coolify.mdx new file mode 100644 index 0000000..b6ad73d --- /dev/null +++ b/llm-gateway/docs/deploy/coolify.mdx @@ -0,0 +1,82 @@ +--- +title: "Deploy Manifest on Coolify" +sidebarTitle: "Coolify" +description: "Deploy Manifest on Coolify with a Docker Compose service stack, generated secrets, and a private PostgreSQL container." +icon: "/icons/deploy/coolify.svg" +keywords: + [ + "Manifest deployment", + "Coolify", + "self-hosted Manifest", + "PostgreSQL", + "Docker", + ] +canonical: "https://manifest.build/llm-gateway/docs/deploy/coolify" +--- + + + Self-hosted PaaS path using Docker Compose, Coolify magic variables, and + PostgreSQL. + + +## Overview + +Coolify is a strong fit for Manifest because its one-click services are Docker Compose templates. Until Manifest is accepted into the Coolify service catalog, you can paste the Compose file from the Manifest repository into a user-defined service stack. + +## Prerequisites + +- A Coolify instance or Coolify Cloud account. +- A server destination configured in Coolify. +- A domain or wildcard domain configured for Coolify's proxy. + +This stack runs on infrastructure you control through Coolify. Your server, storage, and bandwidth costs depend on your provider. + +## Deploy + +1. Open your Coolify project. +2. Click **New Resource**. +3. Choose **Docker Compose Empty** or a user-defined service stack. +4. Paste the contents of [`deploy/coolify/docker-compose.yml`](https://github.com/mnfst/manifest/blob/main/deploy/coolify/docker-compose.yml). +5. Save the service. +6. Confirm Coolify generated values for: + - `SERVICE_URL_MANIFEST_2099` + - `SERVICE_PASSWORD_POSTGRES` + - `SERVICE_HEX_64_AUTH` + - `SERVICE_HEX_64_ENCRYPTION` +7. Deploy. + +The Compose file uses Coolify magic environment variables so the app URL, Postgres password, Better Auth secret, and Manifest encryption key are generated and stay stable between deployments. + +## Open Manifest + +After deployment, open the generated Manifest URL and create the first admin account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. + +Verify the deployment: + +```bash +curl -fsS https:///api/v1/health +``` + +## What Gets Provisioned + +- Manifest Docker image `manifestdotbuild/manifest:6`. +- PostgreSQL 16 container. +- Persistent Docker volume for PostgreSQL data. +- Persistent Docker volume for request logs. +- Generated session, encryption, database password, and public URL values. + +## Production notes + +- Back up both Docker volumes through Coolify or your server provider: PostgreSQL data and request logs. +- Use a real domain with HTTPS before enabling OAuth or email-based login flows. +- The Compose file is the starting point for a future official Coolify one-click service template. + +Relevant Coolify docs: + +- [Coolify services](https://coolify.io/docs/services/introduction) +- [Coolify Docker Compose](https://coolify.io/docs/knowledge-base/docker/compose) +- [Coolify environment variables](https://coolify.io/docs/knowledge-base/environment-variables) diff --git a/llm-gateway/docs/deploy/digitalocean.mdx b/llm-gateway/docs/deploy/digitalocean.mdx new file mode 100644 index 0000000..cb71a2d --- /dev/null +++ b/llm-gateway/docs/deploy/digitalocean.mdx @@ -0,0 +1,77 @@ +--- +title: "Deploy Manifest on DigitalOcean" +sidebarTitle: "DigitalOcean" +description: "Deploy Manifest on DigitalOcean App Platform with the Deploy to DigitalOcean flow and a Dev PostgreSQL database." +icon: "/icons/deploy/digitalocean.svg" +keywords: + [ + "Manifest deployment", + "DigitalOcean", + "self-hosted Manifest", + "PostgreSQL", + "Docker", + ] +canonical: "https://manifest.build/llm-gateway/docs/deploy/digitalocean" +--- + + + App Platform deploy from the public repository with Manifest and a Dev + PostgreSQL database. + + +## Overview + +DigitalOcean App Platform can create a Manifest app from the public repository template. The template uses the public Manifest Docker image and provisions a Dev PostgreSQL database. + +## Prerequisites + +- A DigitalOcean account with billing enabled. +- Access to App Platform in the selected region. +- A private DigitalOcean Space and an access key pair for it, for request logs. + +This stack creates paid resources, including an App Platform service and Dev PostgreSQL database. + +## Deploy + +1. Open the [DigitalOcean deploy link](https://cloud.digitalocean.com/apps/new?repo=https://github.com/mnfst/manifest/tree/main). +2. Review the app spec. +3. Generate two separate secret values: + +```bash +openssl rand -hex 32 +openssl rand -hex 32 +``` + +4. Paste one value into `BETTER_AUTH_SECRET`. +5. Paste the other value into `MANIFEST_ENCRYPTION_KEY`. +6. Fill in the recording storage secrets with your Space's settings: `REQUEST_RECORDING_S3_BUCKET`, `REQUEST_RECORDING_S3_ENDPOINT`, `REQUEST_RECORDING_S3_ACCESS_KEY_ID`, and `REQUEST_RECORDING_S3_SECRET_ACCESS_KEY`. The region defaults to `us-east-1`, which DigitalOcean Spaces accepts as a signing region. +7. Create the app. + +DigitalOcean reads `.do/deploy.template.yaml` from the Manifest repository. The template includes the Manifest service and a Dev PostgreSQL database. + +## Open Manifest + +After App Platform finishes deploying, open the app URL and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. + +Verify the deployment: + +```bash +curl -fsS https:///api/v1/health +``` + +## Notes + +- App Platform has no persistent volumes, so [request logs](/llm-gateway/docs/request-logs) write to your DigitalOcean Space over S3. +- The template appends `uselibpqcompat=true` to DigitalOcean's PostgreSQL URL so Node `pg` handles the platform's `sslmode=require` connection string correctly. +- Upgrade the Dev Database to a managed database for production data. +- Use a custom domain before configuring production OAuth callbacks. + +Relevant DigitalOcean docs: + +- [Deploy to DigitalOcean button](https://docs.digitalocean.com/products/app-platform/how-to/add-deploy-do-button/) +- [App Platform app spec](https://docs.digitalocean.com/products/app-platform/reference/app-spec/) +- [Bindable environment variables](https://docs.digitalocean.com/products/app-platform/how-to/use-environment-variables/#using-bindable-variables-within-environment-variables) diff --git a/llm-gateway/docs/deploy/easypanel.mdx b/llm-gateway/docs/deploy/easypanel.mdx new file mode 100644 index 0000000..065366e --- /dev/null +++ b/llm-gateway/docs/deploy/easypanel.mdx @@ -0,0 +1,112 @@ +--- +title: "Deploy Manifest on Easypanel" +sidebarTitle: "Easypanel" +description: "Deploy Manifest on Easypanel with a Docker image app service, PostgreSQL, generated secrets, and HTTPS." +icon: "/icons/deploy/easypanel.svg" +keywords: + [ + "Manifest deployment", + "Easypanel", + "self-hosted Manifest", + "PostgreSQL", + "Docker", + ] +canonical: "https://manifest.build/llm-gateway/docs/deploy/easypanel" +--- + + + Self-hosted PaaS guide using Easypanel app services, PostgreSQL, and the + Manifest Docker image. + + +## Overview + +Easypanel can run Manifest as a Docker image app service, proxy the dashboard over HTTPS, and provision PostgreSQL in the same project. The Manifest repository includes Easypanel template files that provision the app, database, generated secrets, and proxy port. Until that template is accepted into the Easypanel catalog, you can create the same services manually. + +## Prerequisites + +- A self-hosted Easypanel instance. +- A project with a server destination. +- A domain or generated Easypanel domain for the app. + +This stack runs on infrastructure you control through Easypanel. Your server, storage, and bandwidth costs depend on your provider. + +## Template deploy + +If Manifest is available in your Easypanel template catalog: + +1. Open your Easypanel project. +2. Choose **Templates**. +3. Search for **Manifest**. +4. Keep the default image `manifestdotbuild/manifest:6` unless you need another version. +5. Deploy the template. + +The template provisions Manifest, PostgreSQL, generated secrets, and a domain proxy on port `2099`. + +## Manual deploy + +Create a PostgreSQL service named `manifest-db`, then create an App service from Docker image `manifestdotbuild/manifest:6`. Set the proxy port to `2099`, add a domain, and mark it as the primary domain. + +Set the Manifest environment: + +```env +PORT=2099 +BIND_ADDRESS=0.0.0.0 +DATABASE_URL=postgresql://postgres:@$(PROJECT_NAME)_manifest-db:5432/$(PROJECT_NAME) +BETTER_AUTH_SECRET= +MANIFEST_ENCRYPTION_KEY= +BETTER_AUTH_URL=https://$(PRIMARY_DOMAIN) +MANIFEST_MODE=selfhosted +NODE_ENV=production +SEED_DATA=false +DB_POOL_MAX=10 +AUTH_DB_POOL_MAX=5 +MANIFEST_TELEMETRY_DISABLED=0 +REQUEST_RECORDING_STORAGE=filesystem +REQUEST_RECORDING_FILESYSTEM_PATH=/data/request-recordings +``` + +Generate two different 64-character secrets with `openssl rand -hex 32`. Use an alphanumeric or hex PostgreSQL password in `DATABASE_URL`. If your password contains URL-reserved characters such as `@`, `:`, `/`, `%`, or `#`, percent-encode them before pasting the connection string. + +Add an app volume named `recordings` mounted at `/data/request-recordings` so [request logs](/llm-gateway/docs/request-logs) survive restarts and redeploys. This volume is single-instance; move to S3-compatible storage before running more than one replica. + +`AUTH_DB_POOL_MAX` is read by Manifest's Better Auth database pool. Set `MANIFEST_TELEMETRY_DISABLED=1` if you want to disable anonymous self-hosted telemetry. + +## Open Manifest + +Open the Easypanel domain and create the first admin account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. + +Verify the deployment: + +```bash +curl -fsS https:///api/v1/health +``` + +## Smoke test checklist + +Before submitting a template upstream, verify: + +- The app starts cleanly. +- PostgreSQL connection and migrations succeed. +- `/api/v1/health` returns OK. +- `/setup` opens on the generated domain. +- First admin account creation works. +- Login works on the Easypanel domain without an invalid-origin error. +- Restarting the app and database keeps data persisted in PostgreSQL. + +## Production notes + +- Back up the PostgreSQL service through Easypanel or your server provider. +- Use a real HTTPS domain before enabling OAuth providers. +- Keep `BETTER_AUTH_SECRET` and `MANIFEST_ENCRYPTION_KEY` as separate values. +- If you deploy more than one Manifest replica, lower database pool sizes or upgrade PostgreSQL connection capacity. + +Relevant Easypanel docs: + +- [Easypanel App Service](https://easypanel.io/docs/services/app) +- [Easypanel Services](https://easypanel.io/docs/services) +- [Easypanel templates repository](https://github.com/easypanel-io/templates) diff --git a/llm-gateway/docs/deploy/fly.mdx b/llm-gateway/docs/deploy/fly.mdx new file mode 100644 index 0000000..76061ac --- /dev/null +++ b/llm-gateway/docs/deploy/fly.mdx @@ -0,0 +1,85 @@ +--- +title: "Deploy Manifest on Fly.io" +sidebarTitle: "Fly.io" +description: "Deploy Manifest on Fly.io from the public Docker image with Fly Postgres, generated secrets, and HTTPS." +icon: "/icons/deploy/fly.svg" +keywords: + [ + "Manifest deployment", + "Fly.io", + "self-hosted Manifest", + "PostgreSQL", + "Docker", + ] +canonical: "https://manifest.build/llm-gateway/docs/deploy/fly" +--- + + + CLI-based deploy using the public Docker image, Fly Postgres, and generated + Manifest secrets. + + +## Overview + +Fly.io runs Manifest as a long-running Docker container close to users, with a Fly Postgres database in the same private network. This is not a browser one-click button. The Manifest repository includes a `deploy/fly/deploy.sh` script that creates the app, creates Postgres, attaches `DATABASE_URL`, sets secrets, and deploys the public Docker image. + +## Prerequisites + +- A Fly.io account with billing enabled. +- `flyctl` installed and authenticated with `fly auth login`. +- Permission to create Fly apps and Postgres apps. + +This stack creates paid resources. Review Fly pricing before leaving test apps running. + +## Deploy + +From the Manifest repository root: + +```bash +FLY_APP_NAME=manifest-demo \ +FLY_REGION=cdg \ +FLY_ORG=personal \ +./deploy/fly/deploy.sh +``` + +The script deploys `docker.io/manifestdotbuild/manifest:6`, provisions Fly Postgres, attaches it as `DATABASE_URL`, and generates separate values for `BETTER_AUTH_SECRET` and `MANIFEST_ENCRYPTION_KEY`. Re-running the script keeps existing secrets instead of rotating sessions or encrypted provider credentials. + +## Open Manifest + +Open the deployed app and create the first admin account: + +```text +https://.fly.dev +``` + +Verify the deployment: + +```bash +curl -fsS https://.fly.dev/api/v1/health +``` + +View logs: + +```bash +fly logs --app +``` + +## Production notes + +- The template keeps one Machine running so Manifest is always available for agents. +- For production, choose a larger Postgres configuration or Fly Managed Postgres instead of the small script default. +- Add a custom domain before configuring OAuth callback URLs. +- The deploy script creates a private Tigris bucket, stores its credentials as Fly secrets, and sets `REQUEST_RECORDING_STORAGE=s3` with the Tigris endpoint. [Request logs](/llm-gateway/docs/request-logs) persist across deploys and across scaled Machines. Nothing to attach manually. +- Set `MANIFEST_TELEMETRY_DISABLED=1` as a Fly secret if you want to disable anonymous self-hosted telemetry. +- Destroy both the app and database when testing is done. + +Relevant Fly docs: + +- [Fly app configuration](https://fly.io/docs/reference/configuration/) +- [Deploy an existing Docker image](https://fly.io/docs/reference/configuration/#specify-a-docker-image) +- [Create Postgres](https://fly.io/docs/python/do-more/add-postgres/) +- [Attach Postgres](https://fly.io/docs/postgres/managing/attach-detach/) diff --git a/llm-gateway/docs/deploy/gcp.mdx b/llm-gateway/docs/deploy/gcp.mdx new file mode 100644 index 0000000..8c3390a --- /dev/null +++ b/llm-gateway/docs/deploy/gcp.mdx @@ -0,0 +1,95 @@ +--- +title: "Deploy Manifest on GCP" +sidebarTitle: "GCP" +description: "Deploy Manifest on Google Cloud with Cloud Run, Cloud SQL for PostgreSQL, Secret Manager, and the Cloud Shell DeployStack flow." +icon: "/icons/deploy/gcp.svg" +keywords: + ["Manifest deployment", "GCP", "self-hosted Manifest", "PostgreSQL", "Docker"] +canonical: "https://manifest.build/llm-gateway/docs/deploy/gcp" +--- + + + Cloud Shell guided deploy for Cloud Run, Cloud SQL PostgreSQL, and Secret + Manager. + + +## Overview + +The Google Cloud deploy path uses Cloud Shell and DeployStack to run Terraform. It provisions Cloud Run for the Manifest container, Cloud SQL for PostgreSQL, and Secret Manager for runtime secrets. + +## Prerequisites + +- A Google Cloud project. +- Billing enabled on the selected project. +- Permission to enable APIs and create Cloud Run, Cloud SQL, IAM, and Secret Manager resources. + +This stack creates paid resources, including Cloud SQL. + +## Deploy + +Open the [GCP Cloud Shell guided deploy](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2Fmnfst%2Fmanifest&cloudshell_workspace=deploy%2Fgcp&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io%2Fds-artifacts-cloudshell%2Fdeploystack_custom_image&shellonly=true). + +The walkthrough prompts for: + +- Google Cloud project. +- Region. +- Cloud Run service name. +- Manifest image URL. +- Cloud SQL machine tier. +- Maximum Cloud Run instances. +- Cloud SQL deletion protection. + +Enable the required APIs: + +```bash +gcloud services enable \ + iam.googleapis.com \ + run.googleapis.com \ + sqladmin.googleapis.com \ + secretmanager.googleapis.com +``` + +Run the installer: + +```bash +deploystack install +``` + +The first deploy usually takes 10-15 minutes because Cloud SQL needs time to provision. The Terraform config patches `BETTER_AUTH_URL` to the final Cloud Run URL after the service is created. + +## Open Manifest + +Print the service URL: + +```bash +terraform output service_url +``` + +Open the URL and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. + +Verify the deployment: + +```bash +curl -fsS "$(terraform output -raw health_check_url)" +``` + +## Request logs + +The stack provisions a private Cloud Storage bucket for [request logs](/llm-gateway/docs/request-logs). The deploy prints its name as the `recording_bucket` output. + +## Production notes + +- Cloud SQL deletion protection is enabled by default. +- Configure a custom domain before using OAuth callbacks in production. +- Keep Secret Manager access limited to the Cloud Run service account. +- The request logs bucket keeps its objects on teardown. Export or delete its contents before `deploystack uninstall`. + +Relevant Google Cloud docs: + +- [Cloud Run](https://cloud.google.com/run/docs) +- [Cloud SQL for PostgreSQL](https://cloud.google.com/sql/docs/postgres) +- [Secret Manager](https://cloud.google.com/secret-manager/docs) diff --git a/llm-gateway/docs/deploy/heroku.mdx b/llm-gateway/docs/deploy/heroku.mdx new file mode 100644 index 0000000..557c9fa --- /dev/null +++ b/llm-gateway/docs/deploy/heroku.mdx @@ -0,0 +1,91 @@ +--- +title: "Deploy Manifest on Heroku" +sidebarTitle: "Heroku" +description: "Deploy Manifest on Heroku with a Deploy to Heroku button, a container-stack web dyno, and Heroku Postgres." +icon: "/icons/deploy/heroku.svg" +keywords: + [ + "Manifest deployment", + "Heroku", + "self-hosted Manifest", + "PostgreSQL", + "Docker", + ] +canonical: "https://manifest.build/llm-gateway/docs/deploy/heroku" +--- + + + Deploy-button guide with Heroku Postgres, generated secrets, and required + values for the app URL and S3-compatible recording storage. + + +## Overview + +Heroku can deploy Manifest with the repository `app.json` and `heroku.yml`. The deploy button creates a Cedar container-stack app, provisions Heroku Postgres, generates Manifest secrets, and builds a tiny Heroku Dockerfile that uses the public Manifest image. + +Heroku asks for a few values before it can deploy. Set `BETTER_AUTH_URL` to the public URL for the app name you choose. Heroku dynos have an ephemeral filesystem, so [request logs](/llm-gateway/docs/request-logs) write to external S3-compatible storage rather than local disk: set the bucket, region, and access key for that storage too. + +## Prerequisites + +- A Heroku account with billing enabled. +- Permission to create apps, add-ons, and config vars. +- A private S3-compatible bucket and an access key limited to it, for request logs. + +This stack creates paid resources. Heroku no longer provides free dynos or free Heroku Postgres plans. + +## Deploy + +1. Open the [Heroku deploy link](https://www.heroku.com/deploy?template=https://github.com/mnfst/manifest). +2. Choose an app name. +3. Set `BETTER_AUTH_URL` to: + +```text +https://.herokuapp.com +``` + +4. Fill in the recording storage values: `REQUEST_RECORDING_S3_BUCKET`, `REQUEST_RECORDING_S3_REGION`, `REQUEST_RECORDING_S3_ACCESS_KEY_ID`, and `REQUEST_RECORDING_S3_SECRET_ACCESS_KEY`. Leave `REQUEST_RECORDING_S3_ENDPOINT` blank only when using AWS S3. +5. Deploy the app. + +Heroku generates `BETTER_AUTH_SECRET` and `MANIFEST_ENCRYPTION_KEY`, provisions Postgres as `DATABASE_URL`, builds `Dockerfile.heroku`, and starts the web dyno. + +## Open Manifest + +Open the deployed app and create the first admin account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. + +Verify the deployment: + +```bash +curl -fsS https://.herokuapp.com/api/v1/health +``` + +View logs: + +```bash +heroku logs --tail -a +``` + +## Production notes + +- Heroku sets `PORT` automatically, so the template does not pin a port. +- `PGSSLMODE=no-verify` enables TLS for Heroku Postgres without editing the managed `DATABASE_URL`. +- `DB_POOL_MAX=8` and `AUTH_DB_POOL_MAX=4` leave headroom under the Essential-0 connection limit. +- Use a custom domain before configuring production OAuth callbacks. +- Heroku's filesystem is ephemeral, which is why the deploy collects S3-compatible storage for [request logs](/llm-gateway/docs/request-logs) up front. Recordings live in your bucket, not on the dyno. + +## Tearing it down + +Destroy the app when testing is done to stop billing: + +```bash +heroku apps:destroy -a +``` + +Relevant Heroku docs: + +- [Deploy to Heroku Button](https://devcenter.heroku.com/articles/heroku-button) +- [app.json schema](https://devcenter.heroku.com/articles/app-json-schema) +- [Building Docker Images with heroku.yml](https://devcenter.heroku.com/articles/build-docker-images-heroku-yml) diff --git a/llm-gateway/docs/deploy/koyeb.mdx b/llm-gateway/docs/deploy/koyeb.mdx new file mode 100644 index 0000000..34ad2ba --- /dev/null +++ b/llm-gateway/docs/deploy/koyeb.mdx @@ -0,0 +1,93 @@ +--- +title: "Deploy Manifest on Koyeb" +sidebarTitle: "Koyeb" +description: "Deploy Manifest on Koyeb from the public Docker image with a Koyeb web service and PostgreSQL connection string." +icon: "/icons/deploy/koyeb.svg" +keywords: + [ + "Manifest deployment", + "Koyeb", + "self-hosted Manifest", + "PostgreSQL", + "Docker", + ] +canonical: "https://manifest.build/llm-gateway/docs/deploy/koyeb" +--- + + + Deploy-button guide for Koyeb using the public Docker image, manual Postgres + URL, and runtime secrets. + + +## Overview + +Koyeb can deploy Manifest from the public Docker image with a Deploy to Koyeb button. The button pre-fills the image, HTTP port, and runtime settings, but you must create PostgreSQL separately and replace the placeholder secrets before deploying. + +## Prerequisites + +- A Koyeb account on a paid plan (Pro or higher) with a valid payment method. Since Koyeb joined Mistral AI, deploying requires a paid plan: on an account without one, the deploy link shows an announcement banner instead of the form. +- A Koyeb PostgreSQL Database Service. +- Two random 32+ character secrets for Manifest. +- A private S3-compatible bucket and an access key for it. Koyeb instances have no persistent disk, so request logs need external storage. + +This deployment creates paid resources. + +## Create PostgreSQL + +In Koyeb, create a PostgreSQL Database Service in the same region you plan to use for Manifest. After it is ready, open the database connection details and copy the connection string. + +Manifest uses TLS to connect to Koyeb Postgres, so include `sslmode=require` in the connection string. If the copied URL has no query string, append `?sslmode=require`. If it already has query parameters, append `&sslmode=require`. + +## Generate secrets + +Generate separate values for session signing and at-rest provider credential encryption: + +```bash +openssl rand -hex 32 +openssl rand -hex 32 +``` + +## Deploy Manifest + +1. Open the [Koyeb deploy link](https://app.koyeb.com/deploy?type=docker&image=docker.io%2Fmanifestdotbuild%2Fmanifest%3A6&name=manifest&service_type=web&ports=2099%3Bhttp%3B%2F&env%5BPORT%5D=2099&env%5BDATABASE_URL%5D=postgres%3A%2F%2FUSER%3APASSWORD%40HOST%2FDB%3Fsslmode%3Drequire&env%5BBETTER_AUTH_SECRET%5D=replace-with-openssl-rand-hex-32&env%5BMANIFEST_ENCRYPTION_KEY%5D=replace-with-different-openssl-rand-hex-32&env%5BBETTER_AUTH_URL%5D=https%3A%2F%2F%7B%7B+KOYEB_PUBLIC_DOMAIN+%7D%7D&env%5BMANIFEST_MODE%5D=selfhosted&env%5BBIND_ADDRESS%5D=0.0.0.0&env%5BDB_POOL_MAX%5D=8&env%5BAUTH_DB_POOL_MAX%5D=4&env%5BREQUEST_RECORDING_STORAGE%5D=s3&env%5BREQUEST_RECORDING_S3_BUCKET%5D=replace-with-private-bucket&env%5BREQUEST_RECORDING_S3_ENDPOINT%5D=https%3A%2F%2Fs3.example.com&env%5BREQUEST_RECORDING_S3_REGION%5D=us-east-1&env%5BREQUEST_RECORDING_S3_ACCESS_KEY_ID%5D=replace-with-limited-access-key&env%5BREQUEST_RECORDING_S3_SECRET_ACCESS_KEY%5D=replace-with-secret-key&env%5BREQUEST_RECORDING_S3_FORCE_PATH_STYLE%5D=false). +2. Replace `DATABASE_URL` with your Koyeb Postgres connection string. +3. Replace `BETTER_AUTH_SECRET` with the first generated secret. +4. Replace `MANIFEST_ENCRYPTION_KEY` with the second generated secret. +5. Replace the `REQUEST_RECORDING_S3_*` placeholder values with your bucket, region, endpoint, and access keys. Leave the endpoint blank only for AWS S3. +6. Leave `BETTER_AUTH_URL` as `https://{{ KOYEB_PUBLIC_DOMAIN }}`. +7. Leave `PORT`, `MANIFEST_MODE`, `BIND_ADDRESS`, `DB_POOL_MAX`, `AUTH_DB_POOL_MAX`, and `REQUEST_RECORDING_STORAGE` unchanged for a single-instance deploy. Set `REQUEST_RECORDING_S3_FORCE_PATH_STYLE` to `true` only if your object-storage provider requires path-style URLs. +8. Deploy the service. + +The button deploys `docker.io/manifestdotbuild/manifest:6` and exposes port `2099` over HTTP. + +## Open Manifest + +After the deployment is live, open the public Koyeb domain and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. + +Verify the deployment: + +```bash +curl -fsS https:///api/v1/health +``` + +## Notes + +- Koyeb instances have an ephemeral filesystem, which is why the deploy link collects S3-compatible storage for [request logs](/llm-gateway/docs/request-logs). Recordings live in your bucket, not on the instance. + +## Tearing it down + +Delete both resources when you are done: + +- The Manifest Koyeb Web Service. +- The Koyeb PostgreSQL Database Service. + +Deleting only the web service leaves the database running. + +Relevant Koyeb docs: + +- [Deploy to Koyeb button](https://www.koyeb.com/docs/build-and-deploy/deploy-to-koyeb-button) +- [Koyeb environment variables](https://www.koyeb.com/docs/build-and-deploy/environment-variables) diff --git a/llm-gateway/docs/deploy/railway.mdx b/llm-gateway/docs/deploy/railway.mdx new file mode 100644 index 0000000..8e8aaa7 --- /dev/null +++ b/llm-gateway/docs/deploy/railway.mdx @@ -0,0 +1,66 @@ +--- +title: "Deploy Manifest on Railway" +sidebarTitle: "Railway" +description: "Deploy Manifest on Railway with the published template, including the Manifest service and PostgreSQL." +icon: "/icons/deploy/railway.svg" +keywords: + [ + "Manifest deployment", + "Railway", + "self-hosted Manifest", + "PostgreSQL", + "Docker", + ] +canonical: "https://manifest.build/llm-gateway/docs/deploy/railway" +--- + + + Best one-click path for most users. The template provisions Manifest, + PostgreSQL, and a bucket for request logs. + + +## Overview + +Railway is the fastest hosted path for Manifest. Use the published Manifest template to create a project with the Manifest app, a PostgreSQL database, and a storage bucket for [request logs](/llm-gateway/docs/request-logs), then open the generated domain and create the first admin account. + +## Prerequisites + +- A Railway account. +- A Railway workspace with permission to create projects and services. +- Billing enabled if your usage exceeds the free included resources. + +## Deploy + +1. Open the [Manifest Railway template](https://railway.com/deploy/wild-wild). +2. Choose the Railway workspace and project where you want to deploy Manifest. +3. Review the services that will be created. +4. Deploy the template. +5. Wait for the Manifest service and PostgreSQL database to finish provisioning. + +The template includes the app service, the database, and a `messages-recordings` bucket, so you do not need to create PostgreSQL or recording storage separately. + +## Open Manifest + +Open the generated Railway domain and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. + +Verify the deployment: + +```bash +curl -fsS https:///api/v1/health +``` + +## Notes + +- Use a custom domain before configuring production OAuth callbacks. +- Keep PostgreSQL backups enabled for production data. +- The template provisions a `messages-recordings` bucket for [request logs](/llm-gateway/docs/request-logs), so they survive redeploys without extra setup. +- Use the Railway service logs when debugging failed boots or database connection issues. + +Relevant Railway links: + +- [Railway templates](https://railway.com/templates) +- [Manifest Railway template](https://railway.com/deploy/wild-wild) diff --git a/llm-gateway/docs/deploy/render.mdx b/llm-gateway/docs/deploy/render.mdx new file mode 100644 index 0000000..bcbd4a8 --- /dev/null +++ b/llm-gateway/docs/deploy/render.mdx @@ -0,0 +1,72 @@ +--- +title: "Deploy Manifest on Render" +sidebarTitle: "Render" +description: "Deploy Manifest on Render with the repository Blueprint, including a web service and Render PostgreSQL." +icon: "/icons/deploy/render.svg" +keywords: + [ + "Manifest deployment", + "Render", + "self-hosted Manifest", + "PostgreSQL", + "Docker", + ] +canonical: "https://manifest.build/llm-gateway/docs/deploy/render" +--- + + + Blueprint-based deploy using the public Docker image, generated secrets, and + Render PostgreSQL. + + +## Overview + +Render can deploy Manifest from the repository Blueprint. The Blueprint creates a web service from the public Manifest Docker image and a Render PostgreSQL database, then wires the runtime environment variables. + +## Prerequisites + +- A Render account. +- Permission to create web services and PostgreSQL databases. +- Billing enabled if you select paid plans or outgrow free resources. + +## Deploy + +1. Open the [Render deploy link](https://render.com/deploy?repo=https://github.com/mnfst/manifest). +2. Review the Blueprint resources. +3. Choose the region and plans that fit your deployment. +4. Deploy the Blueprint. +5. Wait for the web service and PostgreSQL database to finish provisioning. + +The repository `render.yaml` sets: + +- `PORT=2099`. +- `DATABASE_URL` from Render PostgreSQL. +- Generated `BETTER_AUTH_SECRET`. +- Generated `MANIFEST_ENCRYPTION_KEY`. +- `BETTER_AUTH_URL` from the Render external service URL. +- `MANIFEST_MODE=selfhosted`. + +## Open Manifest + +Open the Render service URL and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. + +Verify the deployment: + +```bash +curl -fsS https://.onrender.com/api/v1/health +``` + +## Notes + +- Render PostgreSQL connection strings are wired by the Blueprint. +- Use a custom domain before configuring production OAuth callbacks. +- The Blueprint mounts a 1 GB persistent disk `manifest-recordings` at `/data/request-recordings` and sets `REQUEST_RECORDING_STORAGE=filesystem`, so [request logs](/llm-gateway/docs/request-logs) survive redeploys. This disk is single-instance. Point recording at S3-compatible storage before you scale to more than one instance. + +Relevant Render docs: + +- [Blueprints](https://render.com/docs/blueprint-spec) +- [PostgreSQL on Render](https://render.com/docs/databases) diff --git a/llm-gateway/docs/errors.mdx b/llm-gateway/docs/errors.mdx new file mode 100644 index 0000000..05e5ef9 --- /dev/null +++ b/llm-gateway/docs/errors.mdx @@ -0,0 +1,78 @@ +--- +title: "Manifest error codes" +sidebarTitle: "Errors" +description: "Reference for every Manifest proxy error code (M001-M500). What you saw, why it happened, and how to fix it. Covers auth, providers, limits, validation." +icon: "circle-alert" +keywords: + ["Manifest error codes", "M001", "M100", "M102", "M200", "M204", "M302", "M500", "OpenAI compatible errors", "401 Unauthorized", "402 Payment Required", "429 Too Many Requests", "Bearer token error", "chat completions error", "model not available", "troubleshooting", "proxy errors"] +canonical: "https://manifest.build/llm-gateway/docs/errors" +--- + +When Manifest blocks or rejects a request, the response message starts with a code in square brackets: + +```text +[🦚 Manifest M100] No anthropic API key yet. Add one here: https://app.manifest.build/... +See https://manifest.build/llm-gateway/docs/errors/M100 +``` + +Look up the code below to see what it means and how to fix it. + +Except where a section says otherwise, chat and streaming clients receive these codes as an HTTP 200 assistant message carrying the error text, and the HTTP status codes named below apply to non-chat clients (SDKs and tools). + +## Authentication (M001–M005) + +These fire when the bearer token on `/v1/chat/completions` is missing or wrong. They surface as HTTP 401 to non-chat clients. + +| Code | What | +|------|------| +| [M001: Missing Authorization header](/llm-gateway/docs/errors/M001) | No `Authorization` header on the request | +| [M002: Empty Bearer token](/llm-gateway/docs/errors/M002) | Header present, token after `Bearer ` is blank | +| [M003: Invalid key format](/llm-gateway/docs/errors/M003) | Token doesn't start with `mnfst_` | +| [M004: Key expired](/llm-gateway/docs/errors/M004) | Key past its expiration date | +| [M005: Key not recognized](/llm-gateway/docs/errors/M005) | No matching harness for this key | + +## Providers (M100–M102) + +Your Manifest key is fine, but provider credentials are missing or unusable. See [Routing](/llm-gateway/docs/llm-gateway), [API key providers](/llm-gateway/docs/providers/api-key-providers), and [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers). + +| Code | What | +|------|------| +| [M100: Provider API key missing](/llm-gateway/docs/errors/M100) | Routing picked a provider you haven't connected | +| [M101: No providers configured](/llm-gateway/docs/errors/M101) | Harness has zero providers connected | +| [M102: Provider subscription credentials unusable](/llm-gateway/docs/errors/M102) | Subscription OAuth/token exists but cannot be refreshed | + +## Limits (M200–M204) + +You hit a usage cap, rate limit, or Free plan request quota. M200 comes back as an HTTP 200 assistant message for every client: the text rides in the assistant content, not in the HTTP status. M201-M203 surface as HTTP 429 for every client. M204 surfaces as HTTP 402 to non-chat clients because it is a billing plan restriction. + +| Code | What | +|------|------| +| [M200: Usage limit exceeded](/llm-gateway/docs/errors/M200) | Cost or token limit you set was crossed | +| [M201: Per-user rate limit exceeded](/llm-gateway/docs/errors/M201) | More than 200 requests/minute from one user | +| [M202: Per-IP rate limit exceeded](/llm-gateway/docs/errors/M202) | More than 500 requests/minute from one IP | +| [M203: Concurrency limit exceeded](/llm-gateway/docs/errors/M203) | More than 10 in-flight requests at once | +| [M204: Monthly request limit reached](/llm-gateway/docs/errors/M204) | Free plan monthly request quota was exhausted | + +## Request validation and model selection (M300–M303) + +M300 means the request body is malformed and surfaces as HTTP 400. M302 and M303 mean Manifest can't reach the model the request asked for. M302 comes back as an HTTP 200 assistant message for every client. M303 surfaces as HTTP 400 to non-chat clients. + +| Code | What | +|------|------| +| [M300: Missing messages array](/llm-gateway/docs/errors/M300) | Body has no `messages` array, or it's empty | +| [M302: Model not available](/llm-gateway/docs/errors/M302) | Explicit model ID is not available for this harness | +| [M303: Local provider unavailable on Manifest Cloud](/llm-gateway/docs/errors/M303) | Routed to Ollama, LM Studio, or llama.cpp on Cloud | + + + M301 is missing from that list on purpose. It capped requests at 1000 + messages, and the cap was removed. If an older self-hosted build still + returns it, upgrade and the limit goes away. + + +## Server (M500) + +Manifest itself broke. Surfaces as HTTP 500. + +| Code | What | +|------|------| +| [M500: Internal server error](/llm-gateway/docs/errors/M500) | Unhandled error in the proxy pipeline | diff --git a/llm-gateway/docs/errors/M001.mdx b/llm-gateway/docs/errors/M001.mdx new file mode 100644 index 0000000..67d828e --- /dev/null +++ b/llm-gateway/docs/errors/M001.mdx @@ -0,0 +1,34 @@ +--- +title: "M001: Missing Authorization header" +sidebarTitle: "M001" +description: "Manifest error M001 fires when /v1/chat/completions is called without an Authorization header. Returns HTTP 401. Fix: send `Authorization: Bearer mnfst_`." +icon: "key-round" +keywords: + ["M001", "Manifest M001", "Authorization header missing", "Bearer token", "401 Unauthorized", "OpenAI compatible 401", "Manifest auth error", "chat completions authorization"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M001" +--- + +## What you saw + +```text +[🦚 Manifest M001] Missing the Authorization header. Set it to "Bearer mnfst_". +See https://manifest.build/llm-gateway/docs/errors/M001 +``` + +## Why it happened + +Your client called `/v1/chat/completions` without an `Authorization` header. Manifest checks a harness API key on every request, so the header has to be there. + +## How to fix it + +1. Grab your harness's key from the [dashboard](https://app.manifest.build) (Workspace, harness, key). +2. Add the header to every request: `Authorization: Bearer mnfst_`. +3. If you're using an SDK, set the API key in its config instead of building the header by hand. + +## Related + +- [M002: Empty Bearer token](/llm-gateway/docs/errors/M002) +- [M003: Invalid key format](/llm-gateway/docs/errors/M003) +- [M004: Key expired](/llm-gateway/docs/errors/M004) +- [M005: Key not recognized](/llm-gateway/docs/errors/M005) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M002.mdx b/llm-gateway/docs/errors/M002.mdx new file mode 100644 index 0000000..72d2239 --- /dev/null +++ b/llm-gateway/docs/errors/M002.mdx @@ -0,0 +1,33 @@ +--- +title: "M002: Empty Bearer token" +sidebarTitle: "M002" +description: "Manifest error M002 fires when the Authorization header is set but the Bearer token is blank. Often an unexpanded env var. Returns HTTP 401." +icon: "key-round" +keywords: + ["M002", "Manifest M002", "empty Bearer token", "blank token", "401 Unauthorized", "MNFST_KEY env var", "Manifest auth error", "OpenAI compatible 401"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M002" +--- + +## What you saw + +```text +[🦚 Manifest M002] The Bearer token is empty. Paste your Manifest key into it. +See https://manifest.build/llm-gateway/docs/errors/M002 +``` + +## Why it happened + +The `Authorization` header is there, but the token after `Bearer ` is blank. The usual cause is an env var that didn't expand (`Bearer $MNFST_KEY` with `MNFST_KEY` unset), or a config field saved without a value. + +## How to fix it + +1. Log the value your client sends right before the request to confirm the token is actually empty. +2. Check your env vars or config file. If the key is missing, generate one from the [dashboard](https://app.manifest.build). +3. Set `Authorization: Bearer mnfst_` and retry. + +## Related + +- [M001: Missing Authorization header](/llm-gateway/docs/errors/M001) +- [M003: Invalid key format](/llm-gateway/docs/errors/M003) +- [M005: Key not recognized](/llm-gateway/docs/errors/M005) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M003.mdx b/llm-gateway/docs/errors/M003.mdx new file mode 100644 index 0000000..e938a89 --- /dev/null +++ b/llm-gateway/docs/errors/M003.mdx @@ -0,0 +1,34 @@ +--- +title: "M003: Invalid key format" +sidebarTitle: "M003" +description: "Manifest error M003 fires when the Bearer token doesn't start with mnfst_. Often a provider key (sk-...) pasted in the wrong field. Returns HTTP 401." +icon: "key-round" +keywords: + ["M003", "Manifest M003", "invalid API key format", "mnfst_ prefix", "wrong API key", "OpenAI sk- key", "401 Unauthorized", "Manifest auth error"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M003" +--- + +## What you saw + +```text +[🦚 Manifest M003] That doesn't look right. Manifest keys start with "mnfst_". Grab yours from the dashboard. +See https://manifest.build/llm-gateway/docs/errors/M003 +``` + +## Why it happened + +Manifest keys always start with `mnfst_`. Your token has a different prefix, which almost always means a provider key (OpenAI, Anthropic, etc.) got pasted into the wrong field. + +## How to fix it + +1. Open the [dashboard](https://app.manifest.build) and copy your harness's key. It looks like `mnfst_xxxxxxxxxxxx`. +2. Replace the token in your client's config and retry. +3. Provider keys (OpenAI's `sk-...`, etc.) belong on the **Routing** page, not in the Authorization header. + +## Related + +- [M001: Missing Authorization header](/llm-gateway/docs/errors/M001) +- [M005: Key not recognized](/llm-gateway/docs/errors/M005) +- [M100: Provider API key missing](/llm-gateway/docs/errors/M100) +- [API key providers](/llm-gateway/docs/providers/api-key-providers) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M004.mdx b/llm-gateway/docs/errors/M004.mdx new file mode 100644 index 0000000..7734989 --- /dev/null +++ b/llm-gateway/docs/errors/M004.mdx @@ -0,0 +1,32 @@ +--- +title: "M004: Key expired" +sidebarTitle: "M004" +description: "Manifest error M004 fires when the API key you sent is past its expiration date. Returns HTTP 401. Fix: rotate the key in the dashboard." +icon: "key-round" +keywords: + ["M004", "Manifest M004", "API key expired", "key rotation", "401 Unauthorized", "Manifest auth error", "expired token"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M004" +--- + +## What you saw + +```text +[🦚 Manifest M004] This key has expired. Generate a new one here +See https://manifest.build/llm-gateway/docs/errors/M004 +``` + +## Why it happened + +Manifest keys can carry an expiration date. Yours is past that date, so it no longer authenticates anything. + +## How to fix it + +1. Open the [dashboard](https://app.manifest.build) and rotate or generate a new key for this harness. +2. Update your client's config with the new value. +3. If you didn't intend the key to expire, edit the harness's settings and clear the date. + +## Related + +- [M005: Key not recognized](/llm-gateway/docs/errors/M005) +- [M001: Missing Authorization header](/llm-gateway/docs/errors/M001) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M005.mdx b/llm-gateway/docs/errors/M005.mdx new file mode 100644 index 0000000..36811ef --- /dev/null +++ b/llm-gateway/docs/errors/M005.mdx @@ -0,0 +1,34 @@ +--- +title: "M005: Key not recognized" +sidebarTitle: "M005" +description: "Manifest error M005 fires when the mnfst_ key has the right shape but no matching harness exists. Often a rotated or deleted key. Returns HTTP 401." +icon: "key-round" +keywords: + ["M005", "Manifest M005", "invalid API key", "rotated key", "deleted agent", "deleted harness", "401 Unauthorized", "key not found", "Manifest auth error"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M005" +--- + +## What you saw + +```text +[🦚 Manifest M005] I don't recognize this key. It might have been rotated or deleted. Grab the current one from the dashboard. +See https://manifest.build/llm-gateway/docs/errors/M005 +``` + +## Why it happened + +The token shape is right (`mnfst_...`) but no matching harness exists. Usually that means the key was rotated and your client still has the old value, or the harness itself was deleted, or you copied a key from a different workspace. + +## How to fix it + +1. Open the [dashboard](https://app.manifest.build), find your harness, and copy the current key. +2. Paste it into your client's config. If the harness is gone, create a new one and copy its key. +3. On a self-hosted install, double-check that your client is hitting the right backend URL. + +## Related + +- [M003: Invalid key format](/llm-gateway/docs/errors/M003) +- [M004: Key expired](/llm-gateway/docs/errors/M004) +- [M001: Missing Authorization header](/llm-gateway/docs/errors/M001) +- [Self-hosted setup](/llm-gateway/docs/self-hosted) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M100.mdx b/llm-gateway/docs/errors/M100.mdx new file mode 100644 index 0000000..e299f13 --- /dev/null +++ b/llm-gateway/docs/errors/M100.mdx @@ -0,0 +1,38 @@ +--- +title: "M100: Provider API key missing" +sidebarTitle: "M100" +description: "Manifest error M100 fires when routing picks a provider that has no API key for this harness (OpenAI, Anthropic, Gemini, OpenRouter, etc). Fix: add the key from Providers → Usage-based." +icon: "plug-zap" +keywords: + ["M100", "Manifest M100", "provider API key missing", "no OpenAI key", "no Anthropic key", "no Gemini key", "OpenRouter key", "Manifest routing", "provider credentials"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M100" +--- + +## What you saw + +```text +[🦚 Manifest M100] No anthropic API key yet. Add one here: https://app.manifest.build/... +See https://manifest.build/llm-gateway/docs/errors/M100 +``` + +The provider name (`anthropic` in the example) varies based on which provider Manifest selected. + +## Why it happened + +Routing picked a provider that has no key on file for this harness. Manifest won't silently swap vendors on you, so the request stops here. + +## How to fix it + +1. Open the dashboard link in the error message. It goes straight to the harness's **Routing** page. +2. Open **Providers → Usage-based** in the sidebar and add a key for the named provider. Or remove that provider from your tier configuration so routing won't pick it again. +3. Retry the request. + +## Related + +- [M101: No providers configured](/llm-gateway/docs/errors/M101) +- [M102: Provider subscription credentials unusable](/llm-gateway/docs/errors/M102) +- [M003: Invalid key format](/llm-gateway/docs/errors/M003) +- [Routing](/llm-gateway/docs/llm-gateway) +- [API key providers](/llm-gateway/docs/providers/api-key-providers) +- [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M101.mdx b/llm-gateway/docs/errors/M101.mdx new file mode 100644 index 0000000..98d238e --- /dev/null +++ b/llm-gateway/docs/errors/M101.mdx @@ -0,0 +1,36 @@ +--- +title: "M101: No providers configured" +sidebarTitle: "M101" +description: "Manifest error M101 fires when an authenticated harness has zero providers connected. Common on first run. Fix: connect a provider from the Providers section of the dashboard." +icon: "plug-zap" +keywords: + ["M101", "Manifest M101", "no providers configured", "first-time setup", "empty agent", "empty harness", "Manifest routing", "connect provider", "OpenRouter setup"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M101" +--- + +## What you saw + +```text +[🦚 Manifest M101] You're connected, but no providers are set up yet. Add one here: https://app.manifest.build/... +See https://manifest.build/llm-gateway/docs/errors/M101 +``` + +## Why it happened + +You'll usually see this on a fresh harness. Your key works, but no provider credentials are connected yet (OpenAI, Anthropic, OpenRouter, an Ollama instance, anything). + +## How to fix it + +1. Open the dashboard link in the error message. +2. The page offers one card per connection type. Pick **Usage-based** to paste an API key, or **Subscriptions** to reuse a plan you already pay for. OpenRouter or OpenAI with an API key is the fastest path. +3. Retry the request. + +## Related + +- [M100: Provider API key missing](/llm-gateway/docs/errors/M100) +- [M102: Provider subscription credentials unusable](/llm-gateway/docs/errors/M102) +- [Routing](/llm-gateway/docs/llm-gateway) +- [API key providers](/llm-gateway/docs/providers/api-key-providers) +- [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers) +- [Local models (Ollama)](/llm-gateway/docs/providers/local-models) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M102.mdx b/llm-gateway/docs/errors/M102.mdx new file mode 100644 index 0000000..675af13 --- /dev/null +++ b/llm-gateway/docs/errors/M102.mdx @@ -0,0 +1,58 @@ +--- +title: "M102: Provider subscription credentials unusable" +sidebarTitle: "M102" +description: "Manifest error M102 fires when routing picks a subscription provider whose OAuth or refresh credentials cannot be used. Fix: reconnect the subscription from Providers → Subscriptions." +icon: "plug-zap" +keywords: + [ + "M102", + "Manifest M102", + "subscription credentials unusable", + "OAuth refresh failed", + "ChatGPT OAuth", + "Claude subscription", + "dead refresh token", + "Manifest routing", + "reconnect OAuth", + ] +canonical: "https://manifest.build/llm-gateway/docs/errors/M102" +--- + +## What you saw + +```text +[🦚 Manifest M102] openai subscription credentials could not be refreshed. Reconnect OAuth here: https://app.manifest.build/... +See https://manifest.build/llm-gateway/docs/errors/M102 +``` + +The provider name (`openai` in the example) varies based on which subscription route Manifest selected. + +## Why it happened + +Routing elected a **subscription** connection (ChatGPT OAuth, Claude Max token, Copilot device code, and similar), and Manifest still has a stored credential blob for it — but that credential cannot produce a usable access token. + +Typical causes: + +- The OAuth **refresh token expired**, was revoked, or the user signed out of the provider account +- A pasted subscription token is no longer valid +- Provider-side session invalidation after a password change or security reset + +This is different from [M100](/llm-gateway/docs/errors/M100): M100 means there is no key on file. M102 means a subscription row exists and was elected, but the unwrap/refresh step failed. + +When fallback routes are configured, Manifest records the M102 hop and continues the chain. You only see a terminal M102 response when no usable fallback remains. + +## How to fix it + +1. Open the dashboard link in the error message. It goes straight to the harness's **Routing** page. +2. Open **Providers → Subscriptions** in the sidebar, click **Connect** on the provider, and redo the auth: OAuth, device code, or a fresh token. +3. Optionally add a working [API key](/llm-gateway/docs/providers/api-key-providers) or another [subscription](/llm-gateway/docs/providers/subscription-based-providers) as a fallback so a dead primary does not block traffic. +4. Retry the request. + +## Related + +- [M100: Provider API key missing](/llm-gateway/docs/errors/M100) +- [M101: No providers configured](/llm-gateway/docs/errors/M101) +- [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers) +- [API key providers](/llm-gateway/docs/providers/api-key-providers) +- [Routing](/llm-gateway/docs/llm-gateway) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M200.mdx b/llm-gateway/docs/errors/M200.mdx new file mode 100644 index 0000000..8af7aa1 --- /dev/null +++ b/llm-gateway/docs/errors/M200.mdx @@ -0,0 +1,42 @@ +--- +title: "M200: Usage limit exceeded" +sidebarTitle: "M200" +description: "Manifest error M200 fires when a harness crosses a cost or token cap you configured on the Limits page. It comes back as an HTTP 200 assistant message, not a 429. Fix: raise the threshold or wait for the period to reset." +icon: "gauge" +keywords: + ["M200", "Manifest M200", "usage limit exceeded", "cost limit", "token limit", "spending cap", "budget cap", "monthly budget LLM", "Manifest limits", "HTTP 200 assistant message"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M200" +--- + +## What you saw + +```text +[🦚 Manifest M200] You hit your cost limit: $12.50 used, $10.00/day allowed. Adjust it here: https://app.manifest.build/... +See https://manifest.build/llm-gateway/docs/errors/M200 +``` + +The metric (cost or tokens), used amount, threshold, and period vary based on the rule you tripped. + +The response arrives as HTTP 200 with this text in the assistant message content, not as a 429. A client branching on the HTTP status sees a normal completion. + +## Why it happened + +You set a limit on the harness's [Limits](/llm-gateway/docs/observability#spend-alerts) page (say, "$10/day" or "500k tokens/hour"), and the current period's usage just crossed it. Manifest blocks every following request until the period resets or you raise the cap. + +This is different from [M204](/llm-gateway/docs/errors/M204), which is the Manifest Cloud Free plan monthly request quota, and different from provider 402 errors such as "insufficient credits" from OpenRouter. + +## How to fix it + +1. Open the dashboard link in the error message. It goes straight to the harness's **Limits** page. +2. Raise the threshold, change the period, or wait for it to reset. The error message tells you which period was hit. +3. Retry the request. + +## Related + +- [Spend alerts](/llm-gateway/docs/observability#spend-alerts) +- [Hard limits](/llm-gateway/docs/llm-gateway#hard-limits) +- [M201: Per-user rate limit](/llm-gateway/docs/errors/M201) +- [M202: Per-IP rate limit](/llm-gateway/docs/errors/M202) +- [M203: Concurrency limit](/llm-gateway/docs/errors/M203) +- [M204: Monthly request limit reached](/llm-gateway/docs/errors/M204) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M201.mdx b/llm-gateway/docs/errors/M201.mdx new file mode 100644 index 0000000..ccf9184 --- /dev/null +++ b/llm-gateway/docs/errors/M201.mdx @@ -0,0 +1,36 @@ +--- +title: "M201: Per-user rate limit exceeded" +sidebarTitle: "M201" +description: "Manifest error M201 fires when one user sends more than 200 requests per minute. Returns HTTP 429. Fix: back off and retry." +icon: "timer" +keywords: + ["M201", "Manifest M201", "rate limit", "429 Too Many Requests", "per-user rate limit", "throttling", "exponential backoff", "Manifest limits"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M201" +--- + +## What you saw + +```text +[🦚 Manifest M201] Too many requests — wait a few seconds and retry. +See https://manifest.build/llm-gateway/docs/errors/M201 +``` + +The HTTP status is `429`, which most SDKs treat as retryable and back off from on their own. + +## Why it happened + +Manifest caps each user at 200 requests per minute. The cap is there mostly to keep runaway loops from melting the proxy. Your agent crossed it inside a 60-second window. + +## How to fix it + +1. Add a small delay or exponential backoff on retry. Most SDKs handle this automatically when they see a `429`. +2. If parallel workers are hitting Manifest, throttle them so they share the budget. +3. The 200 requests/minute cap is a fixed guardrail. No env var or setting changes it, self-hosted included. Lean on the backoff in step 1. + +## Related + +- [M202: Per-IP rate limit](/llm-gateway/docs/errors/M202) +- [M203: Concurrency limit](/llm-gateway/docs/errors/M203) +- [M200: Usage limit exceeded](/llm-gateway/docs/errors/M200) +- [Self-hosted setup](/llm-gateway/docs/self-hosted) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M202.mdx b/llm-gateway/docs/errors/M202.mdx new file mode 100644 index 0000000..e5a7e00 --- /dev/null +++ b/llm-gateway/docs/errors/M202.mdx @@ -0,0 +1,34 @@ +--- +title: "M202: Per-IP rate limit exceeded" +sidebarTitle: "M202" +description: "Manifest error M202 fires when one IP sends more than 500 requests per minute across all harnesses. Returns HTTP 429. Fix: back off and stagger workers." +icon: "timer" +keywords: + ["M202", "Manifest M202", "per-IP rate limit", "shared IP", "429 Too Many Requests", "Manifest limits", "abuse protection"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M202" +--- + +## What you saw + +```text +[🦚 Manifest M202] Too many requests from this IP — wait a few seconds and retry. +See https://manifest.build/llm-gateway/docs/errors/M202 +``` + +## Why it happened + +There's a separate per-IP cap (500 requests per minute) sitting on top of the per-user limit. It catches abuse, plus dev setups where lots of agents share one IP. You hit it because that IP fired too many requests across all harnesses in a 60-second window. + +## How to fix it + +1. Back off and retry. The window resets after a minute. +2. Running many agents from one machine in dev? Stagger their startup so they don't all hammer the proxy at once. +3. The 500 requests/minute per-IP cap is a fixed guardrail. No env var or setting changes it, self-hosted included. Stagger workers as in step 2. + +## Related + +- [M201: Per-user rate limit](/llm-gateway/docs/errors/M201) +- [M203: Concurrency limit](/llm-gateway/docs/errors/M203) +- [M200: Usage limit exceeded](/llm-gateway/docs/errors/M200) +- [Self-hosted setup](/llm-gateway/docs/self-hosted) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M203.mdx b/llm-gateway/docs/errors/M203.mdx new file mode 100644 index 0000000..08aced8 --- /dev/null +++ b/llm-gateway/docs/errors/M203.mdx @@ -0,0 +1,34 @@ +--- +title: "M203: Concurrency limit exceeded" +sidebarTitle: "M203" +description: "Manifest error M203 fires when a workspace has more than 10 in-flight requests. Returns HTTP 429. Common with parallel streaming. Fix: reduce parallelism." +icon: "timer" +keywords: + ["M203", "Manifest M203", "concurrency limit", "in-flight requests", "429 Too Many Requests", "parallel streams", "Manifest limits"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M203" +--- + +## What you saw + +```text +[🦚 Manifest M203] Too many concurrent requests. Give it a moment. +See https://manifest.build/llm-gateway/docs/errors/M203 +``` + +## Why it happened + +Manifest caps in-flight requests at 10 per workspace, shared by all its harnesses. Streaming completions hold a slot until the stream ends, so an agent that fires off parallel streams can hit this fast. + +## How to fix it + +1. Let in-flight requests finish, then retry. Most SDKs handle this automatically with backoff on `429`. +2. Reduce parallelism on the client side. Most agents only need one or two concurrent calls. +3. The 10 concurrent-request cap is a fixed guardrail. No env var or setting changes it, self-hosted included. Reduce parallelism as in step 2. + +## Related + +- [M201: Per-user rate limit](/llm-gateway/docs/errors/M201) +- [M202: Per-IP rate limit](/llm-gateway/docs/errors/M202) +- [M200: Usage limit exceeded](/llm-gateway/docs/errors/M200) +- [Self-hosted setup](/llm-gateway/docs/self-hosted) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M204.mdx b/llm-gateway/docs/errors/M204.mdx new file mode 100644 index 0000000..78e2b1e --- /dev/null +++ b/llm-gateway/docs/errors/M204.mdx @@ -0,0 +1,34 @@ +--- +title: "M204: Monthly request limit reached" +sidebarTitle: "M204" +description: "Manifest error M204 fires when a Manifest Cloud Free workspace uses all monthly requests. Returns HTTP 402 with PLAN_LIMIT_REQUESTS. Fix: upgrade to Pro or wait for the monthly reset." +icon: "badge-alert" +keywords: + ["M204", "Manifest M204", "PLAN_LIMIT_REQUESTS", "monthly request limit", "free plan request limit", "Manifest billing", "402 Payment Required", "upgrade to Pro"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M204" +--- + +## What you saw + +```text +[🦚 Manifest M204] You've used all 10000 requests included this month on the Free plan. Upgrade to Pro for unlimited requests: https://app.manifest.build/upgrade?reason=requests +See https://manifest.build/llm-gateway/docs/errors/M204 +``` + +SDK and tool callers receive an HTTP 402 JSON error with `error.code` set to `PLAN_LIMIT_REQUESTS`. + +## Why it happened + +Your Manifest Cloud workspace reached the Free plan monthly request quota. This is a Manifest plan restriction, not a provider error and not a limit you configured on a harness. + +## How to fix it + +1. Upgrade to Pro from the dashboard link in the error message. +2. Or wait for the monthly request quota to reset. +3. Retry the request after the plan changes or the quota resets. + +## Related + +- [Upgrade dashboard](https://app.manifest.build/upgrade?reason=requests) +- [M200: Usage limit exceeded](/llm-gateway/docs/errors/M200) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M300.mdx b/llm-gateway/docs/errors/M300.mdx new file mode 100644 index 0000000..b5705a3 --- /dev/null +++ b/llm-gateway/docs/errors/M300.mdx @@ -0,0 +1,41 @@ +--- +title: "M300: Missing messages array" +sidebarTitle: "M300" +description: "Manifest error M300 fires when the /v1/chat/completions body has no messages array (missing, not an array, or empty). Returns HTTP 400." +icon: "message-square-warning" +keywords: + ["M300", "Manifest M300", "messages array required", "empty messages", "400 Bad Request", "OpenAI compatible API", "chat completions validation", "Manifest validation"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M300" +--- + +## What you saw + +```text +[🦚 Manifest M300] `messages` array is required. +See https://manifest.build/llm-gateway/docs/errors/M300 +``` + +## Why it happened + +The OpenAI-compatible chat completions API requires a `messages` array with at least one entry. Manifest got a body where `messages` was missing, not an array, or empty. + +## How to fix it + +Send a body shaped like this: + +```json +{ + "model": "auto", + "messages": [ + { "role": "user", "content": "Hello" } + ] +} +``` + +The OpenAI SDK handles this for you. If you're hand-rolling the request, eyeball the JSON before posting. + +## Related + +- [M303: Local provider unavailable on Manifest Cloud](/llm-gateway/docs/errors/M303) +- [API reference](/llm-gateway/docs/reference/api) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M302.mdx b/llm-gateway/docs/errors/M302.mdx new file mode 100644 index 0000000..9e3ac97 --- /dev/null +++ b/llm-gateway/docs/errors/M302.mdx @@ -0,0 +1,44 @@ +--- +title: "M302: Model not available" +sidebarTitle: "M302" +description: "Manifest error M302 fires when an explicit model ID is not available for the authenticated harness. Fix: use GET /v1/models or make the provider available for this harness." +icon: "message-square-warning" +keywords: + ["M302", "Manifest M302", "model not available", "model not found", "GET /v1/models", "direct model routing", "provider not connected", "provider not enabled", "agent-visible models", "harness-visible models", "Manifest validation"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M302" +--- + +## What you saw + +```text +[🦚 Manifest M302] Model "openai/gpt-4o" is not available for this agent. Use GET /v1/models to list available model IDs, or make the provider available for this agent here: https://app.manifest.build/... +See https://manifest.build/llm-gateway/docs/errors/M302 +``` + +The model name varies based on the `model` value your client sent. + +## Why it happened + +You sent a concrete model ID instead of `auto` to an OpenAI-format endpoint, but that model is not available to this harness. + +This can happen when: + +1. The model ID is misspelled or retired. +2. The provider is not connected yet, or it is connected but not enabled for this harness. +3. You used a provider-native name when Manifest expects the exact ID returned by `GET /v1/models`. +4. Manifest refuses the request instead of falling back to automatic routing. + +## How to fix it + +1. Call `GET /v1/models` with the same Manifest harness key. +2. Use `auto` to let Manifest route, or copy one of the listed model IDs exactly. +3. If the model should be available, open the dashboard link in the error and enable or connect that provider for this harness. +4. Retry the request. + +## Related + +- [Routing: Route a specific model](/llm-gateway/docs/llm-gateway#route-a-specific-model) +- [API reference: Listing models](/llm-gateway/docs/reference/api#listing-models) +- [M300: Missing messages array](/llm-gateway/docs/errors/M300) +- [M303: Local provider unavailable on Manifest Cloud](/llm-gateway/docs/errors/M303) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M303.mdx b/llm-gateway/docs/errors/M303.mdx new file mode 100644 index 0000000..b1cf4df --- /dev/null +++ b/llm-gateway/docs/errors/M303.mdx @@ -0,0 +1,51 @@ +--- +title: "M303: Local provider unavailable on Manifest Cloud" +sidebarTitle: "M303" +description: "Manifest error M303 fires when a Manifest Cloud request routes to Ollama, LM Studio, or llama.cpp. Returns HTTP 400. Fix: self-host, or expose the runtime as a custom provider." +icon: "server-off" +keywords: + ["M303", "Manifest M303", "local provider cloud", "Ollama Manifest Cloud", "LM Studio", "llama.cpp", "400 Bad Request", "custom provider", "tunnel"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M303" +--- + +## What you saw + +```text +[🦚 Manifest M303] Built-in local providers are only available in self-hosted Manifest. On Manifest Cloud, expose the runtime through a public URL or tunnel and connect it as a custom provider. +See https://manifest.build/llm-gateway/docs/errors/M303 +``` + +## Why it happened + +The built-in [local provider](/llm-gateway/docs/providers/local-models) tiles (Ollama, LM Studio, llama.cpp) route to `localhost`. On a self-hosted install that's your machine, which is the whole point. On Manifest Cloud, `localhost` is our server, which has no model running on it and no route back to your laptop. + +Rather than dial an address that can't be yours, Manifest stops the request here. + +## How to fix it + +Pick whichever fits: + + + + Local providers work as designed on a self-hosted install, and your prompts + never leave your network. See [Self-hosted](/llm-gateway/docs/self-hosted) for the one-line + Docker install. + + + Keep using Cloud and give your runtime a public address. A tunnel + (Cloudflare Tunnel, ngrok, Tailscale Funnel) is the usual way. Then add it + under [Custom providers](/llm-gateway/docs/providers/custom-providers) using the public URL + instead of the built-in tile. + + Put authentication in front of it. A public URL with no key is an open + inference endpoint that anyone can bill to your hardware. + + + +## Related + +- [Local models](/llm-gateway/docs/providers/local-models) +- [Custom providers](/llm-gateway/docs/providers/custom-providers) +- [Self-hosted](/llm-gateway/docs/self-hosted) +- [M302: Model not available](/llm-gateway/docs/errors/M302) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M500.mdx b/llm-gateway/docs/errors/M500.mdx new file mode 100644 index 0000000..9a42763 --- /dev/null +++ b/llm-gateway/docs/errors/M500.mdx @@ -0,0 +1,32 @@ +--- +title: "M500: Internal server error" +sidebarTitle: "M500" +description: "Manifest error M500 is a generic 500 fallback for unhandled errors in the proxy. Usually transient. Fix: retry, then check backend logs on self-hosted." +icon: "server-crash" +keywords: + ["M500", "Manifest M500", "internal server error", "500 Internal Server Error", "5xx", "proxy crash", "Manifest server error", "self-hosted debugging"] +canonical: "https://manifest.build/llm-gateway/docs/errors/M500" +--- + +## What you saw + +```text +[🦚 Manifest M500] Something broke on our end. Try again in a moment. +See https://manifest.build/llm-gateway/docs/errors/M500 +``` + +## Why it happened + +Manifest hit an unexpected error while handling your request. Usually that's a database hiccup, a downstream provider returning something unparseable, or a real bug. The actual stack trace is logged server-side, but masked in the response so we don't leak internals. + +## How to fix it + +1. Retry. Most M500s are transient. +2. On a self-hosted install, check the backend logs for the real stack trace. +3. If it sticks around, [open an issue](https://github.com/mnfst/manifest/issues) with the timestamp and your harness name. That's enough for us to correlate it to the server log. + +## Related + +- [Self-hosted setup](/llm-gateway/docs/self-hosted) +- [Fallback](/llm-gateway/docs/llm-gateway#fallback) +- [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/integrations/n8n.mdx b/llm-gateway/docs/integrations/n8n.mdx new file mode 100644 index 0000000..21148f0 --- /dev/null +++ b/llm-gateway/docs/integrations/n8n.mdx @@ -0,0 +1,127 @@ +--- +title: "n8n" +description: "Use Manifest as the chat model behind n8n AI Agents, or call it directly from a workflow, with the n8n-nodes-manifest community node." +icon: "workflow" +keywords: + ["n8n", "n8n community node", "n8n-nodes-manifest", "n8n AI Agent", "Manifest Chat Model", "n8n self-hosted", "n8n LLM router", "n8n OpenAI alternative"] +canonical: "https://manifest.build/llm-gateway/docs/integrations/n8n" +--- + +
+ + +
+ +Manifest ships as a verified n8n community node, [`n8n-nodes-manifest`](https://www.npmjs.com/package/n8n-nodes-manifest). It installs from n8n's own **Community Nodes** screen on n8n Cloud and on self-hosted n8n alike, and gives you two nodes: + + + + A language-model sub-node. Plug it into the **AI Agent** or **Basic LLM Chain** node in place of the OpenAI Chat Model, and Manifest routes every request the agent makes. + + + An action node for calling Manifest directly from a workflow: list models, create a chat completion, or create a Responses API call. + + + +Both nodes are plain HTTP clients for the [gateway](/llm-gateway/docs/llm-gateway). Nothing about routing, limits, or [Autofix](/llm-gateway/docs/autofix) changes because the caller is n8n. + +## Install the node + + + + In n8n, go to **Settings** > **Community Nodes** and select **Install**. + + + Enter `n8n-nodes-manifest`, confirm the community-node notice, and install. + + + Search the node panel for **Manifest**. The Chat Model sub-node appears where n8n lists language models, and the action node appears with the regular nodes. + + + +The Chat Model sub-node needs an n8n version that ships `@n8n/ai-node-sdk` (2026 releases). The action node works on older versions. + +## Create the credential + +Both nodes share one **Manifest API** credential with two fields. + +
+ +| Field | Value | +| ------------ | --------------------------------------------------------------------- | +| **Base URL** | `https://app.manifest.build` | +| **API Key** | The harness key, starting with `mnfst_`, from the harness **Settings** page | + +
+ +
+ +| Field | Value | +| ------------ | --------------------------------------------------------------------- | +| **Base URL** | The origin of your instance, for example `http://192.168.1.20:2099` | +| **API Key** | The harness key, starting with `mnfst_`, from the harness **Settings** page | + +
+ +The Base URL is the origin only, without `/v1`. The nodes append `/v1/models`, `/v1/chat/completions`, and `/v1/responses` themselves. A Base URL ending in `/v1` sends requests to `/v1/v1/...` and every call fails. + +Select **Test** on the credential. It calls `GET /v1/models`, so a passing test confirms that n8n can reach your instance and that the key is valid. + +
+ +### Reaching a self-hosted instance from n8n + +The credential test fails with a connection error when n8n cannot reach Manifest. Where the two run decides the address: + +- **n8n and Manifest in Docker on the same host.** `localhost` inside the n8n container is n8n itself. Put both containers on a shared Docker network and use the Manifest service name as the host, or use `http://host.docker.internal:2099` on Docker Desktop. +- **n8n on another machine.** The bundled compose file binds Manifest to `127.0.0.1` only. Set `HOST_BIND_ADDRESS=0.0.0.0` in Manifest's `.env`, as described in [Exposing on the LAN](/llm-gateway/docs/self-hosted#exposing-on-the-lan), so the port answers on the network. +- **Plain `http://` is fine.** The calls are server to server. The dashboard's browser security rules do not apply to them. + +The keyless loopback shortcut that a development build of Manifest accepts does not exist in the production image. n8n always needs a real `mnfst_` key. + +
+ +## Use Manifest behind an AI Agent + + + + On an **AI Agent** or **Basic LLM Chain** node, select the **Model** connector and choose **Manifest Chat Model**. + + + Select the Manifest credential. The model list is loaded from your instance. Leave it on `auto` to let your [routing rules](/llm-gateway/docs/llm-gateway#routing) pick the model, or choose a model ID to pin one. + + + Streaming and tool calling work as they do with the OpenAI Chat Model, so agent tools run unchanged. Each call appears on the harness's **Requests** page with the model that served it and its cost. + + + +An option on the sub-node switches it from Chat Completions to the [Responses API](/llm-gateway/docs/reference/api#endpoints). Leave it off unless a model you route to needs Responses. + +## Call Manifest from a workflow + +The **Manifest** action node has three operations: + +| Operation | Gateway route | +| -------------------------- | --------------------------- | +| **List Models** | `GET /v1/models` | +| **Create Chat Completion** | `POST /v1/chat/completions` | +| **Create Response** | `POST /v1/responses` | + +For a first test, choose **Create Chat Completion**, set **Model** to `auto`, and use these messages: + +```json +[ + { "role": "user", "content": "Reply with one sentence confirming that Manifest is connected." } +] +``` + +Your routing decides whether the route answers with buffered JSON or a stream. The node handles both: a buffered response is returned as the API's JSON object, and a streamed response is collected until it finishes and returned as its parsed server-sent events. Do not set `stream` in **Additional Body**. The node ignores that field. + +## Troubleshooting + +| Symptom | Cause | +| -------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| Credential test fails with a connection error | n8n cannot reach the Base URL. See [Reaching a self-hosted instance](#reaching-a-self-hosted-instance-from-n8n). | +| Credential test fails with `401` | The key is not a valid `mnfst_` harness key. Copy it again from the harness **Settings** page. | +| Every call returns `404` | The Base URL ends in `/v1`. Remove it. | +| The model list shows only `auto`, or calls fail with [M100](/llm-gateway/docs/errors/M100) or [M101](/llm-gateway/docs/errors/M101) | No provider is connected to the harness yet. Connect one on the harness's **Providers** tab. | diff --git a/llm-gateway/docs/introduction.mdx b/llm-gateway/docs/introduction.mdx new file mode 100644 index 0000000..216e522 --- /dev/null +++ b/llm-gateway/docs/introduction.mdx @@ -0,0 +1,84 @@ +--- +title: "Manifest documentation" +sidebarTitle: "Introduction" +description: "Open-source LLM router for agents. Connect your subscriptions, API keys, and local models to any harness through one endpoint, with fallbacks and spend limits." +icon: "house" +keywords: + ["LLM router", "AI subscriptions", "model routing", "open source", "LLM gateway", "quickstart", "first request", "create a harness", "gateway base URL"] +canonical: "https://manifest.build/llm-gateway/docs/introduction" +--- + +
+ + +
+ +Manifest is an open-source LLM router for agents and AI apps. Connect the providers you already use — subscriptions like ChatGPT or Claude, pay-per-token API keys, local models, and custom endpoints — and route every request through one OpenAI- and Anthropic-compatible URL. + +Send `auto` as the model and Manifest routes each request by the rules you set, with fallbacks when one fails. Pin a real model ID instead, and the request goes straight to that model, with no routing and no fallback. Either way, you cap what each harness spends and see the cost of every request. + +## Key features + + + + Connect API keys, subscriptions you already pay for, local models, or any custom endpoint. + + + Send `auto` and your routing rules pick the model, with fallbacks when one fails. Limits cap what a harness can spend. + + + Repair a malformed request and resend it, instead of failing on something a retry can't fix. + + + See what every harness spent, which models served it, and what failed. + + + Read back the full request and response body of any provider attempt, as a conversation. + + + +## Cloud or self-hosted? + +Manifest is open source and runs two ways. Both speak the same API, so pointing a client at one or the other is a URL change. + + + + Runs on our servers. Sign up, connect a provider, and you're routing. + Recommended if you're starting out. + + + Runs on infrastructure you control: Docker on your own machine, a managed + platform like Railway, your AWS or GCP account, or a server panel. + + + +On the cloud there is nothing to install and nothing to configure on a server. You sign up, connect your providers, point your harness at the gateway URL, and everything else — routing, limits, [Autofix](/llm-gateway/docs/autofix), alerts — is a setting in the dashboard. The [environment variables](/llm-gateway/docs/reference/environment-variables) throughout these docs apply only to a Manifest instance you run yourself. + +Cloud is also subject to [plan limits](/llm-gateway/docs/errors/M204): the free plan includes 10,000 routed requests per calendar month, shown by the usage meter in the dashboard sidebar and reset at 00:00 UTC on the 1st. And it can't reach [local models](/llm-gateway/docs/providers/local-models) running on your machine. A self-hosted instance has neither restriction. + +## Your first request + +
+ +Sign up at [app.manifest.build](https://app.manifest.build) and a guided setup walks you through it. Step one creates a harness. Then connect a provider, and give your default tier a model to route to. The last step shows the gateway address and the harness key, which starts with `mnfst_`. A harness you create later reuses the providers your workspace already has. + + + The Manifest guided setup on step one, Create your first harness, with the five setup steps listed in the sidebar + + +```bash +curl -X POST https://app.manifest.build/v1/chat/completions \ + -H "Authorization: Bearer mnfst_YOUR_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "auto", "messages": [{"role": "user", "content": "Hello"}]}' +``` + +`auto` asks Manifest to route the request. The request appears on the Requests page with the model that served it and its cost. + +
+ +
+ +Install Manifest first: [Self-hosted](/llm-gateway/docs/self-hosted) covers Docker and the managed platforms. Your instance starts empty. [First request](/llm-gateway/docs/self-hosted#first-request) takes you from there to a routed request, against `http://localhost:2099/v1`. + +
diff --git a/llm-gateway/docs/llm-gateway.mdx b/llm-gateway/docs/llm-gateway.mdx new file mode 100644 index 0000000..6fa8c30 --- /dev/null +++ b/llm-gateway/docs/llm-gateway.mdx @@ -0,0 +1,167 @@ +--- +title: "LLM Gateway" +description: "Every request passes through the gateway: limits check it, routing picks the model, and fallback catches it when that model fails." +icon: "split" +keywords: + ["LLM gateway", "model routing", "LLM router", "fallback chain", "hard limit", "custom tier", "default model", "fallback exhausted", "429 too many requests"] +canonical: "https://manifest.build/llm-gateway/docs/llm-gateway" +--- + +Point your client at Manifest instead of a provider, and every request goes through the gateway. Three things happen there: limits decide whether the request runs at all, routing picks the model, and fallback steps in when that model fails. + +## How a request flows + + + + The `Authorization: Bearer mnfst_...` header resolves to one of your harnesses. A bad or missing key stops here with a [401](/llm-gateway/docs/errors). + + + If a hard limit for this harness is already over its threshold, the request is blocked with [M200](/llm-gateway/docs/errors/M200) before any provider is called. Nothing is spent. + + + A custom tier matches on a request header, or the request goes to your default tier. Send a real model ID instead of `auto` to skip routing. A matching custom tier header still wins. + + + Manifest calls the resolved provider with your credentials and streams the response back. + + + If the provider returns an error, [Autofix](/llm-gateway/docs/autofix) may repair and resend the request once. Anything still failing moves down the fallback chain. + + + +Routing runs in-process. There's no extra network call and no added latency. + +## Routing + +Instead of hard-coding one model into every client, you point your client at Manifest and let it pick. Send `auto` as the model, and Manifest resolves the real model based on the rules you set on the dashboard **Routing** page. + + + + One model plus up to 5 fallbacks. Every request lands here unless a custom tier matches. + + + Match a request header to a tier you define, and route it to its own model. + + + +### Default tier + +Every harness has a default tier: one model plus up to five fallbacks. You set it on the **Routing** page and can change it anytime without touching your code. Send `auto` as the model, and any request that doesn't match a custom tier goes to your default. + +### Custom tiers + +Custom tiers route by request header. You create a tier on the dashboard, give it a header key and value, and pin it to a model with its own fallbacks. When an incoming request carries that header, Manifest sends it to that tier's model instead of the default. The header wins even when the body names an explicit model ID. + +The header key is yours to choose (lowercase letters, numbers, and hyphens). A few names are reserved and rejected, including `authorization`, `cookie`, and `x-api-key`. Send the header from your client like any other: + +```ts +import OpenAI from "openai"; + +const client = new OpenAI({ + baseURL: "https://app.manifest.build/v1", + apiKey: "mnfst_YOUR_KEY", + defaultHeaders: { "x-manifest-tier": "batch" }, // your tier's key and value +}); + +const response = await client.chat.completions.create({ + model: "auto", + messages: [{ role: "user", content: "Summarize this report." }], +}); +``` + +Create as many tiers as you need, each with its own model and parameters. This is handy for isolating an agent's subtasks, A/B testing two models, or letting an orchestration layer decide the tier itself. + +### Route a specific model + +To skip routing for a single request, send a real model ID instead of `auto`. Manifest forwards it straight to that model's provider, with no tier lookup and no fallbacks. A matching [custom tier](#custom-tiers) header is the exception. The request then runs on the tier's model and the tier's fallbacks. Call [`GET /v1/models`](/llm-gateway/docs/reference/api#listing-models) to list the model IDs your harness can reach. + +```bash +curl -X POST https://app.manifest.build/v1/chat/completions \ + -H "Authorization: Bearer mnfst_YOUR_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "openai/gpt-4o", + "messages": [{"role": "user", "content": "Hello"}] + }' +``` + +The response comes back with `X-Manifest-Tier: direct`, so you can tell a direct call from a routed one. A direct model ID works on all three endpoints: `/v1/chat/completions`, `/v1/responses`, and the Anthropic `/v1/messages`. + +The model list is scoped to the harness key. A model may be missing because its provider is not connected yet, or because the provider exists in your workspace but is not enabled for this harness. If you send a model ID that has no connected provider route, Manifest returns [M302: Model not available](/llm-gateway/docs/errors/M302). Send `auto` to use routing. + +## Fallback + +When a model fails (provider outage, rate limit, bad request), Manifest retries with a backup model from the same tier. Your agent gets a response instead of an error. + +### What triggers a fallback + +Any HTTP status code **>= 400** triggers a fallback. + +| Status | Example | +|--------|---------| +| **400** | Bad request | +| **401** | Authentication error | +| **403** | Forbidden | +| **429** | Rate limited | +| **500** | Internal server error | +| **502** | Bad gateway | +| **503** | Service unavailable | +| **529** | Provider overloaded | + +A tier also falls back when its pinned model leaves your model list. When the tier has no fallback left, the request returns [M101](/llm-gateway/docs/errors/M101). + +### Configuration + +Fallback models are configured **per tier**. Each tier holds up to **5 fallback models**, tried in order. + + + + Navigate to **Routing** in the dashboard. + + + Click your Default tier or any custom tier. + + + Add up to 5 fallback models. Drag to reorder, since models are tried from top to bottom. + + + + + Connect at least two providers. With a single provider, fallback can only switch between that provider's models. + + +### Hung providers and the per-attempt timeout + +A provider that opens a connection but never returns eventually triggers a fallback via Manifest's per-attempt timeout (default 180 seconds), which surfaces as a synthetic `504 Gateway Timeout` and moves to the next model in the chain. + +If your upstream client has its own timeout that fires at the same time, the client may disconnect first and Manifest will give up before reaching a healthy fallback. On self-hosted installs, lower [`PROVIDER_TIMEOUT_MS`](/llm-gateway/docs/reference/environment-variables) strictly below your client's timeout so the chain has room to run inside the client's window. + +### Response headers + +When a fallback succeeds, the response carries `X-Manifest-Fallback-From` (the primary that failed) and `X-Manifest-Fallback-Index` (its position in the chain) on top of the standard routing headers. When the chain is exhausted, `X-Manifest-Fallback-Exhausted: true` is set, the body carries the provider's own error code (or `fallback_exhausted` when there is none), and the response keeps the primary model's real error status (for example 500 or 529). Full table: [Headers reference](/llm-gateway/docs/reference/headers). + +## Hard limits + +A limit rule can block requests as well as email you. When a rule's action includes blocking and the harness is over its threshold for the current period, the gateway returns [M200](/llm-gateway/docs/errors/M200) as an HTTP `200` response whose assistant message carries the block text, so your agent keeps flowing instead of throwing: + +```text +[🦚 Manifest M200] You hit your cost limit: $12.50 used, $10.00/day allowed. Adjust it here: https://app.manifest.build/... +``` + +The check runs before any provider is contacted, so a blocked request costs nothing. The block clears on its own at the start of the next period, or immediately if you raise the threshold. + + + On Manifest Cloud, running out of the Free plan's monthly requests is a + different thing entirely: that returns [M204](/llm-gateway/docs/errors/M204) with HTTP `402` + and isn't something you configured. + + +Rules are defined per harness, with a metric, a threshold, and a period. See [Spend alerts](/llm-gateway/docs/observability#spend-alerts) for how to create one and how the alerting half works. + +## Related + +- [Autofix](/llm-gateway/docs/autofix) — repair a failing request before fallback runs +- [Observability](/llm-gateway/docs/observability) — what the gateway records, and spend alerts +- [API reference](/llm-gateway/docs/reference/api) +- [Headers reference](/llm-gateway/docs/reference/headers) +- [Error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/observability.mdx b/llm-gateway/docs/observability.mdx new file mode 100644 index 0000000..f2ceac7 --- /dev/null +++ b/llm-gateway/docs/observability.mdx @@ -0,0 +1,145 @@ +--- +title: "Observability" +description: "See what every harness spent, which models served it, and what failed. Set email alerts when spending crosses a threshold." +icon: "activity" +keywords: + ["LLM observability", "AI cost tracking", "token usage", "spend alerts", "request log", "LLM analytics", "cost per agent", "cost per harness", "provider errors"] +canonical: "https://manifest.build/llm-gateway/docs/observability" +--- + +
+ + +
+ +Every request through the gateway is recorded: which model served it, what it cost, how long it took, and what failed. The dashboard is built on that record, and you can query the same numbers from your own code. + +## Requests and provider attempts + +Manifest tracks two things, and the difference matters once fallback is involved: + +| | What it is | When you get more than one | +|---|---|---| +| **Request** | One call from your agent to Manifest | Never. One call in, one Request. | +| **Provider attempt** | One call from Manifest to a provider | Every fallback step and every [Autofix](/llm-gateway/docs/autofix) retry adds another | + +A request that succeeds on the first try is one Request with one attempt. A request that fails twice before succeeding is still one Request, with three attempts. A request Manifest blocked itself, like a [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) or a malformed body, is one Request with one attempt that never reached a provider. That attempt is listed under the model you asked for, or **No provider** when the request named none, and carries the error that stopped it. + +This is why the request count and the provider call count don't match, and why cost is attributed to attempts: a failed attempt can still burn tokens. + +## The dashboard + + + + Spend, tokens, and request volume over time, across all harnesses or one. Includes the provider and model mix. + + + The full log, newest first. Filter it, then open any row for the request body, every provider attempt, and the routing decision. + + + Per-provider usage, spend, and error rates. Useful for spotting one provider degrading. + + + Threshold rules per harness, plus email provider setup. + + + +
+ +On the Free plan, the Overview covers up to 7 days of history; the 30-, 90-, and 365-day ranges need Pro. + +
+ +### What's on each request + +Opening a request in the log shows the whole story: the tier that routed it, the model and provider that served it, input and output tokens, computed cost, latency, the auth type used ([API key, subscription, or local](/llm-gateway/docs/reference/glossary#auth-type)), and every fallback hop with its own status. Failures carry an error code, so a Manifest rejection like [M100](/llm-gateway/docs/errors/M100) reads differently from a provider's own 500. + +That's all metadata. To read the messages themselves, turn on [request logs](/llm-gateway/docs/request-logs) for the harness. Open a request, pick an attempt, and its **Messages** tab shows the body sent to the provider and the body that came back. + + + Local models record `cost = 0` with real token counts and latency, since + nothing was billed. Custom providers do the same, because Manifest can't + infer pricing for a model it doesn't know. + + +## Spend alerts + +A limit rule watches one metric for one harness over a rolling period, and acts when the threshold is crossed. + +| Field | Options | +|---|---| +| **Metric** | Tokens or cost | +| **Threshold** | The number to watch for | +| **Period** | Hour, day, week, or month | +| **Action** | Every rule emails you. Turn on blocking to also stop requests over the threshold | + +Blocking is the gateway half of this and is covered in [hard limits](/llm-gateway/docs/llm-gateway#hard-limits). The rest of this section is the alerting half. + + + + Navigate to your harness's **Limits** page in the dashboard. + + + Click **+ Create rule**. + + + Pick the metric, threshold, and period. Every rule emails you; turn on **Block requests when exceeded** to also stop requests over the threshold. + + + The rule takes effect immediately. + + + +Alerts are evaluated hourly, so an email can lag the threshold by up to an hour. Blocking is not: it's checked on every request. Each rule emails once per period, so a rule that stays over its threshold won't flood your inbox. + +### Email delivery + +
+ +Alerts go to your account email. Check it's current in your profile settings. + +
+ +
+ +On the **Limits** page, under **Configure email provider**, pick Resend, Mailgun, or SendGrid. Enter the API key. Mailgun also asks for a sending domain. **Notification email** defaults to your account email; change it to send alerts elsewhere. **Test & Connect** sends a real test email and saves nothing if it fails. The saved provider appears as a card on the same page, where you can edit or remove it. + +Your workspace holds one provider, shared by every harness. Removing it stops alerts for all of them. + +Alerts use the saved provider first, and fall back to the [`EMAIL_*` environment variables](/llm-gateway/docs/reference/environment-variables) when no provider is saved. Those variables also send verification and password-reset email, which the saved provider never handles. + +Without an email provider from either path, alerts are skipped. Blocking rules still work, since they don't need to send anything. + +
+ +## Reading usage from your own code + +A harness can query its own numbers with its `mnfst_` key, which is useful for showing a budget in your own UI or stopping work before a limit blocks you. + +```bash +curl https://app.manifest.build/api/v1/agent/usage \ + -H "Authorization: Bearer mnfst_YOUR_KEY" +``` + +| Endpoint | Returns | +|---|---| +| `GET /api/v1/agent/usage` | Token usage for the calling harness | +| `GET /api/v1/agent/costs` | Cost data for the calling harness | + +
+ +Self-hosted installs swap the base URL for their own host. + +## Anonymous telemetry + +Separately from your dashboard, each self-hosted install sends a small daily report so the project can see version adoption and which providers deserve attention. It's aggregates only, never prompts or content, and one environment variable turns it off. Full field list and opt-out: [Data and telemetry](/llm-gateway/docs/reference/telemetry). + +
+ +## Related + +- [LLM Gateway](/llm-gateway/docs/llm-gateway) — routing, fallback, and hard limits +- [Request logs](/llm-gateway/docs/request-logs) — the request and response bodies +- [Autofix](/llm-gateway/docs/autofix) +- [Error codes](/llm-gateway/docs/errors) +- [Glossary](/llm-gateway/docs/reference/glossary) diff --git a/llm-gateway/docs/providers/api-key-providers.mdx b/llm-gateway/docs/providers/api-key-providers.mdx new file mode 100644 index 0000000..165f081 --- /dev/null +++ b/llm-gateway/docs/providers/api-key-providers.mdx @@ -0,0 +1,108 @@ +--- +title: "API key providers" +sidebarTitle: "API keys" +description: "Bring your own API key for OpenAI, Anthropic, Google, xAI, DeepSeek, Mistral, Groq, Cerebras, AWS Bedrock, NVIDIA NIM, OpenRouter, Hugging Face, and more." +icon: "key" +keywords: + ["OpenAI", "Anthropic", "Google Gemini", "xAI", "DeepSeek", "Mistral", "Qwen", "Moonshot", "MiniMax", "Z.ai", "OpenRouter", "Groq", "Cerebras", "Fireworks", "AWS Bedrock", "NVIDIA NIM", "Xiaomi MiMo", "Hugging Face", "Gemini Free", "API key"] +canonical: "https://manifest.build/llm-gateway/docs/providers/api-key-providers" +--- + +Most providers work the standard way: sign up, generate an API key, paste it into Manifest. Routed requests go out with that key as the credential. + +## Supported providers + +| Provider | Where to get a key | +| ------------------------------------------------- | ---------------------------------------------------------------- | +| [OpenAI](https://platform.openai.com) | [platform.openai.com](https://platform.openai.com) | +| [Anthropic](https://www.anthropic.com) | [console.anthropic.com](https://console.anthropic.com) | +| [Google](https://ai.google.dev) | [ai.google.dev](https://ai.google.dev) | +| [xAI](https://x.ai) | [x.ai/api](https://x.ai/api) | +| [Meta](https://dev.meta.ai) | [dev.meta.ai](https://dev.meta.ai) | +| [DeepSeek](https://www.deepseek.com) | [platform.deepseek.com](https://platform.deepseek.com) | +| [Mistral](https://mistral.ai) | [console.mistral.ai](https://console.mistral.ai) | +| [Alibaba Cloud (Qwen)](https://qwen.ai) | [bailian.console.aliyun.com](https://bailian.console.aliyun.com) | +| [Moonshot (Kimi)](https://platform.moonshot.cn) | [platform.moonshot.cn](https://platform.moonshot.cn) | +| [MiniMax](https://www.minimaxi.com) | [platform.minimaxi.com](https://platform.minimaxi.com) | +| [Z.ai (Zhipu)](https://z.ai) | [bigmodel.cn](https://bigmodel.cn) | +| [OpenRouter](https://openrouter.ai) | [openrouter.ai/keys](https://openrouter.ai/keys) | +| [Groq](https://groq.com) | [console.groq.com](https://console.groq.com) | +| [Cerebras](https://cerebras.ai) | [cloud.cerebras.ai](https://cloud.cerebras.ai) | +| [Fireworks AI](https://fireworks.ai) | [fireworks.ai](https://fireworks.ai) | +| [NVIDIA NIM](https://build.nvidia.com) | [build.nvidia.com](https://build.nvidia.com) | +| [AWS Bedrock](https://aws.amazon.com/bedrock) | [aws.amazon.com/bedrock](https://aws.amazon.com/bedrock) | +| [OpenCode Zen](https://opencode.ai) | [opencode.ai/auth](https://opencode.ai/auth) | +| [Xiaomi MiMo](https://platform.xiaomimimo.com) | [platform.xiaomimimo.com](https://platform.xiaomimimo.com/console/api-keys) | +| [Kilo](https://kilo.ai) | [app.kilo.ai](https://app.kilo.ai) | +| [Pioneer](https://pioneer.ai) | [pioneer.ai](https://pioneer.ai) | +| [Hugging Face](https://huggingface.co) | [huggingface.co/settings/tokens](https://huggingface.co/settings/tokens) | + +New providers are added often, so **Providers → Usage-based** in the dashboard always shows the current list. + +### Gemini Free + +Gemini Free is a managed tile. Manifest provisions the credential through its own gateway, so there's no key to fetch from Google first: add the tile and you have a free model to route to. It behaves like any other provider once connected. + + + BytePlus, NousResearch, Command Code, ClinePass, Kiro, GitHub Copilot, Ollama + Cloud, and OpenCode Go connect through a plan rather than an API key, so they + have no API Keys tab. See + [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers). + + +## Add a provider + + + + In the dashboard sidebar, open **Providers → Usage-based**. Under + **Supported usage-based providers** you get a card per provider. + + + **Connect** opens a panel where you paste your API key. + + + Manifest validates the key by fetching the provider's model list. That list + is what routing sees until you refresh it. If the key is rejected, + double-check the prefix below. + + + + + The Providers → Usage-based page showing the sidebar Providers section, the connections table, and the grid of supported provider cards with Connect buttons + + +## Refreshing the model list + +A connection keeps the model list it discovered when you saved it. Nothing re-checks it on a schedule. When a provider adds, renames, or drops a model, click **Refresh models**: on your harness's **Routing** page, or on the connection itself under **Providers**. + +The refresh replaces the list, it does not merge it. A tier or a fallback pinned to a model id that no longer exists stops matching, so pin it to one of the new ids. + +## Key prefixes + +Each provider issues keys with a recognizable prefix. Useful for catching paste mistakes: + +| Provider | Prefix | Example | +| ---------- | --------- | ------------ | +| OpenAI | `sk-` | `sk-...` | +| Anthropic | `sk-ant-` | `sk-ant-...` | +| xAI | `xai-` | `xai-...` | +| Meta | `LLM_` | `LLM_...` | +| DeepSeek | `sk-` | `sk-...` | +| Moonshot | `sk-` | `sk-...` | +| MiniMax | `sk-` | `sk-...` | +| Qwen | `sk-` | `sk-...` | +| OpenRouter | `sk-or-` | `sk-or-...` | +| Google | none | API key | +| Mistral | none | API key | +| Z.ai | none | API key | +| Fireworks | `fw_` | `fw_...` | +| Groq | `gsk_` | `gsk_...` | +| Pioneer | `pio_sk_` | `pio_sk_...` | +| Xiaomi MiMo | `sk-` | `sk-...` | +| Hugging Face | `hf_` | `hf_...` | + +Providers not listed here (Cerebras, NVIDIA NIM, AWS Bedrock, Kilo, OpenCode Zen) issue keys with no fixed prefix. + +## Regional providers + +Some providers serve more than one region. Alibaba Cloud, AWS Bedrock, and MiniMax let you pick a region when you connect them, and Manifest routes to the matching endpoint. diff --git a/llm-gateway/docs/providers/custom-providers.mdx b/llm-gateway/docs/providers/custom-providers.mdx new file mode 100644 index 0000000..14e2499 --- /dev/null +++ b/llm-gateway/docs/providers/custom-providers.mdx @@ -0,0 +1,66 @@ +--- +title: "Custom providers" +sidebarTitle: "Custom" +description: "Plug in any OpenAI- or Anthropic-compatible endpoint, including vLLM, TGI, LocalAI, or your own internal service." +icon: "plug" +keywords: + ["vLLM", "TGI", "text-generation-inference", "LocalAI", "Xinference", "OpenLLM", "OpenAI-compatible", "Anthropic-compatible", "custom provider"] +canonical: "https://manifest.build/llm-gateway/docs/providers/custom-providers" +--- + +If your endpoint speaks OpenAI or Anthropic, Manifest can route to it. Useful for self-hosted inference servers (vLLM, TGI, LocalAI), internal endpoints behind your VPN, or providers that aren't on the built-in list yet. + +## Compatible servers + +Any server exposing one of these endpoints works out of the box: + +| Format | Endpoint | +| -------------------- | --------------------------- | +| OpenAI-compatible | `POST /v1/chat/completions` | +| Anthropic-compatible | `POST /v1/messages` | + +Common options that ship with one of these formats: [vLLM](https://github.com/vllm-project/vllm), [TGI](https://github.com/huggingface/text-generation-inference), [LocalAI](https://localai.io), [Xinference](https://github.com/xorbitsai/inference), [OpenLLM](https://github.com/bentoml/OpenLLM). + +## Add a custom provider + + + + In the dashboard sidebar, open **Providers → Usage-based** and click + **Add custom provider**. + + + Paste the base URL of your endpoint, e.g. `https://my-vllm.internal:8000/v1`. Manifest normalizes the trailing `/v1` automatically. + + + Choose **OpenAI** (`/v1/chat/completions`) or **Anthropic** (`/v1/messages`), whichever your server speaks. + + + If the endpoint requires authentication, paste an API key. It's sent as `Authorization: Bearer ` for OpenAI-format endpoints, or `x-api-key: ` for Anthropic-format endpoints. + + + Manifest calls `GET /v1/models` against your base URL and lists every model the endpoint reports. Pin one to your default or a custom tier and you're routed. + + + +## Model discovery + +Manifest discovers models by hitting `GET /v1/models` and reading the `data[].id` field. If your server doesn't expose `/v1/models`, you can register models manually from the same panel. + + + Older builds of `llama.cpp` (pre-b3800) don't expose `/v1/models`. Either + upgrade `llama-server` or register models by hand. + + +## Security + +User-supplied URLs are revalidated on every request to defend against SSRF. On Manifest Cloud, Manifest blocks resolution to private IP ranges (`10.x`, `192.168.x`, `127.x`, link-local): a custom provider must use a public `https://` URL. Self-hosted deployments accept private addresses and plain `http://`, so they can reach a server inside your own network. + + + Custom providers run with your Manifest instance's network access. If you + expose Manifest publicly, anyone with a valid harness key can route requests + through any custom provider you've added. Gate access accordingly. + + +## Cost tracking + +Manifest can't infer pricing for unknown models. Custom-provider requests show up in the dashboard with `cost = 0` and `model = `. Token counts and latency are still recorded, so [hard limits](/llm-gateway/docs/llm-gateway#hard-limits) on token volume still work. diff --git a/llm-gateway/docs/providers/local-models.mdx b/llm-gateway/docs/providers/local-models.mdx new file mode 100644 index 0000000..da4714c --- /dev/null +++ b/llm-gateway/docs/providers/local-models.mdx @@ -0,0 +1,110 @@ +--- +title: "Local models" +description: "Run any GGUF model on your own hardware with Ollama, LM Studio, or llama.cpp. No API costs, no data leaving your machine." +icon: "server" +keywords: + ["Ollama", "LM Studio", "llama.cpp", "GGUF", "local LLM", "self-hosted inference", "host.docker.internal"] +canonical: "https://manifest.build/llm-gateway/docs/providers/local-models" +--- + +Local model providers run entirely on your own hardware. Manifest detects the running server, fetches the model list, and routes requests to `http://localhost:` like any other provider. No API key, no network egress, no per-token cost. + + + The built-in Ollama, LM Studio, and llama.cpp connections are self-hosted only. To reach a local model from [Manifest Cloud](https://app.manifest.build), expose the runtime on a public URL or tunnel and add it as a [custom provider](/llm-gateway/docs/providers/custom-providers). + + +## Supported runtimes + +| Runtime | Default port | Install | +| -------------------------------------------------- | ------------ | ---------------------------------------------------------------------------------------------- | +| [Ollama](https://ollama.com) | `11434` | [ollama.com/download](https://ollama.com/download) | +| [LM Studio](https://lmstudio.ai) | `1234` | [lmstudio.ai](https://lmstudio.ai) | +| [llama.cpp](https://github.com/ggml-org/llama.cpp) | `8080` | [llama.cpp build guide](https://github.com/ggml-org/llama.cpp#obtaining-and-quantizing-models) | + +All three speak OpenAI-compatible `/v1/chat/completions` and accept any GGUF model file. + +## Start the server + + + + ```bash + ollama pull llama3.1:8b # then: + ollama serve + ``` + + + ```bash + lms server start + ``` + Or open the app: **Developer** tab → **Start server**. + + + ```bash + llama-server -m .gguf --port 8080 + ``` + Replace `.gguf` with the path to a GGUF file on your machine. + + + +## Connect to Manifest + + + + In the dashboard sidebar, open **Providers → Local** and click **Connect** on + your runtime (Ollama, LM Studio, or llama.cpp). + + + Manifest probes `http://localhost:/v1/models`. If the probe succeeds, every loaded model appears for routing. + + + Open your default or a custom tier and pick a local model as the primary. You can mix local and cloud models in the same fallback chain. + + + +## Running Manifest in Docker + +If you self-host Manifest in Docker, the container can't reach a local server bound to `127.0.0.1` on the host. Two of the three runtimes default to loopback and need an explicit override: + + + + Either flip the GUI toggle (**LM Studio → ⚙ Developer → Serve on Local Network**) or rebind from the CLI: + + ```bash + lms server start --bind 0.0.0.0 --port 1234 --cors + ``` + + LM Studio remembers the last `--bind`, so this is one-time setup. + + + + `llama-server` only listens on `0.0.0.0` if you pass `--host`: + + ```bash + llama-server -m .gguf --host 0.0.0.0 --port 8080 + ``` + + + + Ollama already binds `0.0.0.0` by default. No change needed. + + + + + Inside the Manifest container, the host is reachable as + `host.docker.internal`. Manifest sets this automatically when probing local + providers. + + +## Cost & privacy + +| Aspect | Local | +| --------------------- | -------------------------------------------------------------- | +| **API cost** | $0. The model runs on your hardware. | +| **Network egress** | None. Requests never leave the machine. | +| **Cost in dashboard** | Recorded as `0`. Token counts and latency are still tracked. | +| **Pricing data** | Not applicable. Local providers are excluded from pricing sync.| + + + Mix local and cloud in one chain: set a local model as your default for + day-to-day calls, and fall back to a cloud model when the local server is offline. + diff --git a/llm-gateway/docs/providers/subscription-based-providers.mdx b/llm-gateway/docs/providers/subscription-based-providers.mdx new file mode 100644 index 0000000..4992a58 --- /dev/null +++ b/llm-gateway/docs/providers/subscription-based-providers.mdx @@ -0,0 +1,107 @@ +--- +title: "Subscription-based providers" +sidebarTitle: "Subscription-based" +description: "Reuse a paid plan you already have: ChatGPT, Claude, Gemini, Grok, GitHub Copilot, GLM Coding Plan, Kimi, Mistral Vibe, and more." +icon: "credit-card" +keywords: + ["ChatGPT Plus", "Claude Max", "GitHub Copilot", "GLM Coding Plan", "Mistral Vibe", "Gemini", "Grok", "Kimi", "OAuth", "device code"] +canonical: "https://manifest.build/llm-gateway/docs/providers/subscription-based-providers" +--- + +If you already pay for ChatGPT Plus, Claude Max, GitHub Copilot, or one of the other plans listed below, Manifest can route through the subscription instead of an API key. Auth is OAuth, a device code, or a pasted subscription token, depending on the provider. + +## Supported subscriptions + +| Provider | Plan | Auth flow | +| ----------------------------------------------------- | -------------------- | ------------------ | +| [OpenAI](https://openai.com/chatgpt) | ChatGPT Plus/Pro | OAuth (browser) | +| [Anthropic](https://www.anthropic.com/claude) | Claude Pro/Max | Subscription token | +| [Google](https://gemini.google.com) | Gemini | OAuth (browser) | +| [xAI](https://x.ai) | Grok | OAuth (browser) | +| [GitHub Copilot](https://github.com/features/copilot) | Copilot | Device code | +| [MiniMax](https://www.minimaxi.com) | Coding Plan | Device code | +| [Kiro](https://kiro.dev) | Kiro | Device code | +| [Z.ai (Zhipu)](https://z.ai) | GLM Coding Plan | Subscription token | +| [Alibaba Cloud (Qwen)](https://qwen.ai) | Qwen Token Plan | Subscription token | +| [Moonshot](https://platform.moonshot.cn) | Kimi Coding Plan | Subscription token | +| [Mistral](https://mistral.ai) | Vibe | Subscription token | +| [BytePlus](https://www.byteplus.com) | ModelArk Coding Plan | Subscription token | +| Xiaomi MiMo | Token Plan | Subscription token | +| [NousResearch](https://nousresearch.com) | Subscription | Subscription token | +| Command Code | Subscription | Subscription token | +| [ClinePass](https://cline.bot) | ClinePass | Subscription token | +| [OpenCode](https://opencode.ai) | OpenCode Go | Subscription token | +| [Ollama Cloud](https://ollama.com) | Ollama Cloud | Subscription token | + + + The Kimi Coding Plan serves `k3`, `k3-256k`, `kimi-for-coding` and + `kimi-for-coding-highspeed`. If a tier or a fallback still points at + `kimi-k3`, click **Refresh models** on the connection and pin one of these + ids instead. + + +## Connect a subscription + + + + In the dashboard sidebar, open **Providers → Subscriptions** and click + **Connect** on the provider. + + + Manifest opens the provider's OAuth page, shows you a device code to enter, + or asks you to paste a token, depending on the provider. Once it's saved, + you're routed. + + + +A model your plan just gained doesn't show up on its own. Click **Refresh models** on the connection to pick it up. + +## How auth works per provider + + + + Standard OAuth against `auth.openai.com`. Once approved, requests route + through the Codex backend at `chatgpt.com/backend-api/codex/responses` + rather than the public OpenAI API. Tokens refresh automatically. + + + Generate a token with the Claude CLI (`claude setup-token`) and paste it into + Manifest. Requests carry the `anthropic-beta: oauth-2025-04-20` header so + Anthropic recognizes the subscription token. Available models match what your + plan grants in claude.ai. + + + GitHub's device-code flow. Manifest gives you a short user code, you visit + [github.com/login/device](https://github.com/login/device), paste it, and + approve. No API key is ever issued. Requests then go to + `api.githubcopilot.com/chat/completions` using a short-lived token Manifest + refreshes for you. + + + MiniMax's device-code flow. Manifest gives you a short user code to approve, + then requests use the Anthropic protocol via `api.minimax.io/anthropic` + (international) or `api.minimaxi.com/anthropic` (China). + + + Paste the subscription token issued by Z.ai and pick a region in the + connect form. Requests route to `api.z.ai` (outside China, the default) + or `open.bigmodel.cn` (China Mainland). + + + Paste the subscription token from your OpenCode account. Requests use the + Anthropic protocol against `opencode.ai/zen/go`, authenticated via the + `x-api-key` header. + + + + + Subscription auth is rate-limited by the provider's plan, not by Manifest. If + you hit a plan ceiling, requests fall through to your next + [fallback](/llm-gateway/docs/llm-gateway#fallback) model, usually a different provider on an API key. + + +## Why mix subscriptions and API keys + +A common setup: subscription as the primary (predictable monthly cost), API-key provider as the fallback for when you hit the plan limit or want a model the subscription doesn't include. + +Pin your subscription model to your default or a custom tier and add API-key models to the [fallback list](/llm-gateway/docs/llm-gateway#configuration). Manifest handles the switch. diff --git a/llm-gateway/docs/reference/api.mdx b/llm-gateway/docs/reference/api.mdx new file mode 100644 index 0000000..2fb2a71 --- /dev/null +++ b/llm-gateway/docs/reference/api.mdx @@ -0,0 +1,221 @@ +--- +title: "API" +description: "The Manifest proxy speaks both OpenAI and Anthropic. Endpoints, auth, streaming, and error responses." +icon: "code" +keywords: + ["chat completions", "responses API", "Anthropic messages", "auto", "v1/models", "streaming", "SSE", "fallback exhausted"] +canonical: "https://manifest.build/llm-gateway/docs/reference/api" +--- + +Manifest exposes both OpenAI and Anthropic-format endpoints on one proxy. Point your client at the Manifest URL, send `auto` as the model, and routing picks the real model behind the scenes. + +## Base URL + +| Mode | URL | +|---|---| +| Cloud | `https://app.manifest.build` | +| Self-hosted | `http://localhost:2099` (or your custom port) | + +## Authentication + +Every request requires a Manifest harness key: + +```http +Authorization: Bearer mnfst_YOUR_KEY_HERE +``` + +Generate a key from the dashboard's **Harnesses** page. Keys always start with `mnfst_`. + +## Endpoints + +| Method | Path | Format | Use it for | +|---|---|---|---| +| `POST` | `/v1/chat/completions` | OpenAI | Most clients (OpenAI SDK, LangChain, Vercel AI SDK, custom HTTP) | +| `POST` | `/v1/responses` | OpenAI Responses | Codex, `*-pro`, `o1-pro`, deep-research models | +| `POST` | `/v1/messages` | Anthropic | Anthropic SDK, Claude Code, anything that speaks the Messages API | +| `GET` | `/v1/models` | OpenAI | Listing the models your harness can route to | + +The proxy translates between formats internally, so you can send an OpenAI-shaped request and Manifest will reshape it before forwarding to an Anthropic-only model. The reverse works too. + +Translation carries what the request itself contains: messages, tools, and tool results. Manifest does not resolve `previous_response_id`, so send the full conversation in `input` on every `/v1/responses` request. + +## Chat completions + +```bash +curl -X POST http://localhost:2099/v1/chat/completions \ + -H "Authorization: Bearer mnfst_YOUR_KEY_HERE" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "auto", + "messages": [ + {"role": "user", "content": "What is the capital of France?"} + ] + }' +``` + +Manifest adapts the body to the provider it picked. It replaces `model` with the real model ID. It converts the body to the format that provider expects. It renames parameters the provider spells differently, like `max_tokens` and `max_completion_tokens`. And it merges the model parameters saved on your harness into every attempt. Some fields only exist at OpenAI: `stream_options`, `reasoning_effort`, `modalities`, `audio`, and `prediction`. Those reach OpenAI and OpenRouter. Every other provider gets the request without them, and nothing warns you. + +## Anthropic messages + +```bash +curl -X POST http://localhost:2099/v1/messages \ + -H "Authorization: Bearer mnfst_YOUR_KEY_HERE" \ + -H "Content-Type: application/json" \ + -H "anthropic-version: 2023-06-01" \ + -d '{ + "model": "auto", + "max_tokens": 1024, + "messages": [ + {"role": "user", "content": "Hello"} + ] + }' +``` + +## Listing models + +`GET /v1/models` returns the models your harness can reach, in OpenAI format. The first entry is always `auto` (routing); the rest are the real model IDs from your connected providers. + +```bash +curl http://localhost:2099/v1/models \ + -H "Authorization: Bearer mnfst_YOUR_KEY_HERE" +``` + +Send `auto` to let Manifest route, or send any listed model ID to skip routing and go straight to that provider. If you send a model ID that no connected provider can serve, Manifest returns [M302: Model not available](/llm-gateway/docs/errors/M302). See [Routing → Route a specific model](/llm-gateway/docs/llm-gateway#route-a-specific-model). + +### Inspect model capabilities + +Add `?capabilities=true` to include known capability metadata for each concrete model. Without this query parameter, the response keeps the standard OpenAI model-list shape. + +```bash +curl "http://localhost:2099/v1/models?capabilities=true" \ + -H "Authorization: Bearer mnfst_YOUR_KEY_HERE" +``` + +```json +{ + "object": "list", + "data": [ + { + "id": "auto", + "object": "model", + "created": 0, + "owned_by": "manifest" + }, + { + "id": "openai/gpt-5.4-mini-subscription", + "object": "model", + "created": 0, + "owned_by": "openai", + "capabilities": { + "input_modalities": ["text", "image"], + "output_modalities": ["text"], + "features": ["stream", "tools"] + } + } + ] +} +``` + +| Field | Meaning | +|---|---| +| `input_modalities` | Input types the model accepts, such as `text` or `image` | +| `output_modalities` | Output types the model can produce | +| `features` | Known feature support: `stream` and `tools` | +| `supported_endpoints` | API endpoint formats the model supports. Present only when the provider publishes them in its own model list | + +Capability fields are optional. A missing field means that support is unknown, not that the model does not support it. Manifest omits the entire `capabilities` object when it has no known metadata for a model. + +The synthetic `auto` model never includes capabilities because it can resolve to a different model for each request. Concrete model IDs remain directly routable exactly as listed, including IDs with the `-subscription` suffix. + +### Inspect model costs + +Add `?cost=true` to include known token prices for each concrete model. Prices are in USD per million tokens. Without this query parameter, the response keeps the standard OpenAI model-list shape. + +```bash +curl "http://localhost:2099/v1/models?cost=true" \ + -H "Authorization: Bearer mnfst_YOUR_KEY_HERE" +``` + +```json +{ + "object": "list", + "data": [ + { + "id": "auto", + "object": "model", + "created": 0, + "owned_by": "manifest" + }, + { + "id": "openai/gpt-5.4-mini", + "object": "model", + "created": 0, + "owned_by": "openai", + "cost": { + "input": 0.75, + "output": 4.5 + } + }, + { + "id": "openrouter/example-free-model", + "object": "model", + "created": 0, + "owned_by": "openrouter", + "cost": { + "input": 0, + "output": 0 + } + } + ] +} +``` + +| Field | Meaning | +|---|---| +| `input` | USD per million input tokens | +| `output` | USD per million output tokens | + +A zero value means the model has no per-token charge, as with some free or subscription-backed routes. If one price is unknown, Manifest omits only that field. If both prices are unknown, Manifest omits the entire `cost` object. + +The synthetic `auto` model never includes cost because its concrete model is selected for each request. To inspect both metadata types in one response, combine the query parameters: `?capabilities=true&cost=true`. + +## Streaming + +Set `"stream": true` to get an SSE stream back. The stream format matches the upstream protocol: OpenAI-style `data: {...}` chunks for `/v1/chat/completions`, Anthropic event blocks for `/v1/messages`. + +Routing and fallback both work with streams. If the primary model fails before the first chunk, the request restarts on the fallback. If it fails mid-stream, the connection closes. There's no silent mid-stream retry. + +## Errors + +Errors come in two shapes, depending on the caller. A tool or SDK call gets a real HTTP status and the JSON envelope below. A chat or streaming client gets an HTTP `200` that looks like a normal completion, and the error text sits inside the assistant message. + +```json +{ + "error": { + "message": "[🦚 Manifest M005] I don't recognize this key. ...", + "type": "auth_error", + "code": "manifest_auth" + } +} +``` + +`error.code` is a string identifier, never the numeric HTTP status. + +| Status | Meaning | +|---|---| +| `401` | Invalid or missing `Authorization` header | +| `402` | Manifest Free plan quota reached (`error.code = PLAN_LIMIT_REQUESTS`, [M204](/llm-gateway/docs/errors/M204)), or provider billing/quota error from the upstream | +| `429` | Manifest rate limit tripped ([M201](/llm-gateway/docs/errors/M201), [M202](/llm-gateway/docs/errors/M202), [M203](/llm-gateway/docs/errors/M203)), or a rate limit the provider itself returned | +| `5xx` | Upstream provider error (triggers [fallback](/llm-gateway/docs/llm-gateway#fallback)) | + +A [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) block ([M200](/llm-gateway/docs/errors/M200)) is not in this table: it comes back as an HTTP `200` chat completion whose assistant message carries the block text. When the fallback chain is exhausted, the response keeps the primary model's real error status and carries `X-Manifest-Fallback-Exhausted: true`; the body keeps the provider's own error code, or `fallback_exhausted` when there is none. + +## Rate limits + +The gateway enforces three caps. A workspace can send **200 requests per minute** ([M201](/llm-gateway/docs/errors/M201)). An IP can send **500 requests per minute** ([M202](/llm-gateway/docs/errors/M202)). A workspace can have **10 requests in flight at once** ([M203](/llm-gateway/docs/errors/M203)). The caps are the same on Cloud and self-hosted. No setting changes them. + +Your Cloud plan sets your monthly request quota ([M204](/llm-gateway/docs/errors/M204)). The per-minute caps stay the same on every plan. + +## Response headers + +Routed responses carry [routing headers](/llm-gateway/docs/reference/headers). They tell your client which model and tier handled the request, with no need to parse the response body. A request rejected before routing (bad key, quota, rate limit) carries none. diff --git a/llm-gateway/docs/reference/environment-variables.mdx b/llm-gateway/docs/reference/environment-variables.mdx new file mode 100644 index 0000000..102a0f7 --- /dev/null +++ b/llm-gateway/docs/reference/environment-variables.mdx @@ -0,0 +1,180 @@ +--- +title: "Environment variables" +description: "Every environment variable Manifest reads at startup, grouped by what it controls: database, auth, rate limiting, email, OAuth, telemetry." +icon: "settings" +keywords: + ["DATABASE_URL", "BETTER_AUTH_SECRET", "BETTER_AUTH_URL", "PORT", "BIND_ADDRESS", "THROTTLE_LIMIT", "EMAIL_PROVIDER", "Resend", "Mailgun", "SendGrid", "MANIFEST_TELEMETRY_DISABLED"] +canonical: "https://manifest.build/llm-gateway/docs/reference/environment-variables" +--- + +Manifest reads its configuration from environment variables. In the bundled Docker setup these come from `~/manifest/.env`. For `docker run`, pass them with `-e`. For non-Docker installs, export them in the shell before launching the backend. + + + This page is for [self-hosted](/llm-gateway/docs/self-hosted) instances only. On [Manifest + Cloud](https://app.manifest.build) there is no server to configure and no + `.env` to edit — everything you can change lives in the dashboard. + + +## Core + +| Variable | Required | Default | Description | +|---|---|---|---| +| `DATABASE_URL` | Yes | — | PostgreSQL connection string. Format: `postgresql://user:pass@host:5432/dbname` | +| `BETTER_AUTH_SECRET` | Yes | — | Session signing secret. Min 32 chars. Generate with `openssl rand -hex 32` | +| `BETTER_AUTH_URL` | No | `http://localhost:2099` | Public URL the dashboard is reachable on. Must match the browser URL | +| `PORT` | No | `2099` | Dashboard port. Under the bundled compose file this sets both the published host port and the internal listener, and `BETTER_AUTH_URL` follows it | +| `NODE_ENV` | No | `production` | Node environment. Telemetry is disabled when this isn't `production`. Fixed to `production` by the bundled compose file — the image is a production artifact | +| `MANIFEST_MODE` | No | — | Marks the install as self-hosted when Manifest can't tell on its own. Accepted value: `selfhosted` (`local` is a legacy alias with the same effect). Unset, Manifest detects Docker, Podman, and Kubernetes automatically; some platforms hide the container, which is why the [Render](/llm-gateway/docs/deploy/render), [Koyeb](/llm-gateway/docs/deploy/koyeb), [Easypanel](/llm-gateway/docs/deploy/easypanel), and [AWS](/llm-gateway/docs/deploy/aws) guides set it | + +## Network & security + +| Variable | Default | Description | +|---|---|---| +| `BIND_ADDRESS` | `127.0.0.1` | Interface the server binds to. Set to `0.0.0.0` for LAN access. The Docker image already sets `0.0.0.0`; under the bundled compose file, control host exposure with `HOST_BIND_ADDRESS` instead | +| `HOST_BIND_ADDRESS` | `127.0.0.1` | Host interface the bundled compose file binds the dashboard port to. Set `0.0.0.0` to expose it on the LAN | +| `CORS_ORIGIN` | `http://localhost:3000` | Allowed CORS origin for browser-based dashboard requests. Development only — production uses the built-in allowlist plus `WINGMAN_CORS_ORIGINS` | +| `API_KEY` | — | Internal API key for system endpoints | +| `MANIFEST_ENCRYPTION_KEY` | `BETTER_AUTH_SECRET` | Key used to encrypt stored provider credentials at rest. Falls back to `BETTER_AUTH_SECRET` when unset — set a separate 32+ char value so a session-cookie leak doesn't also decrypt every stored provider key. Set it before first boot | +| `MANIFEST_DISABLE_HSTS` | — | Set `1` to silence the boot warning about HSTS being unavailable on a plain-HTTP deployment. Prefer an `https://` `BETTER_AUTH_URL` anywhere reachable from the internet | +| `WINGMAN_CORS_ORIGINS` | — | Extra browser origins allowed to call the gateway, comma-separated. Useful when you host your own copy of [Wingman](https://github.com/mnfst/wingman), the open-source gateway tester; the hosted one at `wingman.manifest.build` is always allowed | +| `THROTTLE_TTL` | `60000` | Window (ms) for the rate limit on the dashboard's own API calls. Does not apply to gateway traffic on `/v1` | +| `THROTTLE_LIMIT` | `100` | Max dashboard API calls per window. The gateway has its own fixed caps, see [Rate limits](/llm-gateway/docs/reference/api#rate-limits) | + +## Database + +| Variable | Default | Description | +|---|---|---| +| `DB_POOL_MAX` | `10` | Max PostgreSQL connections in the main pool | +| `AUTH_DB_POOL_MAX` | `5` | Separate pool used by Better Auth. Counted on top of `DB_POOL_MAX` when sizing your server's `max_connections` | +| `RUN_MIGRATIONS_ON_BOOT` | `true` | Run pending migrations at startup. Set `false` on multi-replica deploys so only one instance migrates | +| `DB_TUNE_SESSION` | `true` | Apply Manifest's planner defaults at boot. Set `false` on managed Postgres where your role can't `ALTER ROLE` itself | +| `SEED_DATA` | `false` | Seed demo data on first boot. Development only — the seeder refuses to run under `NODE_ENV=production`, so it has no effect on a Docker self-host. Use the first-run setup wizard to create your admin account | + +## LLM proxy + +How Manifest talks to upstream providers. + +| Variable | Default | Description | +|---|---|---| +| `PROVIDER_TIMEOUT_MS` | `180000` | Per-attempt timeout (ms) for upstream requests. A hung provider surfaces as a synthetic `504` and triggers the next [fallback](/llm-gateway/docs/llm-gateway#fallback). Set it strictly below your client's own timeout; slow local models may need it raised | +| `STREAM_WARMUP_MS` | `15000` | How long to wait for the first streamed chunk before falling back to the next model | +| `OLLAMA_HOST` | `http://localhost:11434` | Base URL for a local Ollama server. In Docker, set it to `http://host.docker.internal:11434` | + +## Request recordings + +Storage and retention for [request logs](/llm-gateway/docs/request-logs). The bundled Docker setup already sets the filesystem path and mounts a volume for it, so a default install needs none of these. + +| Variable | Default | Description | +|---|---|---| +| `REQUEST_RECORDING_STORAGE` | `auto` | `auto`, `s3`, `filesystem`, or `disabled`. Under `auto`, any S3 variable being set selects S3; otherwise the filesystem path is used | +| `REQUEST_RECORDING_FILESYSTEM_PATH` | `/data/request-recordings` | Where recordings are written on disk. Must be a persistent volume | +| `REQUEST_RECORDING_RETENTION_DAYS` | `365` | Days to keep recordings before the sweep deletes them | +| `REQUEST_RECORDING_S3_BUCKET` | — | Bucket name. Required for S3 | +| `REQUEST_RECORDING_S3_REGION` | — | Region. Use `auto` for R2 and similar. Required for S3 | +| `REQUEST_RECORDING_S3_ENDPOINT` | — | Custom endpoint for non-AWS S3-compatible storage | +| `REQUEST_RECORDING_S3_ACCESS_KEY_ID` | — | Access key. Omit both key variables to use ambient credentials | +| `REQUEST_RECORDING_S3_SECRET_ACCESS_KEY` | — | Secret key. Must be set together with the access key ID | +| `REQUEST_RECORDING_S3_FORCE_PATH_STYLE` | `false` | Set `true` for storage that needs path-style bucket addressing (MinIO) | + + + A partial S3 configuration disables recording rather than falling back to + local disk. Set bucket **and** region, and either both access-key variables or + neither. + + +## Autofix + +[Autofix](/llm-gateway/docs/autofix) works with zero configuration; `AUTOFIX_GLOBAL_ENABLED=false` is the kill switch. + +| Variable | Default | Description | +|---|---|---| +| `AUTOFIX_GLOBAL_ENABLED` | `true` | Set `false` to turn Autofix off for the whole deployment. No call reaches the healing service, including the boot health check | +| `AUTOFIX_TIMEOUT_MS` | `10000` | Timeout per healing call, in milliseconds | +| `AUTOFIX_REPAIRABLE_STATUSES` | `400,404,422` | Which provider statuses are eligible for a repair | +| `AUTOFIX_HEALING_API_KEY` | unset | Sent as `x-api-key` to the healing service. Self-hosted installs don't need it; with no key set, the install announces its anonymous install id instead. Only relevant if Manifest issues you one | +| `AUTOFIX_REPORT_ALL_4XX` | `false` | Opt-in. Also report request-side 4xx errors (except `401`/`402`/`403`/`429`) from harnesses with Autofix on, as diagnostic evidence with the secret-scrubbed request body. Bodies over 256 KB are dropped, not truncated. No fix comes back from it | + +## Email + +Used for both Better Auth transactional emails (signup verification, password reset) **and** [threshold alerts](/llm-gateway/docs/observability#spend-alerts). Set one provider block. + +A provider saved on a harness's [Limits page](/llm-gateway/docs/observability#email-delivery) takes precedence for threshold alerts. These variables stay the only path for verification and password-reset email. + +| Variable | Description | +|---|---| +| `EMAIL_PROVIDER` | `resend`, `mailgun`, or `sendgrid` | +| `EMAIL_API_KEY` | API key for the chosen provider | +| `EMAIL_DOMAIN` | Sending domain (Mailgun only) | +| `EMAIL_FROM` | From-address for outbound mail | + + + Without an email provider, signup verification is waived (users are created as unverified-but-usable) and password reset silently no-ops. Hard-limit blocks still work, and alerts still email if a provider is saved on a harness's **Limits** page. + + +### Legacy Mailgun-only + +Older deployments used these. Kept for backward compatibility; new installs should use `EMAIL_*` instead. + +| Variable | Description | +|---|---| +| `MAILGUN_API_KEY` | Mailgun API key | +| `MAILGUN_DOMAIN` | Mailgun domain | +| `NOTIFICATION_FROM_EMAIL` | Sender address for alerts | + +## OAuth logins + +Each provider activates automatically when both `*_CLIENT_ID` and `*_CLIENT_SECRET` are set. Configure the callback URL in the provider's console as `${BETTER_AUTH_URL}/api/auth/callback/`. + +| Variable | Provider | +|---|---| +| `GOOGLE_CLIENT_ID` / `GOOGLE_CLIENT_SECRET` | Google | +| `GITHUB_CLIENT_ID` / `GITHUB_CLIENT_SECRET` | GitHub | +| `DISCORD_CLIENT_ID` / `DISCORD_CLIENT_SECRET` | Discord | + +Subscription OAuth (ChatGPT, Claude, MiniMax) uses Manifest-side client IDs by default. Override only if you've registered your own app: + +| Variable | Description | +|---|---| +| `OPENAI_OAUTH_CLIENT_ID` | Custom OpenAI OAuth client ID | +| `MINIMAX_OAUTH_CLIENT_ID` | Custom MiniMax OAuth client ID | + +## Telemetry + +| Variable | Default | Description | +|---|---|---| +| `MANIFEST_TELEMETRY_DISABLED` | unset | Set to `1` to disable [anonymous telemetry](/llm-gateway/docs/reference/telemetry) | +| `TELEMETRY_ENDPOINT` | `https://telemetry.manifest.build/...` | Send reports to your own endpoint | +| `MANIFEST_PUBLIC_STATS` | `false` | Set `true` to expose `/api/v1/public/*` aggregate stats without auth | + +## Error monitoring + +Sentry is opt-in and stays off unless you give it a DSN. + +| Variable | Default | Description | +|---|---|---| +| `SENTRY_DSN` | unset | Your Sentry project DSN. Leave it unset to disable Sentry entirely | +| `SENTRY_ENVIRONMENT` | `NODE_ENV` | Environment tag on reported events | +| `SENTRY_RELEASE` | unset | Release tag on reported events | + +## Operations + +| Variable | Default | Description | +|---|---|---| +| `SHUTDOWN_DRAIN_MS` | `10000` | How long the server keeps serving traffic after a termination signal while reporting unhealthy, so a load balancer can deregister it first. Set `0` to shut down immediately | + +## Image version + +| Variable | Default | Description | +|---|---|---| +| `MANIFEST_VERSION` | `latest` | Image tag the bundled compose file pulls. Set `6` to follow a major line, or `6.18.0` to pin one release. Docker Compose substitutes it into the image name; the app itself never reads it. See [Image tags](/llm-gateway/docs/self-hosted#image-tags) | + +## Postgres bundled volume + +When you run the bundled compose file, Manifest brings up its own Postgres container. To use a stronger password than the default, set **both** of these. They must agree, and special characters in the password must be percent-encoded in `DATABASE_URL`: + +| Variable | Description | +|---|---| +| `POSTGRES_PASSWORD` | Password for the bundled Postgres user | +| `DATABASE_URL` | Connection string with the matching password | + +Special-char encoding: `@` → `%40`, `:` → `%3A`, `/` → `%2F`. diff --git a/llm-gateway/docs/reference/glossary.mdx b/llm-gateway/docs/reference/glossary.mdx new file mode 100644 index 0000000..70c673a --- /dev/null +++ b/llm-gateway/docs/reference/glossary.mdx @@ -0,0 +1,71 @@ +--- +title: "Glossary" +description: "Definitions for harness, request, attempt, Autofix, tier, default and custom routing, direct routing, fallback chain, auto, and provider auth types." +icon: "book" +keywords: + ["agent", "harness", "request", "attempt", "autofix", "tier", "custom tier", "default routing", "direct routing", "fallback chain", "auto", "provider", "auth type"] +canonical: "https://manifest.build/llm-gateway/docs/reference/glossary" +--- + +## Harness + +A configured client connection that sends requests through Manifest. Each harness has its own API key (`mnfst_...`), its own routing rules, and its own usage page. A harness typically corresponds to one tool or workflow (your IDE plugin, a Slack bot, a scheduled job), not one user. + +## Request + +One call your client makes through Manifest. The dashboard's **Requests** page lists every request with its model, cost, and status. A single request can contain several [attempts](#attempt) when fallback or Autofix stepped in. + +## Attempt + +One provider call inside a request. The initial call, each [fallback](#fallback), and an [Autofix](#autofix) retry are separate attempts, each with its own provider, model, and result. Open a request to see its attempts side by side. + +## Autofix + +The repair step for requests that fail because of the request itself: a parameter the provider rejects, a tool schema in the wrong shape, a model name that moved. Manifest corrects the failing request and sends it once more, before fallback. See [Autofix](/llm-gateway/docs/autofix). + +## Auth type + +The credential category Manifest uses to talk to a provider: + +- **`api_key`** — classic per-token API key (most providers) +- **`subscription`** — OAuth or a subscription token tied to a paid plan (ChatGPT Plus, Claude Max, GLM Coding Plan, and others) +- **`local`** — no credential, the server runs on your machine (Ollama, LM Studio, llama.cpp) + +Auth type is recorded on every request and shows up in the dashboard's distribution chart. + +## Auto + +The model ID you send to opt into routing. When Manifest sees `auto`, it applies your routing rules and picks the real model. Send a real model ID instead to skip routing and target one provider directly (see [Direct routing](#direct-routing)). + +## Custom tier + +A routing rule matched on a request header. You choose the header key and value on the dashboard and pin the tier to its own model and fallback chain. When a request carries that header, it routes to the tier's model instead of the default. That holds even when the body names an explicit model ID. See [Routing → Custom](/llm-gateway/docs/llm-gateway#custom-tiers). + +## Direct routing + +Sending a real model ID (one that `GET /v1/models` lists) instead of `auto`. Manifest forwards the request straight to that provider and skips tiers and fallbacks. The response carries `X-Manifest-Tier: direct`. One exception: if the request also carries a matching custom-tier header, the header wins. + +## Fallback + +The retry mechanism that kicks in when the primary model fails. Manifest tries the next model in the tier's fallback list, then the next, until one succeeds or the list is exhausted. See [Fallback](/llm-gateway/docs/llm-gateway#fallback) for triggers and config. + +## Fallback chain + +The ordered list of models tried for a single tier, primary first. Up to 5 models. When all of them fail, Manifest returns the primary model's real error status, plus the header `X-Manifest-Fallback-Exhausted: true`. + +## Provider + +An upstream LLM service Manifest can route to. There are four kinds: + +- [API key](/llm-gateway/docs/providers/api-key-providers) — pay-per-token (OpenAI, Anthropic, Google, and more) +- [Subscription](/llm-gateway/docs/providers/subscription-based-providers) — a plan you already pay for (ChatGPT Plus, Claude Max, and more) +- [Local](/llm-gateway/docs/providers/local-models) — runs on your hardware (Ollama, LM Studio, llama.cpp) +- [Custom](/llm-gateway/docs/providers/custom-providers) — any OpenAI- or Anthropic-compatible HTTP endpoint + +## Routing + +The step that decides which model handles a request. Manifest checks your custom tiers first, then sends anything that doesn't match to your default model. It runs in-process with no extra network call. See [Routing](/llm-gateway/docs/llm-gateway). + +## Tier + +A routing lane with its own model and fallback chain. Every harness has a **Default** tier; you can add **Custom** tiers matched on request headers. See [Routing](/llm-gateway/docs/llm-gateway). diff --git a/llm-gateway/docs/reference/headers.mdx b/llm-gateway/docs/reference/headers.mdx new file mode 100644 index 0000000..563f8db --- /dev/null +++ b/llm-gateway/docs/reference/headers.mdx @@ -0,0 +1,76 @@ +--- +title: "Headers" +description: "The request headers Manifest reads and the X-Manifest-* response headers it returns, with what each one does." +icon: "list" +keywords: + ["X-Manifest-Tier", "X-Manifest-Model", "X-Manifest-Provider", "X-Manifest-Reason", "X-Manifest-Fallback-From", "x-session-key", "custom routing header"] +canonical: "https://manifest.build/llm-gateway/docs/reference/headers" +--- + +Manifest reads a few request headers, and returns a set of `X-Manifest-*` response headers so clients can see what happened without parsing the response body. + +## Request headers + +| Header | Value | Effect | +| ----------------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Authorization` | `Bearer mnfst_` | **Required.** Authenticates the harness. | +| `Content-Type` | `application/json` | **Required.** | +| `anthropic-version` | `2023-06-01` | Accepted and ignored. Anthropic SDKs send it on their own. Manifest picks the version it sends upstream. | +| Your custom tier header | your value | Routes the request to a [custom tier](/llm-gateway/docs/llm-gateway#custom-tiers). You set the key and value when you create the tier, so the exact header name is up to you. | +| `x-session-key` | any string | Groups requests into a session. Manifest keeps a session on the same upstream where the provider supports it, for sticky routing and prompt caching. | + +The custom tier header is the only one that changes routing: send the header you configured on a tier and the request goes to that tier's model. The header wins even when the body names an explicit model ID. See [Routing → Custom](/llm-gateway/docs/llm-gateway#custom-tiers). + +## Response headers (routed requests) + +| Header | Description | Example | +| ---------------------------- | -------------------------------------------------------------------------------- | --------------- | +| `X-Manifest-Tier` | How the request was routed: `standard` after a custom-tier header match, `direct` for an explicit model ID, `simple` when Manifest answered without calling a provider | `direct` | +| `X-Manifest-Model` | Model that actually served the response | `claude-sonnet-4-6` | +| `X-Manifest-Provider` | Upstream provider | `anthropic` | +| `X-Manifest-Reason` | Why that route was picked, for example `header-match`, `direct`, or `default`. Other values exist | `header-match` | +| `X-Manifest-Output-Modality` | Output modality of the response | `text` | +| `X-Manifest-Response-Mode` | Whether the response was streamed or `buffered` | `buffered` | + +Your custom tier's name appears on the request in the dashboard, never in a header. When Manifest answers a request itself (a spend block, no provider connected, a model it cannot serve), `X-Manifest-Model` and `X-Manifest-Provider` both report `manifest`. + +## Response headers (fallback only) + +When the primary model fails and Manifest succeeds on a fallback, two extra headers are added: + +| Header | Description | Example | +| --------------------------- | --------------------------------------------------- | ------- | +| `X-Manifest-Fallback-From` | The primary model that was attempted first | `gpt-5` | +| `X-Manifest-Fallback-Index` | Position in the fallback chain (0 = first fallback) | `0` | + +When **every** model in the chain fails: + +| Header | Description | +| ------------------------------- | ---------------------------------------- | +| `X-Manifest-Fallback-Exhausted` | Set to `true`. The response keeps the primary model's real error status; the body carries code `fallback_exhausted`. | + +## Reading headers in code + +```javascript +const response = await fetch("http://localhost:2099/v1/chat/completions", { + method: "POST", + headers: { + Authorization: `Bearer ${process.env.MANIFEST_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: "auto", + messages: [{ role: "user", content: "Hello" }], + }), +}); + +console.log(response.headers.get("x-manifest-tier")); // → "default" +console.log(response.headers.get("x-manifest-model")); // → "gpt-5-mini" +console.log(response.headers.get("x-manifest-provider")); // → "openai" +``` + + + Header names are case-insensitive in HTTP, but most browser `fetch` + implementations lowercase them when reading. The proxy emits them as + `X-Manifest-*`. + diff --git a/llm-gateway/docs/reference/telemetry.mdx b/llm-gateway/docs/reference/telemetry.mdx new file mode 100644 index 0000000..6d9882e --- /dev/null +++ b/llm-gateway/docs/reference/telemetry.mdx @@ -0,0 +1,97 @@ +--- +title: "Data and telemetry" +sidebarTitle: "Data & telemetry" +description: "What Manifest stores about your requests, what's optional, and the anonymous daily report each self-hosted install sends. Includes the full field list and how to opt out." +icon: "shield" +keywords: + ["data privacy", "prompt storage", "message recording", "anonymous telemetry", "MANIFEST_TELEMETRY_DISABLED", "TELEMETRY_ENDPOINT", "install_id", "opt out", "self-hosted telemetry"] +canonical: "https://manifest.build/llm-gateway/docs/reference/telemetry" +--- + +Two separate things happen to data as requests flow through Manifest: what your own instance records for the dashboard, and what a self-hosted install reports back to the project. This page covers both. + +## What Manifest stores + +Manifest always keeps metadata about each request: model, provider, tier, token counts, cost, latency, and an error message. Error text is scrubbed for secrets before it lands, and the request headers Manifest stores drop credentials and IP addresses. The usage and cost views in the dashboard are built from that metadata alone. See [Observability](/llm-gateway/docs/observability) for what the record looks like in practice. + +Message bodies are separate, and optional. With [request logs](/llm-gateway/docs/request-logs) on for a harness, Manifest also stores the request and response bodies of each provider attempt so you can read them back in the dashboard. New harnesses have it enabled; to make sure a harness records its logs, check its **Settings → Logs**. + +A stored body is exactly what went over the wire: every message, system instruction, tool definition and inline image, at full length. Manifest doesn't redact or truncate it, and there is no size limit. Stored bodies are held outside the database, in the object storage your instance is configured for, and deleted on a retention schedule. With logs off for a harness, that harness's prompts and completions stay between you and the provider. + + + If a harness handles data you'd rather Manifest never hold onto, switch + **Enable logs** off in its **Settings → Logs** section. Metadata collection + isn't affected either way. + + +## Anonymous telemetry + +Once a day, each self-hosted install sends us a small anonymous report. That's how we know whether anyone's actually using the thing, and which providers are popular enough to deserve more work. It's aggregates, never content: no prompts, no messages, no keys, nothing tied to a user. Fifteen fields total. + + + This section applies to self-hosted installs only. Manifest Cloud has no + separate telemetry report — your usage is already in your own dashboard. + + +### What gets sent + +| Field | Example | Purpose | +|-------|---------|---------| +| `schema_version` | `1` | So the shape can grow without breaking old clients. Stays `1` for additive changes; bumps on breaking ones | +| `install_id` | random UUIDv4 | Count distinct installs. Generated once on first boot, persisted, never rotated | +| `manifest_version` | `5.47.0` | Version adoption across the fleet | +| `messages_total` | `1284` | Daily activity per install. One count per provider attempt, so a request that fell back twice counts three. Requests Manifest blocked count too | +| `messages_by_provider` | `{"anthropic": 700, "openai": 500}` | Provider mix. Anything we don't recognize collapses to `"custom"`, so self-hosted provider names and URLs stay local. Activity with no provider, like a blocked request, lands in `"unknown"` | +| `messages_by_tier` | `{"default": 900, "simple": 300, ...}` | Routing tier usage. A tier outside the known set collapses to `"other"`, and activity with no tier lands in `"unknown"` | +| `messages_by_auth_type` | `{"api_key": 1200, "subscription": 84}` | API key, paid subscription or local usage. Anything else collapses to `"other"`, and activity with no auth type lands in `"unknown"` | +| `tokens_input_total` | `1_450_000` | Volume-weighted signal | +| `tokens_output_total` | `890_000` | Same | +| `cost_usd_total` † | `47.83` | Sum of `cost_usd` Manifest computed at routing time, rounded to cents. Lets us see real dollar throughput instead of guessing from token counts. `0` for Ollama-only / free-API installs | +| `cost_usd_by_provider` † | `{"anthropic": 30.50, "openai": 17.33}` | Per-provider split of `cost_usd_total`, rounded to cents. Same `"custom"` collapse rule as `messages_by_provider` — admin-configured BYOK pricing is never keyed by the raw provider name | +| `agents_total` | `4` | Configuration scale | +| `agents_by_platform` | `{"openclaw": 3, "hermes": 1}` | Which agent clients people use. A harness whose platform is `other` is keyed by its category instead: `personal:other`, `app:other` or `coding:other` | +| `platform` | `linux` / `darwin` / `windows` | OS distribution | +| `arch` | `x64` / `arm64` | Architecture distribution | + +† *Optional. Installs running older Manifest versions omit these fields; receivers should feature-detect on presence rather than on `schema_version`. Cost values are derived from the same `input_tokens` / `output_tokens` we already ship, multiplied by Manifest's per-model pricing table — no new data leaves the box, just a rolled-up dollar figure for what's already disclosed.* + +### Never sent + +Tenant IDs, user IDs, emails, API keys, prompts, message contents, model names, custom provider URLs, OAuth client IDs, hostnames, raw IPs. The ingest takes a SHA-256 of your IP and throws the original away; we keep the hash so we can rate-limit bad actors without knowing where they actually live. + +### When + +- Once every 24 hours, per install. +- The first report is delayed by a random 0–24h offset, so a fleet of containers rebooted together doesn't all hit the endpoint at the same minute. +- Off by default when `NODE_ENV != production`. Dev machines are never going to accidentally send. +- If the endpoint is down, we log it and try again on the next hourly tick. Your proxy keeps serving requests — the sender never gets in the way. + +### Turning it off + +Put this in your `.env` (or `docker-compose.yml`) and restart the container: + +```bash +MANIFEST_TELEMETRY_DISABLED=1 +``` + +The sender checks the flag before doing anything else. No database read, no DNS lookup, no request leaves the box. + +### Sending it somewhere else + +If you'd rather run your own fleet dashboard, point `TELEMETRY_ENDPOINT` at a URL you control. Add it to `.env` next to your compose file and restart: + +```bash +TELEMETRY_ENDPOINT=https://telemetry.mycompany.internal/v1/report +``` + +```bash +docker compose up -d +``` + +Your collector receives the same payload documented above, and nothing is sent to `telemetry.manifest.build`. + +## Related + +- [Observability](/llm-gateway/docs/observability) — what the recorded metadata gets you +- [Environment variables](/llm-gateway/docs/reference/environment-variables) — `MANIFEST_TELEMETRY_DISABLED` and `TELEMETRY_ENDPOINT` in context +- [Self-hosting with Docker](/llm-gateway/docs/self-hosted) — where to put these variables diff --git a/llm-gateway/docs/request-logs.mdx b/llm-gateway/docs/request-logs.mdx new file mode 100644 index 0000000..66c9441 --- /dev/null +++ b/llm-gateway/docs/request-logs.mdx @@ -0,0 +1,146 @@ +--- +title: "Request logs" +description: "Store the full request and response body of every provider attempt, and read them back as a conversation from the Requests page. One toggle per harness." +icon: "logs" +keywords: + ["request logs", "message recording", "request body", "response body", "LLM conversation log", "prompt logging", "full body log", "tool calls", "log retention", "S3 log storage"] +canonical: "https://manifest.build/llm-gateway/docs/request-logs" +--- + +
+ + +
+ +The [Requests page](/llm-gateway/docs/observability) tells you what a request cost and which model served it. Request logs tell you what was actually said: the body Manifest sent to the provider, and the body that came back. The dashboard calls this feature **Logs**. + +It's one toggle per harness. + +## What gets recorded + +Recording happens per **provider attempt**, not per request. A request that falls back twice before succeeding produces three recordings, each with its own body pair, so you can see how the conversation was reshaped for each provider. An [Autofix](/llm-gateway/docs/autofix) retry is its own attempt too, which is how you compare the request that failed against the patched one that worked. + +| | What's stored | +|---|---| +| **Request** | The body Manifest forwarded to the provider, after routing rewrote the model and translated the protocol. Not the body your client sent | +| **Response** | The parsed JSON body, or for a streamed attempt, the raw SSE stream as it arrived | +| **Wire format** | Which protocol the exchange used, so the drawer can render it correctly | + +A request Manifest blocked itself, on a [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) or a malformed body, still lists one attempt, labelled with the model you asked for or **No provider**. It carries the failure status and no **Messages**, **Tools**, or **Raw** tab: Manifest sent nothing to a provider, so there is nothing to read back. + +## Reading it back + +Open a request in the dashboard's **Requests** page and pick an attempt in the **Attempts** list. The initial call, each fallback, and an [Autofix](/llm-gateway/docs/autofix) retry are separate attempts, and each one shows its own conversation. The **Messages**, **Tools**, and **Raw** tabs appear when that attempt has a recording. + + + A request opened on the Requests page, showing the Attempts list and the per-attempt tabs + + +The **Messages** tab renders the exchange as a conversation, with a rail down the side listing every turn. Search the rail or filter it by role, and click any row to jump to that turn in the main pane. Assistant turns show the tool calls the model actually made, not the tool definitions your client offered it. + + + A request's Messages tab showing the turn rail with role filter pills and the conversation cards, including a tool call and its result + + +## Turning it on + +The switch is **Enable logs**, in the **Logs** section of each harness's **Settings** page. + +New harnesses have it on. To make sure a harness records its logs, check **Settings → Logs**. + + + A harness Settings page with the Logs section and the Enable logs switch on + + + + Recording stores your prompts and completions. Everything else Manifest keeps + is metadata ([Data and telemetry](/llm-gateway/docs/reference/telemetry) covers the + distinction). If a harness handles data you'd rather Manifest never hold onto, + leave recording off for that harness. + + +## Retention + +Recordings are deleted on a schedule. The metadata row in the request log stays; only the stored body pair goes away. + +| Deployment | Retention | +|---|---| +| Cloud Free | 7 days | +| Cloud Pro | 365 days | +| Self-hosted | 365 days | + +Cloud Pro keeps 365 days only while the subscription is active or in a trial. A canceled workspace, or one whose payment failed, drops to 7 days, and the nightly cleanup applies that on its next run, deleting everything older than 7 days. + +Self-hosted installs can override this with `REQUEST_RECORDING_RETENTION_DAYS`. Setting it also overrides the per-plan split, so every recording on the instance expires on the same schedule. + +## Self-hosted storage + +
+ + + Logs were reintroduced in **v6.18.0**. On an older self-hosted install, + [upgrade Manifest](/llm-gateway/docs/self-hosted#upgrading) first. + + +
+ +Recordings don't live in Postgres. They're gzipped and written to object storage, which you pick per instance. + + + + The bundled compose file mounts a named volume `manifest_request_recordings` + at `/data/request-recordings` and points Manifest at it. Nothing to + configure when your compose file includes that volume; an install from + before v6.18.0 needs [one upgrade step](/llm-gateway/docs/self-hosted#upgrading) first. The + volume survives `docker compose down` the same way the Postgres volume + does. + + + Set a bucket and region, and Manifest writes there instead. Required for + multi-replica deploys, since replicas can't share a local disk, and for any + platform without a persistent volume. + + ```bash + REQUEST_RECORDING_S3_BUCKET=manifest-recordings + REQUEST_RECORDING_S3_REGION=auto + REQUEST_RECORDING_S3_ENDPOINT=https://.r2.cloudflarestorage.com + REQUEST_RECORDING_S3_ACCESS_KEY_ID=... + REQUEST_RECORDING_S3_SECRET_ACCESS_KEY=... + ``` + + Credentials are optional if the instance already has them from its + environment (an IAM role, for example). Provide both or neither. + + + +`REQUEST_RECORDING_STORAGE` defaults to `auto`, which picks the backend by what you've configured: any S3 setting present means S3, otherwise the mounted filesystem path. Set it to `s3`, `filesystem`, or `disabled` to decide explicitly. + + + Under `auto`, setting *some* S3 variables commits Manifest to S3. A bucket + without a region, or one access-key half without the other, leaves recording + with no working backend instead of falling back to local disk. Configure it + fully or not at all. + + +Full variable list: [Environment variables](/llm-gateway/docs/reference/environment-variables#request-recordings). + +### Check it's working + +A storage problem never stops traffic, and the dashboard never mentions it. Flipping **Enable logs** on still confirms the change, and attempts come back without **Messages**, **Tools**, or **Raw** tabs, exactly as if logs were off. The server output is the only place the problem shows. + +```bash +docker compose logs manifest | grep "Request recording storage" +``` + +A line reading `Request recording storage unavailable: ` means Manifest is storing nothing, and the reason names what to fix. No such line means the backend resolved. + +### Platforms without a persistent disk + +The one-click templates set this up for you. Render mounts a persistent disk. Railway, Fly, AWS, and Google Cloud provision object storage during deploy. Heroku, Koyeb, and DigitalOcean collect your bucket and keys as part of template setup. On any other platform with an ephemeral filesystem, point recording at S3-compatible storage yourself, or recordings only survive until the next deploy. + +## Related + +- [Observability](/llm-gateway/docs/observability) — the request log these recordings hang off +- [Autofix](/llm-gateway/docs/autofix) — comparing a failed request against its patched retry +- [Data and telemetry](/llm-gateway/docs/reference/telemetry) — what Manifest stores by default +- [Environment variables](/llm-gateway/docs/reference/environment-variables#request-recordings) diff --git a/llm-gateway/docs/self-hosted.mdx b/llm-gateway/docs/self-hosted.mdx new file mode 100644 index 0000000..5033921 --- /dev/null +++ b/llm-gateway/docs/self-hosted.mdx @@ -0,0 +1,367 @@ +--- +title: "Self-hosting with Docker" +sidebarTitle: "Docker" +description: "Run Manifest on your own machine with Docker. Covers the quick installer, docker-compose, bringing your own PostgreSQL, signed images, upgrades, and backups." +icon: "docker" +keywords: + ["self-hosted LLM router", "Docker", "docker-compose", "PostgreSQL", "BETTER_AUTH_SECRET", "docker run", "upgrade", "backup"] +canonical: "https://manifest.build/llm-gateway/docs/self-hosted" +--- + +Run the full Manifest stack on your own machine. No Node.js required, just Docker. + +To run Manifest somewhere other than your own machine, see the [other self-hosting paths](/llm-gateway/docs/deploy). + +All three paths end in the same place: a running stack at [http://localhost:2099](http://localhost:2099). On first access, Manifest takes you to a setup screen where you create the admin account. No demo credentials are pre-seeded. + + + The bundled compose file binds port 2099 to `127.0.0.1` only, so the dashboard is reachable on the host machine but not over the LAN. See [Exposing on the LAN](#exposing-on-the-lan) to change this. + + +## Installation + + + + One command. The installer downloads the compose file into `~/manifest`, generates the secrets, and brings up the stack. First boot pulls the app image and Postgres, so give it up to a couple of minutes. + + ```bash + bash <(curl -sSL https://raw.githubusercontent.com/mnfst/manifest/main/docker/install.sh) + ``` + + + ```bash + curl -sSLO https://raw.githubusercontent.com/mnfst/manifest/main/docker/install.sh + less install.sh + bash install.sh + ``` + + + Useful flags: `--dir ` to install elsewhere, `--port ` to serve on a port other than 2099, `--dry-run` to preview, `--yes` to skip the confirmation prompt. + + Re-running the installer against an existing install directory resumes it. The compose file and your generated secrets are left untouched. + + When the installer finishes, open [http://localhost:2099](http://localhost:2099): a setup screen asks you to create the admin account. Then connect a provider and send your first request — see [First request](#first-request). + + + Same underlying flow as the install script, but you drive it yourself so you can edit the config before booting the stack. + + + + ```bash + curl -O https://raw.githubusercontent.com/mnfst/manifest/main/docker/docker-compose.yml + curl -O https://raw.githubusercontent.com/mnfst/manifest/main/docker/.env.example + cp .env.example .env + ``` + + + Open `.env` in your editor and set `BETTER_AUTH_SECRET` to a random string. You can generate one with: + + ```bash + openssl rand -hex 32 + ``` + + Optional: to use a stronger database password, set both `POSTGRES_PASSWORD` and `DATABASE_URL` in `.env` — they must agree, and any special characters in the password need to be percent-encoded in the URL. + + + ```bash + docker compose up -d + ``` + + Give it up to a couple of minutes on a cold pull — you can watch startup with `docker compose logs -f manifest`. + + + Go to [http://localhost:2099](http://localhost:2099): a setup screen asks you to create the admin account. + + + Connect a provider and send your first request — see [First request](#first-request). + + + + + Before exposing this instance beyond localhost, double-check that `BETTER_AUTH_SECRET` is a real random value, and if you enable email verification, set `BETTER_AUTH_URL` to a reachable public URL so the verification links resolve. + + + + If you already have a PostgreSQL instance, replace `user`, `pass`, and `host` with your actual database credentials: + + ```bash + docker run -d \ + -p 2099:2099 \ + -e PORT=2099 \ + -e DATABASE_URL=postgresql://user:pass@host:5432/manifest \ + -e BETTER_AUTH_SECRET=$(openssl rand -hex 32) \ + -e BETTER_AUTH_URL=http://localhost:2099 \ + manifestdotbuild/manifest + ``` + + + ```powershell + $secret = -join ((48..57 + 97..122) | Get-Random -Count 64 | ForEach-Object { [char]$_ }) + + docker run -d ` + -p 2099:2099 ` + -e PORT=2099 ` + -e DATABASE_URL=postgresql://user:pass@host:5432/manifest ` + -e BETTER_AUTH_SECRET=$secret ` + -e BETTER_AUTH_URL=http://localhost:2099 ` + manifestdotbuild/manifest + ``` + + + + Generate a 64-character hex secret with any tool you trust, then: + + ```cmd + docker run -d ^ + -p 2099:2099 ^ + -e PORT=2099 ^ + -e DATABASE_URL=postgresql://user:pass@host:5432/manifest ^ + -e BETTER_AUTH_SECRET= ^ + -e BETTER_AUTH_URL=http://localhost:2099 ^ + manifestdotbuild/manifest + ``` + + + + +## First request + +Signing up leaves you with an empty instance. Three steps to a routed request. + + + + In the dashboard sidebar, open **Providers** and pick how you want to connect: + + - **Usage-based** — paste an API key (OpenAI, Anthropic, Gemini, DeepSeek, …) + - **Subscriptions** — reuse a plan you already pay for (ChatGPT, Claude, GLM Coding Plan, …) + - **Local** — Ollama, LM Studio, or llama.cpp running on the host + + Manifest discovers the available models as soon as the connection is saved. + + + Every harness has its own key, shown when you create it and again under the harness's **Settings**. It starts with `mnfst_`. + + + The endpoint is OpenAI-compatible, so any SDK or agent that accepts a base URL works — point it at `http://localhost:2099/v1` with the `mnfst_` key. To check it end to end: + + ```bash + curl -X POST http://localhost:2099/v1/chat/completions \ + -H "Authorization: Bearer mnfst_YOUR_KEY_HERE" \ + -H "Content-Type: application/json" \ + -d '{"model": "auto", "messages": [{"role": "user", "content": "Hello"}]}' + ``` + + `"model": "auto"` asks Manifest to route the request. Any other name is treated as an explicit choice: if the model isn't available to your harness, the request fails with [M302](/llm-gateway/docs/errors/M302). Send `auto` to use routing. + + + +The request shows up in the dashboard straight away, with the model that served it and what it cost. + + + Errors raised by Manifest itself carry an `M###` code, a plain-English cause, and a link to the matching page under [manifest.build/docs/errors](/llm-gateway/docs/errors). The three you are most likely to see on a fresh install: + + | Code | Means | + | --- | --- | + | `M003` | The token isn't a Manifest key — it doesn't start with `mnfst_` | + | `M005` | Well-formed key, but this instance doesn't know it. Copying the literal `mnfst_YOUR_KEY_HERE` above gets you this — replace it with the real key | + | `M101` | The key is fine; no provider is connected yet | + | `M100` | Routing picked a provider that has no API key on file | + + +## Verifying the image signature + +Published images are signed with cosign keyless signing (Sigstore). Verify before pulling: + +```bash +cosign verify manifestdotbuild/manifest: \ + --certificate-identity-regexp="^https://github.com/mnfst/manifest/" \ + --certificate-oidc-issuer="https://token.actions.githubusercontent.com" +``` + +## Custom port + +If port 2099 is taken, set `PORT` in `.env`. That is the whole change: + +```bash +PORT=8080 +``` + +The compose file reads `${PORT:-2099}` for both the published host port and the backend's internal listener, and `BETTER_AUTH_URL` defaults to `http://localhost:${PORT:-2099}` — so one line covers all three, with no YAML edit. The install script writes it for you if you pass `--port 8080`. + +For a `docker run` install there is no `.env`, so pass the mapping and the URL explicitly. The container keeps listening on 2099 and Docker remaps it: + +```bash +docker run -d \ + -p 8080:2099 \ + -e PORT=2099 \ + -e BETTER_AUTH_URL=http://localhost:8080 \ + ... +``` + +If you see an "Invalid origin" error on the login page, `BETTER_AUTH_URL` doesn't match the URL you're accessing the dashboard on. The host matters as much as the port. + + + **Upgrading from a pre-2099 install?** Your existing stack keeps running on port 3001 with no changes — the backend's own fallback is still `3001`, so the new image works against your old compose file. If you want to refresh your compose file but stay on the legacy port (to avoid reconfiguring OAuth callbacks, reverse proxies, or bookmarks), set `PORT=3001` in `.env` and the bundled compose file will honour it for both the host binding and the internal listener. + + +## Exposing on the LAN + +By default the compose file binds port 2099 to `127.0.0.1` only. The dashboard is reachable from the host but not from other machines on the network. To expose it on the LAN: + + + + In `.env`, set: + + ```bash + HOST_BIND_ADDRESS=0.0.0.0 + ``` + + Editing `docker-compose.yml` by hand does not survive an upgrade; `.env` does. If you exposed the LAN by editing the `ports:` line in `docker-compose.yml`, move that change to `HOST_BIND_ADDRESS` here: `bash install.sh --upgrade` replaces the compose file and your LAN access goes with it. + + + In `.env`, set `BETTER_AUTH_URL` to the host you'll reach the dashboard on, e.g. `http://192.168.1.20:2099` or `https://manifest.mydomain.com`. This must match the URL in the browser or Better Auth will reject the login with "Invalid origin". If you sign in with Google, GitHub or Discord, update the redirect URI in the provider's console to `${BETTER_AUTH_URL}/api/auth/callback/`, or the provider rejects the login. + + + ```bash + docker compose up -d + ``` + + + +You can reach the setup screen from another machine while `BETTER_AUTH_URL` still points at localhost. The admin account is created, then sign-in fails with "Invalid origin" and drops you back on the login page. The setup screen does not come back, because the account now exists. That account is fine. Set `BETTER_AUTH_URL` to the URL you type in the browser, run `docker compose up -d`, then sign in with it. + +## Image tags + +Every release is published with the following tags: + +| Tag | Example | Description | +|-----|---------|-------------| +| `major.minor.patch` | `6.18.0` | Fully pinned | +| `major.minor` | `6.18` | Latest patch within a minor | +| `major` | `6` | Latest minor+patch within a major | +| `latest` | — | Latest stable release | +| `sha-` | — | Exact commit for rollback | + +Images are built for both `linux/amd64` and `linux/arm64`. + +## Upgrading + +Manifest ships a new image on every release. To upgrade an existing compose install: + +```bash +docker compose pull +docker compose up -d +``` + +Database migrations run automatically on boot, no manual steps. Your data in the `pgdata` volume is preserved across upgrades. To control when major upgrades happen, pin the image tag with `MANIFEST_VERSION` in `.env`, for example `MANIFEST_VERSION=6`, then run the two commands above. Unset, the compose file pulls `latest`. Pin here rather than in `docker-compose.yml`, which `bash install.sh --upgrade` replaces. + +The two commands above only pull new images. Your `docker-compose.yml` stays as it is, hand edits included. `bash install.sh --upgrade` is the other path: it downloads a fresh `docker-compose.yml` and replaces yours, after copying the old one next to it as `docker-compose.yml.backup.`. Your `.env` is left alone. Keep custom compose changes in `docker-compose.override.yml`, which the installer never downloads. + + + **Upgrading from before v6.18.0 and using [logs](/llm-gateway/docs/request-logs)?** Run the installer once with the upgrade flag, or log bodies are written inside the container and lost when it is recreated: + + ```bash + curl -sSLO https://raw.githubusercontent.com/mnfst/manifest/main/docker/install.sh + bash install.sh --upgrade + ``` + + (`--upgrade` looks for the install in `~/manifest`. Installed elsewhere? Pass `--dir /your/path`, otherwise the script stops with "No installation found at ...".) + + It adds the `manifest_request_recordings` volume to your compose file. Later releases include it from the start. + + + Declare the named volume and mount it on the backend service: + + ```yaml + services: + manifest: + volumes: + - manifest_request_recordings:/data/request-recordings + + volumes: + manifest_request_recordings: + name: manifest_request_recordings + ``` + + + +## Backup and persistence + +The stack uses two named volumes: + +| Volume | Mounted at | Holds | +|---|---|---| +| `manifest_pgdata` | `/var/lib/postgresql/data` in `postgres` | Everything: accounts, harnesses, provider credentials, the request log | +| `manifest_request_recordings` | `/data/request-recordings` in `manifest` | Stored [request logs](/llm-gateway/docs/request-logs) bodies | + +The database volume is the one to back up. Losing the recordings volume costs you stored message bodies and nothing else, and those expire on a retention schedule regardless. + +Back up (from the host, with the stack running): + +```bash +docker compose exec -T postgres pg_dump -U manifest manifest > manifest-backup-$(date +%F).sql +``` + +Restore into a fresh stack: + +```bash +docker compose up -d postgres +cat manifest-backup.sql | docker compose exec -T postgres psql -U manifest manifest +docker compose up -d +``` + +To list or remove the volume manually: + +```bash +docker volume ls | grep manifest_ +docker compose down -v # destroys all data +``` + +## Environment variables + +**Core** + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `DATABASE_URL` | Yes | — | PostgreSQL connection string | +| `BETTER_AUTH_SECRET` | Yes | — | Session signing secret (min 32 chars) | +| `MANIFEST_ENCRYPTION_KEY` | Recommended | falls back to `BETTER_AUTH_SECRET` | Separate 32+ char key encrypting stored provider keys and OAuth tokens | +| `BETTER_AUTH_URL` | No | `http://localhost:${PORT}` | Public URL. Must match the URL in your browser | +| `PORT` | No | `2099` | Dashboard port — sets the published host port and the internal listener | +| `HOST_BIND_ADDRESS` | No | `127.0.0.1` | Host interface the dashboard port binds to. Set `0.0.0.0` to reach it over the LAN | +| `MANIFEST_DISABLE_HSTS` | No | unset | Set `1` to silence the boot warning about serving over plain HTTP | + +The first two are required and the installer generates both, so a default install boots without you setting anything. + + + The installer also generates `MANIFEST_ENCRYPTION_KEY`. If you are installing by hand, set it too. Left unset, Manifest falls back to `BETTER_AUTH_SECRET` for at-rest encryption and warns on every boot — meaning one leaked session-signing secret also decrypts every stored provider key and OAuth token. Set it **before first boot**: introducing it later means re-encrypting what is already in the database. + + +Everything else is optional: provider timeouts, email delivery for alerts and password resets, OAuth logins, connection-pool sizing, recording storage and retention, Autofix, and Sentry. See [Environment variables](/llm-gateway/docs/reference/environment-variables) for the full list with defaults. + + + `NODE_ENV` and `SEED_DATA` are fixed by the bundled compose file and are not knobs for a self-hosted install. The image is a production artifact, and the demo-data seeder refuses to run under `NODE_ENV=production` whatever `SEED_DATA` says — use the first-run setup wizard to create your admin account. + + `BIND_ADDRESS` is likewise set by the image (`0.0.0.0`, so the container is reachable through Docker's port mapping); control host exposure with `HOST_BIND_ADDRESS` in `.env`, not this variable. + + +## Stop and clean up + +```bash +docker compose down # Stop services (keeps data) +docker compose down -v # Stop and delete all data +``` + +## Data and privacy + +Manifest always keeps metadata about each request — model, provider, tier, token counts, cost, latency. Message bodies are separate and optional: with [request logs](/llm-gateway/docs/request-logs) on for a harness, prompts and completions are stored too, in the recordings volume rather than the database. New harnesses have it enabled. See [Data and telemetry](/llm-gateway/docs/reference/telemetry). + +## Telemetry + +Once a day, each install sends an anonymous aggregate report: version, provider mix, token and cost totals. Never prompts, keys, or anything tied to a user. Set `MANIFEST_TELEMETRY_DISABLED=1` in your `.env` to turn it off. + +Full field list, what's never sent, and how to point it at your own endpoint: [Data and telemetry](/llm-gateway/docs/reference/telemetry). + +## Docker Hub + +The image is available at [manifestdotbuild/manifest](https://hub.docker.com/r/manifestdotbuild/manifest) on Docker Hub. diff --git a/observability.mdx b/observability.mdx index a173af5..f2ceac7 100644 --- a/observability.mdx +++ b/observability.mdx @@ -4,6 +4,7 @@ description: "See what every harness spent, which models served it, and what fai icon: "activity" keywords: ["LLM observability", "AI cost tracking", "token usage", "spend alerts", "request log", "LLM analytics", "cost per agent", "cost per harness", "provider errors"] +canonical: "https://manifest.build/llm-gateway/docs/observability" ---
@@ -20,9 +21,9 @@ Manifest tracks two things, and the difference matters once fallback is involved | | What it is | When you get more than one | |---|---|---| | **Request** | One call from your agent to Manifest | Never. One call in, one Request. | -| **Provider attempt** | One call from Manifest to a provider | Every fallback step and every [Autofix](/autofix) retry adds another | +| **Provider attempt** | One call from Manifest to a provider | Every fallback step and every [Autofix](/llm-gateway/docs/autofix) retry adds another | -A request that succeeds on the first try is one Request with one attempt. A request that fails twice before succeeding is still one Request, with three attempts. A request Manifest blocked itself, like a [hard limit](/llm-gateway#hard-limits) or a malformed body, is one Request with one attempt that never reached a provider. That attempt is listed under the model you asked for, or **No provider** when the request named none, and carries the error that stopped it. +A request that succeeds on the first try is one Request with one attempt. A request that fails twice before succeeding is still one Request, with three attempts. A request Manifest blocked itself, like a [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) or a malformed body, is one Request with one attempt that never reached a provider. That attempt is listed under the model you asked for, or **No provider** when the request named none, and carries the error that stopped it. This is why the request count and the provider call count don't match, and why cost is attributed to attempts: a failed attempt can still burn tokens. @@ -51,9 +52,9 @@ On the Free plan, the Overview covers up to 7 days of history; the 30-, 90-, and ### What's on each request -Opening a request in the log shows the whole story: the tier that routed it, the model and provider that served it, input and output tokens, computed cost, latency, the auth type used ([API key, subscription, or local](/reference/glossary#auth-type)), and every fallback hop with its own status. Failures carry an error code, so a Manifest rejection like [M100](/errors/M100) reads differently from a provider's own 500. +Opening a request in the log shows the whole story: the tier that routed it, the model and provider that served it, input and output tokens, computed cost, latency, the auth type used ([API key, subscription, or local](/llm-gateway/docs/reference/glossary#auth-type)), and every fallback hop with its own status. Failures carry an error code, so a Manifest rejection like [M100](/llm-gateway/docs/errors/M100) reads differently from a provider's own 500. -That's all metadata. To read the messages themselves, turn on [request logs](/request-logs) for the harness. Open a request, pick an attempt, and its **Messages** tab shows the body sent to the provider and the body that came back. +That's all metadata. To read the messages themselves, turn on [request logs](/llm-gateway/docs/request-logs) for the harness. Open a request, pick an attempt, and its **Messages** tab shows the body sent to the provider and the body that came back. Local models record `cost = 0` with real token counts and latency, since @@ -72,7 +73,7 @@ A limit rule watches one metric for one harness over a rolling period, and acts | **Period** | Hour, day, week, or month | | **Action** | Every rule emails you. Turn on blocking to also stop requests over the threshold | -Blocking is the gateway half of this and is covered in [hard limits](/llm-gateway#hard-limits). The rest of this section is the alerting half. +Blocking is the gateway half of this and is covered in [hard limits](/llm-gateway/docs/llm-gateway#hard-limits). The rest of this section is the alerting half. @@ -105,7 +106,7 @@ On the **Limits** page, under **Configure email provider**, pick Resend, Mailgun Your workspace holds one provider, shared by every harness. Removing it stops alerts for all of them. -Alerts use the saved provider first, and fall back to the [`EMAIL_*` environment variables](/reference/environment-variables) when no provider is saved. Those variables also send verification and password-reset email, which the saved provider never handles. +Alerts use the saved provider first, and fall back to the [`EMAIL_*` environment variables](/llm-gateway/docs/reference/environment-variables) when no provider is saved. Those variables also send verification and password-reset email, which the saved provider never handles. Without an email provider from either path, alerts are skipped. Blocking rules still work, since they don't need to send anything. @@ -131,14 +132,14 @@ Self-hosted installs swap the base URL for their own host. ## Anonymous telemetry -Separately from your dashboard, each self-hosted install sends a small daily report so the project can see version adoption and which providers deserve attention. It's aggregates only, never prompts or content, and one environment variable turns it off. Full field list and opt-out: [Data and telemetry](/reference/telemetry). +Separately from your dashboard, each self-hosted install sends a small daily report so the project can see version adoption and which providers deserve attention. It's aggregates only, never prompts or content, and one environment variable turns it off. Full field list and opt-out: [Data and telemetry](/llm-gateway/docs/reference/telemetry).
## Related -- [LLM Gateway](/llm-gateway) — routing, fallback, and hard limits -- [Request logs](/request-logs) — the request and response bodies -- [Autofix](/autofix) -- [Error codes](/errors) -- [Glossary](/reference/glossary) +- [LLM Gateway](/llm-gateway/docs/llm-gateway) — routing, fallback, and hard limits +- [Request logs](/llm-gateway/docs/request-logs) — the request and response bodies +- [Autofix](/llm-gateway/docs/autofix) +- [Error codes](/llm-gateway/docs/errors) +- [Glossary](/llm-gateway/docs/reference/glossary) diff --git a/providers/api-key-providers.mdx b/providers/api-key-providers.mdx index df5c61e..165f081 100644 --- a/providers/api-key-providers.mdx +++ b/providers/api-key-providers.mdx @@ -5,6 +5,7 @@ description: "Bring your own API key for OpenAI, Anthropic, Google, xAI, DeepSee icon: "key" keywords: ["OpenAI", "Anthropic", "Google Gemini", "xAI", "DeepSeek", "Mistral", "Qwen", "Moonshot", "MiniMax", "Z.ai", "OpenRouter", "Groq", "Cerebras", "Fireworks", "AWS Bedrock", "NVIDIA NIM", "Xiaomi MiMo", "Hugging Face", "Gemini Free", "API key"] +canonical: "https://manifest.build/llm-gateway/docs/providers/api-key-providers" --- Most providers work the standard way: sign up, generate an API key, paste it into Manifest. Routed requests go out with that key as the credential. @@ -46,7 +47,7 @@ Gemini Free is a managed tile. Manifest provisions the credential through its ow BytePlus, NousResearch, Command Code, ClinePass, Kiro, GitHub Copilot, Ollama Cloud, and OpenCode Go connect through a plan rather than an API key, so they have no API Keys tab. See - [Subscription-based providers](/providers/subscription-based-providers). + [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers). ## Add a provider diff --git a/providers/custom-providers.mdx b/providers/custom-providers.mdx index fa177f9..14e2499 100644 --- a/providers/custom-providers.mdx +++ b/providers/custom-providers.mdx @@ -5,6 +5,7 @@ description: "Plug in any OpenAI- or Anthropic-compatible endpoint, including vL icon: "plug" keywords: ["vLLM", "TGI", "text-generation-inference", "LocalAI", "Xinference", "OpenLLM", "OpenAI-compatible", "Anthropic-compatible", "custom provider"] +canonical: "https://manifest.build/llm-gateway/docs/providers/custom-providers" --- If your endpoint speaks OpenAI or Anthropic, Manifest can route to it. Useful for self-hosted inference servers (vLLM, TGI, LocalAI), internal endpoints behind your VPN, or providers that aren't on the built-in list yet. @@ -62,4 +63,4 @@ User-supplied URLs are revalidated on every request to defend against SSRF. On M ## Cost tracking -Manifest can't infer pricing for unknown models. Custom-provider requests show up in the dashboard with `cost = 0` and `model = `. Token counts and latency are still recorded, so [hard limits](/llm-gateway#hard-limits) on token volume still work. +Manifest can't infer pricing for unknown models. Custom-provider requests show up in the dashboard with `cost = 0` and `model = `. Token counts and latency are still recorded, so [hard limits](/llm-gateway/docs/llm-gateway#hard-limits) on token volume still work. diff --git a/providers/local-models.mdx b/providers/local-models.mdx index 98b6cd1..da4714c 100644 --- a/providers/local-models.mdx +++ b/providers/local-models.mdx @@ -4,12 +4,13 @@ description: "Run any GGUF model on your own hardware with Ollama, LM Studio, or icon: "server" keywords: ["Ollama", "LM Studio", "llama.cpp", "GGUF", "local LLM", "self-hosted inference", "host.docker.internal"] +canonical: "https://manifest.build/llm-gateway/docs/providers/local-models" --- Local model providers run entirely on your own hardware. Manifest detects the running server, fetches the model list, and routes requests to `http://localhost:` like any other provider. No API key, no network egress, no per-token cost. - The built-in Ollama, LM Studio, and llama.cpp connections are self-hosted only. To reach a local model from [Manifest Cloud](https://app.manifest.build), expose the runtime on a public URL or tunnel and add it as a [custom provider](/providers/custom-providers). + The built-in Ollama, LM Studio, and llama.cpp connections are self-hosted only. To reach a local model from [Manifest Cloud](https://app.manifest.build), expose the runtime on a public URL or tunnel and add it as a [custom provider](/llm-gateway/docs/providers/custom-providers). ## Supported runtimes diff --git a/providers/subscription-based-providers.mdx b/providers/subscription-based-providers.mdx index 01af1c8..4992a58 100644 --- a/providers/subscription-based-providers.mdx +++ b/providers/subscription-based-providers.mdx @@ -5,6 +5,7 @@ description: "Reuse a paid plan you already have: ChatGPT, Claude, Gemini, Grok, icon: "credit-card" keywords: ["ChatGPT Plus", "Claude Max", "GitHub Copilot", "GLM Coding Plan", "Mistral Vibe", "Gemini", "Grok", "Kimi", "OAuth", "device code"] +canonical: "https://manifest.build/llm-gateway/docs/providers/subscription-based-providers" --- If you already pay for ChatGPT Plus, Claude Max, GitHub Copilot, or one of the other plans listed below, Manifest can route through the subscription instead of an API key. Auth is OAuth, a device code, or a pasted subscription token, depending on the provider. @@ -96,11 +97,11 @@ A model your plan just gained doesn't show up on its own. Click **Refresh models Subscription auth is rate-limited by the provider's plan, not by Manifest. If you hit a plan ceiling, requests fall through to your next - [fallback](/llm-gateway#fallback) model, usually a different provider on an API key. + [fallback](/llm-gateway/docs/llm-gateway#fallback) model, usually a different provider on an API key. ## Why mix subscriptions and API keys A common setup: subscription as the primary (predictable monthly cost), API-key provider as the fallback for when you hit the plan limit or want a model the subscription doesn't include. -Pin your subscription model to your default or a custom tier and add API-key models to the [fallback list](/llm-gateway#configuration). Manifest handles the switch. +Pin your subscription model to your default or a custom tier and add API-key models to the [fallback list](/llm-gateway/docs/llm-gateway#configuration). Manifest handles the switch. diff --git a/reference/api.mdx b/reference/api.mdx index 9a158d8..2fb2a71 100644 --- a/reference/api.mdx +++ b/reference/api.mdx @@ -4,6 +4,7 @@ description: "The Manifest proxy speaks both OpenAI and Anthropic. Endpoints, au icon: "code" keywords: ["chat completions", "responses API", "Anthropic messages", "auto", "v1/models", "streaming", "SSE", "fallback exhausted"] +canonical: "https://manifest.build/llm-gateway/docs/reference/api" --- Manifest exposes both OpenAI and Anthropic-format endpoints on one proxy. Point your client at the Manifest URL, send `auto` as the model, and routing picks the real model behind the scenes. @@ -79,7 +80,7 @@ curl http://localhost:2099/v1/models \ -H "Authorization: Bearer mnfst_YOUR_KEY_HERE" ``` -Send `auto` to let Manifest route, or send any listed model ID to skip routing and go straight to that provider. If you send a model ID that no connected provider can serve, Manifest returns [M302: Model not available](/errors/M302). See [Routing → Route a specific model](/llm-gateway#route-a-specific-model). +Send `auto` to let Manifest route, or send any listed model ID to skip routing and go straight to that provider. If you send a model ID that no connected provider can serve, Manifest returns [M302: Model not available](/llm-gateway/docs/errors/M302). See [Routing → Route a specific model](/llm-gateway/docs/llm-gateway#route-a-specific-model). ### Inspect model capabilities @@ -203,18 +204,18 @@ Errors come in two shapes, depending on the caller. A tool or SDK call gets a re | Status | Meaning | |---|---| | `401` | Invalid or missing `Authorization` header | -| `402` | Manifest Free plan quota reached (`error.code = PLAN_LIMIT_REQUESTS`, [M204](/errors/M204)), or provider billing/quota error from the upstream | -| `429` | Manifest rate limit tripped ([M201](/errors/M201), [M202](/errors/M202), [M203](/errors/M203)), or a rate limit the provider itself returned | -| `5xx` | Upstream provider error (triggers [fallback](/llm-gateway#fallback)) | +| `402` | Manifest Free plan quota reached (`error.code = PLAN_LIMIT_REQUESTS`, [M204](/llm-gateway/docs/errors/M204)), or provider billing/quota error from the upstream | +| `429` | Manifest rate limit tripped ([M201](/llm-gateway/docs/errors/M201), [M202](/llm-gateway/docs/errors/M202), [M203](/llm-gateway/docs/errors/M203)), or a rate limit the provider itself returned | +| `5xx` | Upstream provider error (triggers [fallback](/llm-gateway/docs/llm-gateway#fallback)) | -A [hard limit](/llm-gateway#hard-limits) block ([M200](/errors/M200)) is not in this table: it comes back as an HTTP `200` chat completion whose assistant message carries the block text. When the fallback chain is exhausted, the response keeps the primary model's real error status and carries `X-Manifest-Fallback-Exhausted: true`; the body keeps the provider's own error code, or `fallback_exhausted` when there is none. +A [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) block ([M200](/llm-gateway/docs/errors/M200)) is not in this table: it comes back as an HTTP `200` chat completion whose assistant message carries the block text. When the fallback chain is exhausted, the response keeps the primary model's real error status and carries `X-Manifest-Fallback-Exhausted: true`; the body keeps the provider's own error code, or `fallback_exhausted` when there is none. ## Rate limits -The gateway enforces three caps. A workspace can send **200 requests per minute** ([M201](/errors/M201)). An IP can send **500 requests per minute** ([M202](/errors/M202)). A workspace can have **10 requests in flight at once** ([M203](/errors/M203)). The caps are the same on Cloud and self-hosted. No setting changes them. +The gateway enforces three caps. A workspace can send **200 requests per minute** ([M201](/llm-gateway/docs/errors/M201)). An IP can send **500 requests per minute** ([M202](/llm-gateway/docs/errors/M202)). A workspace can have **10 requests in flight at once** ([M203](/llm-gateway/docs/errors/M203)). The caps are the same on Cloud and self-hosted. No setting changes them. -Your Cloud plan sets your monthly request quota ([M204](/errors/M204)). The per-minute caps stay the same on every plan. +Your Cloud plan sets your monthly request quota ([M204](/llm-gateway/docs/errors/M204)). The per-minute caps stay the same on every plan. ## Response headers -Routed responses carry [routing headers](/reference/headers). They tell your client which model and tier handled the request, with no need to parse the response body. A request rejected before routing (bad key, quota, rate limit) carries none. +Routed responses carry [routing headers](/llm-gateway/docs/reference/headers). They tell your client which model and tier handled the request, with no need to parse the response body. A request rejected before routing (bad key, quota, rate limit) carries none. diff --git a/reference/environment-variables.mdx b/reference/environment-variables.mdx index 101cf4f..102a0f7 100644 --- a/reference/environment-variables.mdx +++ b/reference/environment-variables.mdx @@ -4,12 +4,13 @@ description: "Every environment variable Manifest reads at startup, grouped by w icon: "settings" keywords: ["DATABASE_URL", "BETTER_AUTH_SECRET", "BETTER_AUTH_URL", "PORT", "BIND_ADDRESS", "THROTTLE_LIMIT", "EMAIL_PROVIDER", "Resend", "Mailgun", "SendGrid", "MANIFEST_TELEMETRY_DISABLED"] +canonical: "https://manifest.build/llm-gateway/docs/reference/environment-variables" --- Manifest reads its configuration from environment variables. In the bundled Docker setup these come from `~/manifest/.env`. For `docker run`, pass them with `-e`. For non-Docker installs, export them in the shell before launching the backend. - This page is for [self-hosted](/self-hosted) instances only. On [Manifest + This page is for [self-hosted](/llm-gateway/docs/self-hosted) instances only. On [Manifest Cloud](https://app.manifest.build) there is no server to configure and no `.env` to edit — everything you can change lives in the dashboard. @@ -23,7 +24,7 @@ Manifest reads its configuration from environment variables. In the bundled Dock | `BETTER_AUTH_URL` | No | `http://localhost:2099` | Public URL the dashboard is reachable on. Must match the browser URL | | `PORT` | No | `2099` | Dashboard port. Under the bundled compose file this sets both the published host port and the internal listener, and `BETTER_AUTH_URL` follows it | | `NODE_ENV` | No | `production` | Node environment. Telemetry is disabled when this isn't `production`. Fixed to `production` by the bundled compose file — the image is a production artifact | -| `MANIFEST_MODE` | No | — | Marks the install as self-hosted when Manifest can't tell on its own. Accepted value: `selfhosted` (`local` is a legacy alias with the same effect). Unset, Manifest detects Docker, Podman, and Kubernetes automatically; some platforms hide the container, which is why the [Render](/deploy/render), [Koyeb](/deploy/koyeb), [Easypanel](/deploy/easypanel), and [AWS](/deploy/aws) guides set it | +| `MANIFEST_MODE` | No | — | Marks the install as self-hosted when Manifest can't tell on its own. Accepted value: `selfhosted` (`local` is a legacy alias with the same effect). Unset, Manifest detects Docker, Podman, and Kubernetes automatically; some platforms hide the container, which is why the [Render](/llm-gateway/docs/deploy/render), [Koyeb](/llm-gateway/docs/deploy/koyeb), [Easypanel](/llm-gateway/docs/deploy/easypanel), and [AWS](/llm-gateway/docs/deploy/aws) guides set it | ## Network & security @@ -37,7 +38,7 @@ Manifest reads its configuration from environment variables. In the bundled Dock | `MANIFEST_DISABLE_HSTS` | — | Set `1` to silence the boot warning about HSTS being unavailable on a plain-HTTP deployment. Prefer an `https://` `BETTER_AUTH_URL` anywhere reachable from the internet | | `WINGMAN_CORS_ORIGINS` | — | Extra browser origins allowed to call the gateway, comma-separated. Useful when you host your own copy of [Wingman](https://github.com/mnfst/wingman), the open-source gateway tester; the hosted one at `wingman.manifest.build` is always allowed | | `THROTTLE_TTL` | `60000` | Window (ms) for the rate limit on the dashboard's own API calls. Does not apply to gateway traffic on `/v1` | -| `THROTTLE_LIMIT` | `100` | Max dashboard API calls per window. The gateway has its own fixed caps, see [Rate limits](/reference/api#rate-limits) | +| `THROTTLE_LIMIT` | `100` | Max dashboard API calls per window. The gateway has its own fixed caps, see [Rate limits](/llm-gateway/docs/reference/api#rate-limits) | ## Database @@ -55,13 +56,13 @@ How Manifest talks to upstream providers. | Variable | Default | Description | |---|---|---| -| `PROVIDER_TIMEOUT_MS` | `180000` | Per-attempt timeout (ms) for upstream requests. A hung provider surfaces as a synthetic `504` and triggers the next [fallback](/llm-gateway#fallback). Set it strictly below your client's own timeout; slow local models may need it raised | +| `PROVIDER_TIMEOUT_MS` | `180000` | Per-attempt timeout (ms) for upstream requests. A hung provider surfaces as a synthetic `504` and triggers the next [fallback](/llm-gateway/docs/llm-gateway#fallback). Set it strictly below your client's own timeout; slow local models may need it raised | | `STREAM_WARMUP_MS` | `15000` | How long to wait for the first streamed chunk before falling back to the next model | | `OLLAMA_HOST` | `http://localhost:11434` | Base URL for a local Ollama server. In Docker, set it to `http://host.docker.internal:11434` | ## Request recordings -Storage and retention for [request logs](/request-logs). The bundled Docker setup already sets the filesystem path and mounts a volume for it, so a default install needs none of these. +Storage and retention for [request logs](/llm-gateway/docs/request-logs). The bundled Docker setup already sets the filesystem path and mounts a volume for it, so a default install needs none of these. | Variable | Default | Description | |---|---|---| @@ -83,7 +84,7 @@ Storage and retention for [request logs](/request-logs). The bundled Docker setu ## Autofix -[Autofix](/autofix) works with zero configuration; `AUTOFIX_GLOBAL_ENABLED=false` is the kill switch. +[Autofix](/llm-gateway/docs/autofix) works with zero configuration; `AUTOFIX_GLOBAL_ENABLED=false` is the kill switch. | Variable | Default | Description | |---|---|---| @@ -95,9 +96,9 @@ Storage and retention for [request logs](/request-logs). The bundled Docker setu ## Email -Used for both Better Auth transactional emails (signup verification, password reset) **and** [threshold alerts](/observability#spend-alerts). Set one provider block. +Used for both Better Auth transactional emails (signup verification, password reset) **and** [threshold alerts](/llm-gateway/docs/observability#spend-alerts). Set one provider block. -A provider saved on a harness's [Limits page](/observability#email-delivery) takes precedence for threshold alerts. These variables stay the only path for verification and password-reset email. +A provider saved on a harness's [Limits page](/llm-gateway/docs/observability#email-delivery) takes precedence for threshold alerts. These variables stay the only path for verification and password-reset email. | Variable | Description | |---|---| @@ -141,7 +142,7 @@ Subscription OAuth (ChatGPT, Claude, MiniMax) uses Manifest-side client IDs by d | Variable | Default | Description | |---|---|---| -| `MANIFEST_TELEMETRY_DISABLED` | unset | Set to `1` to disable [anonymous telemetry](/reference/telemetry) | +| `MANIFEST_TELEMETRY_DISABLED` | unset | Set to `1` to disable [anonymous telemetry](/llm-gateway/docs/reference/telemetry) | | `TELEMETRY_ENDPOINT` | `https://telemetry.manifest.build/...` | Send reports to your own endpoint | | `MANIFEST_PUBLIC_STATS` | `false` | Set `true` to expose `/api/v1/public/*` aggregate stats without auth | @@ -165,7 +166,7 @@ Sentry is opt-in and stays off unless you give it a DSN. | Variable | Default | Description | |---|---|---| -| `MANIFEST_VERSION` | `latest` | Image tag the bundled compose file pulls. Set `6` to follow a major line, or `6.18.0` to pin one release. Docker Compose substitutes it into the image name; the app itself never reads it. See [Image tags](/self-hosted#image-tags) | +| `MANIFEST_VERSION` | `latest` | Image tag the bundled compose file pulls. Set `6` to follow a major line, or `6.18.0` to pin one release. Docker Compose substitutes it into the image name; the app itself never reads it. See [Image tags](/llm-gateway/docs/self-hosted#image-tags) | ## Postgres bundled volume diff --git a/reference/glossary.mdx b/reference/glossary.mdx index a38e222..70c673a 100644 --- a/reference/glossary.mdx +++ b/reference/glossary.mdx @@ -4,6 +4,7 @@ description: "Definitions for harness, request, attempt, Autofix, tier, default icon: "book" keywords: ["agent", "harness", "request", "attempt", "autofix", "tier", "custom tier", "default routing", "direct routing", "fallback chain", "auto", "provider", "auth type"] +canonical: "https://manifest.build/llm-gateway/docs/reference/glossary" --- ## Harness @@ -20,7 +21,7 @@ One provider call inside a request. The initial call, each [fallback](#fallback) ## Autofix -The repair step for requests that fail because of the request itself: a parameter the provider rejects, a tool schema in the wrong shape, a model name that moved. Manifest corrects the failing request and sends it once more, before fallback. See [Autofix](/autofix). +The repair step for requests that fail because of the request itself: a parameter the provider rejects, a tool schema in the wrong shape, a model name that moved. Manifest corrects the failing request and sends it once more, before fallback. See [Autofix](/llm-gateway/docs/autofix). ## Auth type @@ -38,7 +39,7 @@ The model ID you send to opt into routing. When Manifest sees `auto`, it applies ## Custom tier -A routing rule matched on a request header. You choose the header key and value on the dashboard and pin the tier to its own model and fallback chain. When a request carries that header, it routes to the tier's model instead of the default. That holds even when the body names an explicit model ID. See [Routing → Custom](/llm-gateway#custom-tiers). +A routing rule matched on a request header. You choose the header key and value on the dashboard and pin the tier to its own model and fallback chain. When a request carries that header, it routes to the tier's model instead of the default. That holds even when the body names an explicit model ID. See [Routing → Custom](/llm-gateway/docs/llm-gateway#custom-tiers). ## Direct routing @@ -46,7 +47,7 @@ Sending a real model ID (one that `GET /v1/models` lists) instead of `auto`. Man ## Fallback -The retry mechanism that kicks in when the primary model fails. Manifest tries the next model in the tier's fallback list, then the next, until one succeeds or the list is exhausted. See [Fallback](/llm-gateway#fallback) for triggers and config. +The retry mechanism that kicks in when the primary model fails. Manifest tries the next model in the tier's fallback list, then the next, until one succeeds or the list is exhausted. See [Fallback](/llm-gateway/docs/llm-gateway#fallback) for triggers and config. ## Fallback chain @@ -56,15 +57,15 @@ The ordered list of models tried for a single tier, primary first. Up to 5 model An upstream LLM service Manifest can route to. There are four kinds: -- [API key](/providers/api-key-providers) — pay-per-token (OpenAI, Anthropic, Google, and more) -- [Subscription](/providers/subscription-based-providers) — a plan you already pay for (ChatGPT Plus, Claude Max, and more) -- [Local](/providers/local-models) — runs on your hardware (Ollama, LM Studio, llama.cpp) -- [Custom](/providers/custom-providers) — any OpenAI- or Anthropic-compatible HTTP endpoint +- [API key](/llm-gateway/docs/providers/api-key-providers) — pay-per-token (OpenAI, Anthropic, Google, and more) +- [Subscription](/llm-gateway/docs/providers/subscription-based-providers) — a plan you already pay for (ChatGPT Plus, Claude Max, and more) +- [Local](/llm-gateway/docs/providers/local-models) — runs on your hardware (Ollama, LM Studio, llama.cpp) +- [Custom](/llm-gateway/docs/providers/custom-providers) — any OpenAI- or Anthropic-compatible HTTP endpoint ## Routing -The step that decides which model handles a request. Manifest checks your custom tiers first, then sends anything that doesn't match to your default model. It runs in-process with no extra network call. See [Routing](/llm-gateway). +The step that decides which model handles a request. Manifest checks your custom tiers first, then sends anything that doesn't match to your default model. It runs in-process with no extra network call. See [Routing](/llm-gateway/docs/llm-gateway). ## Tier -A routing lane with its own model and fallback chain. Every harness has a **Default** tier; you can add **Custom** tiers matched on request headers. See [Routing](/llm-gateway). +A routing lane with its own model and fallback chain. Every harness has a **Default** tier; you can add **Custom** tiers matched on request headers. See [Routing](/llm-gateway/docs/llm-gateway). diff --git a/reference/headers.mdx b/reference/headers.mdx index 0753d4f..563f8db 100644 --- a/reference/headers.mdx +++ b/reference/headers.mdx @@ -4,6 +4,7 @@ description: "The request headers Manifest reads and the X-Manifest-* response h icon: "list" keywords: ["X-Manifest-Tier", "X-Manifest-Model", "X-Manifest-Provider", "X-Manifest-Reason", "X-Manifest-Fallback-From", "x-session-key", "custom routing header"] +canonical: "https://manifest.build/llm-gateway/docs/reference/headers" --- Manifest reads a few request headers, and returns a set of `X-Manifest-*` response headers so clients can see what happened without parsing the response body. @@ -15,10 +16,10 @@ Manifest reads a few request headers, and returns a set of `X-Manifest-*` respon | `Authorization` | `Bearer mnfst_` | **Required.** Authenticates the harness. | | `Content-Type` | `application/json` | **Required.** | | `anthropic-version` | `2023-06-01` | Accepted and ignored. Anthropic SDKs send it on their own. Manifest picks the version it sends upstream. | -| Your custom tier header | your value | Routes the request to a [custom tier](/llm-gateway#custom-tiers). You set the key and value when you create the tier, so the exact header name is up to you. | +| Your custom tier header | your value | Routes the request to a [custom tier](/llm-gateway/docs/llm-gateway#custom-tiers). You set the key and value when you create the tier, so the exact header name is up to you. | | `x-session-key` | any string | Groups requests into a session. Manifest keeps a session on the same upstream where the provider supports it, for sticky routing and prompt caching. | -The custom tier header is the only one that changes routing: send the header you configured on a tier and the request goes to that tier's model. The header wins even when the body names an explicit model ID. See [Routing → Custom](/llm-gateway#custom-tiers). +The custom tier header is the only one that changes routing: send the header you configured on a tier and the request goes to that tier's model. The header wins even when the body names an explicit model ID. See [Routing → Custom](/llm-gateway/docs/llm-gateway#custom-tiers). ## Response headers (routed requests) diff --git a/reference/telemetry.mdx b/reference/telemetry.mdx index 65bf904..6d9882e 100644 --- a/reference/telemetry.mdx +++ b/reference/telemetry.mdx @@ -5,15 +5,16 @@ description: "What Manifest stores about your requests, what's optional, and the icon: "shield" keywords: ["data privacy", "prompt storage", "message recording", "anonymous telemetry", "MANIFEST_TELEMETRY_DISABLED", "TELEMETRY_ENDPOINT", "install_id", "opt out", "self-hosted telemetry"] +canonical: "https://manifest.build/llm-gateway/docs/reference/telemetry" --- Two separate things happen to data as requests flow through Manifest: what your own instance records for the dashboard, and what a self-hosted install reports back to the project. This page covers both. ## What Manifest stores -Manifest always keeps metadata about each request: model, provider, tier, token counts, cost, latency, and an error message. Error text is scrubbed for secrets before it lands, and the request headers Manifest stores drop credentials and IP addresses. The usage and cost views in the dashboard are built from that metadata alone. See [Observability](/observability) for what the record looks like in practice. +Manifest always keeps metadata about each request: model, provider, tier, token counts, cost, latency, and an error message. Error text is scrubbed for secrets before it lands, and the request headers Manifest stores drop credentials and IP addresses. The usage and cost views in the dashboard are built from that metadata alone. See [Observability](/llm-gateway/docs/observability) for what the record looks like in practice. -Message bodies are separate, and optional. With [request logs](/request-logs) on for a harness, Manifest also stores the request and response bodies of each provider attempt so you can read them back in the dashboard. New harnesses have it enabled; to make sure a harness records its logs, check its **Settings → Logs**. +Message bodies are separate, and optional. With [request logs](/llm-gateway/docs/request-logs) on for a harness, Manifest also stores the request and response bodies of each provider attempt so you can read them back in the dashboard. New harnesses have it enabled; to make sure a harness records its logs, check its **Settings → Logs**. A stored body is exactly what went over the wire: every message, system instruction, tool definition and inline image, at full length. Manifest doesn't redact or truncate it, and there is no size limit. Stored bodies are held outside the database, in the object storage your instance is configured for, and deleted on a retention schedule. With logs off for a harness, that harness's prompts and completions stay between you and the provider. @@ -91,6 +92,6 @@ Your collector receives the same payload documented above, and nothing is sent t ## Related -- [Observability](/observability) — what the recorded metadata gets you -- [Environment variables](/reference/environment-variables) — `MANIFEST_TELEMETRY_DISABLED` and `TELEMETRY_ENDPOINT` in context -- [Self-hosting with Docker](/self-hosted) — where to put these variables +- [Observability](/llm-gateway/docs/observability) — what the recorded metadata gets you +- [Environment variables](/llm-gateway/docs/reference/environment-variables) — `MANIFEST_TELEMETRY_DISABLED` and `TELEMETRY_ENDPOINT` in context +- [Self-hosting with Docker](/llm-gateway/docs/self-hosted) — where to put these variables diff --git a/request-logs.mdx b/request-logs.mdx index b8d81f7..66c9441 100644 --- a/request-logs.mdx +++ b/request-logs.mdx @@ -4,6 +4,7 @@ description: "Store the full request and response body of every provider attempt icon: "logs" keywords: ["request logs", "message recording", "request body", "response body", "LLM conversation log", "prompt logging", "full body log", "tool calls", "log retention", "S3 log storage"] +canonical: "https://manifest.build/llm-gateway/docs/request-logs" ---
@@ -11,13 +12,13 @@ keywords:
-The [Requests page](/observability) tells you what a request cost and which model served it. Request logs tell you what was actually said: the body Manifest sent to the provider, and the body that came back. The dashboard calls this feature **Logs**. +The [Requests page](/llm-gateway/docs/observability) tells you what a request cost and which model served it. Request logs tell you what was actually said: the body Manifest sent to the provider, and the body that came back. The dashboard calls this feature **Logs**. It's one toggle per harness. ## What gets recorded -Recording happens per **provider attempt**, not per request. A request that falls back twice before succeeding produces three recordings, each with its own body pair, so you can see how the conversation was reshaped for each provider. An [Autofix](/autofix) retry is its own attempt too, which is how you compare the request that failed against the patched one that worked. +Recording happens per **provider attempt**, not per request. A request that falls back twice before succeeding produces three recordings, each with its own body pair, so you can see how the conversation was reshaped for each provider. An [Autofix](/llm-gateway/docs/autofix) retry is its own attempt too, which is how you compare the request that failed against the patched one that worked. | | What's stored | |---|---| @@ -25,11 +26,11 @@ Recording happens per **provider attempt**, not per request. A request that fall | **Response** | The parsed JSON body, or for a streamed attempt, the raw SSE stream as it arrived | | **Wire format** | Which protocol the exchange used, so the drawer can render it correctly | -A request Manifest blocked itself, on a [hard limit](/llm-gateway#hard-limits) or a malformed body, still lists one attempt, labelled with the model you asked for or **No provider**. It carries the failure status and no **Messages**, **Tools**, or **Raw** tab: Manifest sent nothing to a provider, so there is nothing to read back. +A request Manifest blocked itself, on a [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) or a malformed body, still lists one attempt, labelled with the model you asked for or **No provider**. It carries the failure status and no **Messages**, **Tools**, or **Raw** tab: Manifest sent nothing to a provider, so there is nothing to read back. ## Reading it back -Open a request in the dashboard's **Requests** page and pick an attempt in the **Attempts** list. The initial call, each fallback, and an [Autofix](/autofix) retry are separate attempts, and each one shows its own conversation. The **Messages**, **Tools**, and **Raw** tabs appear when that attempt has a recording. +Open a request in the dashboard's **Requests** page and pick an attempt in the **Attempts** list. The initial call, each fallback, and an [Autofix](/llm-gateway/docs/autofix) retry are separate attempts, and each one shows its own conversation. The **Messages**, **Tools**, and **Raw** tabs appear when that attempt has a recording. A request opened on the Requests page, showing the Attempts list and the per-attempt tabs @@ -53,7 +54,7 @@ New harnesses have it on. To make sure a harness records its logs, check **Setti Recording stores your prompts and completions. Everything else Manifest keeps - is metadata ([Data and telemetry](/reference/telemetry) covers the + is metadata ([Data and telemetry](/llm-gateway/docs/reference/telemetry) covers the distinction). If a harness handles data you'd rather Manifest never hold onto, leave recording off for that harness. @@ -78,7 +79,7 @@ Self-hosted installs can override this with `REQUEST_RECORDING_RETENTION_DAYS`. Logs were reintroduced in **v6.18.0**. On an older self-hosted install, - [upgrade Manifest](/self-hosted#upgrading) first. + [upgrade Manifest](/llm-gateway/docs/self-hosted#upgrading) first.
@@ -90,7 +91,7 @@ Recordings don't live in Postgres. They're gzipped and written to object storage The bundled compose file mounts a named volume `manifest_request_recordings` at `/data/request-recordings` and points Manifest at it. Nothing to configure when your compose file includes that volume; an install from - before v6.18.0 needs [one upgrade step](/self-hosted#upgrading) first. The + before v6.18.0 needs [one upgrade step](/llm-gateway/docs/self-hosted#upgrading) first. The volume survives `docker compose down` the same way the Postgres volume does. @@ -121,7 +122,7 @@ Recordings don't live in Postgres. They're gzipped and written to object storage fully or not at all. -Full variable list: [Environment variables](/reference/environment-variables#request-recordings). +Full variable list: [Environment variables](/llm-gateway/docs/reference/environment-variables#request-recordings). ### Check it's working @@ -139,7 +140,7 @@ The one-click templates set this up for you. Render mounts a persistent disk. Ra ## Related -- [Observability](/observability) — the request log these recordings hang off -- [Autofix](/autofix) — comparing a failed request against its patched retry -- [Data and telemetry](/reference/telemetry) — what Manifest stores by default -- [Environment variables](/reference/environment-variables#request-recordings) +- [Observability](/llm-gateway/docs/observability) — the request log these recordings hang off +- [Autofix](/llm-gateway/docs/autofix) — comparing a failed request against its patched retry +- [Data and telemetry](/llm-gateway/docs/reference/telemetry) — what Manifest stores by default +- [Environment variables](/llm-gateway/docs/reference/environment-variables#request-recordings) diff --git a/self-hosted.mdx b/self-hosted.mdx index 96b797c..5033921 100644 --- a/self-hosted.mdx +++ b/self-hosted.mdx @@ -5,11 +5,12 @@ description: "Run Manifest on your own machine with Docker. Covers the quick ins icon: "docker" keywords: ["self-hosted LLM router", "Docker", "docker-compose", "PostgreSQL", "BETTER_AUTH_SECRET", "docker run", "upgrade", "backup"] +canonical: "https://manifest.build/llm-gateway/docs/self-hosted" --- Run the full Manifest stack on your own machine. No Node.js required, just Docker. -To run Manifest somewhere other than your own machine, see the [other self-hosting paths](/deploy). +To run Manifest somewhere other than your own machine, see the [other self-hosting paths](/llm-gateway/docs/deploy). All three paths end in the same place: a running stack at [http://localhost:2099](http://localhost:2099). On first access, Manifest takes you to a setup screen where you create the admin account. No demo credentials are pre-seeded. @@ -150,14 +151,14 @@ Signing up leaves you with an empty instance. Three steps to a routed request. -d '{"model": "auto", "messages": [{"role": "user", "content": "Hello"}]}' ``` - `"model": "auto"` asks Manifest to route the request. Any other name is treated as an explicit choice: if the model isn't available to your harness, the request fails with [M302](/errors/M302). Send `auto` to use routing. + `"model": "auto"` asks Manifest to route the request. Any other name is treated as an explicit choice: if the model isn't available to your harness, the request fails with [M302](/llm-gateway/docs/errors/M302). Send `auto` to use routing. The request shows up in the dashboard straight away, with the model that served it and what it cost. - Errors raised by Manifest itself carry an `M###` code, a plain-English cause, and a link to the matching page under [manifest.build/docs/errors](/errors). The three you are most likely to see on a fresh install: + Errors raised by Manifest itself carry an `M###` code, a plain-English cause, and a link to the matching page under [manifest.build/docs/errors](/llm-gateway/docs/errors). The three you are most likely to see on a fresh install: | Code | Means | | --- | --- | @@ -257,7 +258,7 @@ Database migrations run automatically on boot, no manual steps. Your data in the The two commands above only pull new images. Your `docker-compose.yml` stays as it is, hand edits included. `bash install.sh --upgrade` is the other path: it downloads a fresh `docker-compose.yml` and replaces yours, after copying the old one next to it as `docker-compose.yml.backup.`. Your `.env` is left alone. Keep custom compose changes in `docker-compose.override.yml`, which the installer never downloads. - **Upgrading from before v6.18.0 and using [logs](/request-logs)?** Run the installer once with the upgrade flag, or log bodies are written inside the container and lost when it is recreated: + **Upgrading from before v6.18.0 and using [logs](/llm-gateway/docs/request-logs)?** Run the installer once with the upgrade flag, or log bodies are written inside the container and lost when it is recreated: ```bash curl -sSLO https://raw.githubusercontent.com/mnfst/manifest/main/docker/install.sh @@ -291,7 +292,7 @@ The stack uses two named volumes: | Volume | Mounted at | Holds | |---|---|---| | `manifest_pgdata` | `/var/lib/postgresql/data` in `postgres` | Everything: accounts, harnesses, provider credentials, the request log | -| `manifest_request_recordings` | `/data/request-recordings` in `manifest` | Stored [request logs](/request-logs) bodies | +| `manifest_request_recordings` | `/data/request-recordings` in `manifest` | Stored [request logs](/llm-gateway/docs/request-logs) bodies | The database volume is the one to back up. Losing the recordings volume costs you stored message bodies and nothing else, and those expire on a retention schedule regardless. @@ -336,7 +337,7 @@ The first two are required and the installer generates both, so a default instal The installer also generates `MANIFEST_ENCRYPTION_KEY`. If you are installing by hand, set it too. Left unset, Manifest falls back to `BETTER_AUTH_SECRET` for at-rest encryption and warns on every boot — meaning one leaked session-signing secret also decrypts every stored provider key and OAuth token. Set it **before first boot**: introducing it later means re-encrypting what is already in the database. -Everything else is optional: provider timeouts, email delivery for alerts and password resets, OAuth logins, connection-pool sizing, recording storage and retention, Autofix, and Sentry. See [Environment variables](/reference/environment-variables) for the full list with defaults. +Everything else is optional: provider timeouts, email delivery for alerts and password resets, OAuth logins, connection-pool sizing, recording storage and retention, Autofix, and Sentry. See [Environment variables](/llm-gateway/docs/reference/environment-variables) for the full list with defaults. `NODE_ENV` and `SEED_DATA` are fixed by the bundled compose file and are not knobs for a self-hosted install. The image is a production artifact, and the demo-data seeder refuses to run under `NODE_ENV=production` whatever `SEED_DATA` says — use the first-run setup wizard to create your admin account. @@ -353,13 +354,13 @@ docker compose down -v # Stop and delete all data ## Data and privacy -Manifest always keeps metadata about each request — model, provider, tier, token counts, cost, latency. Message bodies are separate and optional: with [request logs](/request-logs) on for a harness, prompts and completions are stored too, in the recordings volume rather than the database. New harnesses have it enabled. See [Data and telemetry](/reference/telemetry). +Manifest always keeps metadata about each request — model, provider, tier, token counts, cost, latency. Message bodies are separate and optional: with [request logs](/llm-gateway/docs/request-logs) on for a harness, prompts and completions are stored too, in the recordings volume rather than the database. New harnesses have it enabled. See [Data and telemetry](/llm-gateway/docs/reference/telemetry). ## Telemetry Once a day, each install sends an anonymous aggregate report: version, provider mix, token and cost totals. Never prompts, keys, or anything tied to a user. Set `MANIFEST_TELEMETRY_DISABLED=1` in your `.env` to turn it off. -Full field list, what's never sent, and how to point it at your own endpoint: [Data and telemetry](/reference/telemetry). +Full field list, what's never sent, and how to point it at your own endpoint: [Data and telemetry](/llm-gateway/docs/reference/telemetry). ## Docker Hub From b000814e22212ccf4441dbce61732fd793c5d973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Conejo?= Date: Wed, 9 Sep 2026 18:12:09 +0200 Subject: [PATCH 2/3] Clarify LLM Gateway documentation positioning --- autofix.mdx | 32 +++++++++---------- deploy.mdx | 14 ++++---- deploy/aws.mdx | 14 ++++---- deploy/coolify.mdx | 12 +++---- deploy/digitalocean.mdx | 14 ++++---- deploy/easypanel.mdx | 20 ++++++------ deploy/fly.mdx | 14 ++++---- deploy/gcp.mdx | 12 +++---- deploy/heroku.mdx | 10 +++--- deploy/koyeb.mdx | 18 +++++------ deploy/railway.mdx | 16 +++++----- deploy/render.mdx | 10 +++--- docs.json | 12 +++---- errors.mdx | 14 ++++---- errors/M001.mdx | 4 +-- errors/M002.mdx | 2 +- errors/M003.mdx | 4 +-- errors/M004.mdx | 4 +-- errors/M005.mdx | 2 +- errors/M100.mdx | 6 ++-- errors/M101.mdx | 2 +- errors/M102.mdx | 8 ++--- errors/M200.mdx | 6 ++-- errors/M201.mdx | 6 ++-- errors/M202.mdx | 2 +- errors/M203.mdx | 4 +-- errors/M204.mdx | 4 +-- errors/M300.mdx | 6 ++-- errors/M302.mdx | 12 +++---- errors/M303.mdx | 8 ++--- errors/M500.mdx | 4 +-- integrations/n8n.mdx | 24 +++++++------- introduction.mdx | 20 ++++++------ llm-gateway.mdx | 22 ++++++------- llm-gateway/docs/autofix.mdx | 32 +++++++++---------- llm-gateway/docs/deploy.mdx | 14 ++++---- llm-gateway/docs/deploy/aws.mdx | 14 ++++---- llm-gateway/docs/deploy/coolify.mdx | 12 +++---- llm-gateway/docs/deploy/digitalocean.mdx | 14 ++++---- llm-gateway/docs/deploy/easypanel.mdx | 20 ++++++------ llm-gateway/docs/deploy/fly.mdx | 14 ++++---- llm-gateway/docs/deploy/gcp.mdx | 12 +++---- llm-gateway/docs/deploy/heroku.mdx | 10 +++--- llm-gateway/docs/deploy/koyeb.mdx | 18 +++++------ llm-gateway/docs/deploy/railway.mdx | 16 +++++----- llm-gateway/docs/deploy/render.mdx | 10 +++--- llm-gateway/docs/errors.mdx | 14 ++++---- llm-gateway/docs/errors/M001.mdx | 4 +-- llm-gateway/docs/errors/M002.mdx | 2 +- llm-gateway/docs/errors/M003.mdx | 4 +-- llm-gateway/docs/errors/M004.mdx | 4 +-- llm-gateway/docs/errors/M005.mdx | 2 +- llm-gateway/docs/errors/M100.mdx | 6 ++-- llm-gateway/docs/errors/M101.mdx | 2 +- llm-gateway/docs/errors/M102.mdx | 8 ++--- llm-gateway/docs/errors/M200.mdx | 6 ++-- llm-gateway/docs/errors/M201.mdx | 6 ++-- llm-gateway/docs/errors/M202.mdx | 2 +- llm-gateway/docs/errors/M203.mdx | 4 +-- llm-gateway/docs/errors/M204.mdx | 4 +-- llm-gateway/docs/errors/M300.mdx | 6 ++-- llm-gateway/docs/errors/M302.mdx | 12 +++---- llm-gateway/docs/errors/M303.mdx | 8 ++--- llm-gateway/docs/errors/M500.mdx | 4 +-- llm-gateway/docs/integrations/n8n.mdx | 24 +++++++------- llm-gateway/docs/introduction.mdx | 20 ++++++------ llm-gateway/docs/llm-gateway.mdx | 22 ++++++------- llm-gateway/docs/observability.mdx | 14 ++++---- .../docs/providers/api-key-providers.mdx | 10 +++--- .../docs/providers/custom-providers.mdx | 18 +++++------ llm-gateway/docs/providers/local-models.mdx | 18 +++++------ .../subscription-based-providers.mdx | 18 +++++------ llm-gateway/docs/reference/api.mdx | 22 ++++++------- .../docs/reference/environment-variables.mdx | 27 +++++++++------- llm-gateway/docs/reference/glossary.mdx | 22 ++++++------- llm-gateway/docs/reference/headers.mdx | 10 +++--- llm-gateway/docs/reference/telemetry.mdx | 22 ++++++------- llm-gateway/docs/request-logs.mdx | 24 +++++++------- llm-gateway/docs/self-hosted.mdx | 22 ++++++------- observability.mdx | 14 ++++---- providers/api-key-providers.mdx | 10 +++--- providers/custom-providers.mdx | 18 +++++------ providers/local-models.mdx | 18 +++++------ providers/subscription-based-providers.mdx | 18 +++++------ reference/api.mdx | 22 ++++++------- reference/environment-variables.mdx | 27 +++++++++------- reference/glossary.mdx | 22 ++++++------- reference/headers.mdx | 10 +++--- reference/telemetry.mdx | 22 ++++++------- request-logs.mdx | 24 +++++++------- self-hosted.mdx | 22 ++++++------- 91 files changed, 586 insertions(+), 580 deletions(-) diff --git a/autofix.mdx b/autofix.mdx index 1a34e7e..ef42075 100644 --- a/autofix.mdx +++ b/autofix.mdx @@ -1,6 +1,6 @@ --- title: "Autofix" -description: "Repair a malformed request and resend it once, before the fallback chain runs. Turn it on per harness." +description: "Use Manifest Autofix through the LLM Gateway to repair a malformed request and retry it once before fallback." icon: "wrench" keywords: ["auto-fix", "autofix", "self-healing", "LLM request repair", "malformed request", "400 bad request", "422 unprocessable", "model not found", "agent debugging", "harness debugging"] @@ -14,7 +14,7 @@ canonical: "https://manifest.build/llm-gateway/docs/autofix" Some requests fail because of the request, not the model. A parameter the provider doesn't accept, a tool schema in the wrong shape, a model name that moved. Switching to a fallback model doesn't help, because the next model rejects the same body. -Autofix handles that case. When a request fails with a repairable error, Manifest sends the failed request and the provider's error to a healing service, gets a corrected request back, and sends it once more. It runs **before** the fallback chain, so fallback is still the safety net if the repair doesn't land. +Autofix handles that case. When a request fails with a repairable error, the gateway sends the failed request and the provider's error to a healing service, gets a corrected request back, and sends it once more. It runs **before** the fallback chain, so fallback is still the safety net if the repair doesn't land. ## When it runs @@ -28,25 +28,25 @@ Autofix only touches request-side 4xx errors, where the provider is telling you It deliberately skips `401`, `403`, `429`, and every `5xx`. A bad key, a rate limit, or a provider outage isn't something a rewritten body can fix, so those go straight to [fallback](/llm-gateway/docs/llm-gateway#fallback). -Renamed models are covered through the provider itself. When a request names a model Manifest hasn't catalogued but the provider and credentials are unambiguous, Manifest forwards it anyway. The provider answers with its real error, typically a `404` for a model that no longer exists, and that error goes through the normal Autofix path. A patch that corrects the model name serves the request. +Renamed models are covered through the provider itself. When a request names a model the gateway hasn't catalogued but the provider and credentials are unambiguous, the gateway forwards it anyway. The provider answers with its real error, typically a `404` for a model that no longer exists, and that error goes through the normal Autofix path. A patch that corrects the model name serves the request. ## How it works - A provider returns a repairable 4xx. Manifest normalizes the error into a + A provider returns a repairable 4xx. The gateway normalizes the error into a message, type, and parameter. - + The failed request and the normalized error go to the hosted healing service, which returns a corrected request body. - Manifest sends the patched request. There is no retry budget: it tries + The gateway sends the patched request. There is no retry budget: it tries exactly once. - If the patched request fails too, Manifest reports the outcome and hands + If the patched request fails too, the gateway reports the outcome and hands off to your fallback chain as normal. @@ -59,7 +59,7 @@ Autofix is a per-harness toggle, and you meet it at two moments.
-Nothing to deploy or configure: your install talks to Manifest's hosted healing service directly. +Nothing to deploy or configure: your install talks to the hosted Manifest Autofix service directly.
@@ -73,11 +73,11 @@ The Connect Harness modal shows an Autofix toggle under the name field, pre-set ### Harnesses you already have -A harness that existed before Autofix keeps its state: Manifest never flips the toggle for you. +A harness that existed before Autofix keeps its state: the gateway never flips the toggle for you.
-On Manifest Cloud, those harnesses are already on. You can switch any of them off on its **Settings** page. +On LLM Gateway Cloud, those harnesses are already on. You can switch any of them off on its **Settings** page.
@@ -113,7 +113,7 @@ The first time you enable Autofix, your install records a one-time consent: enab
-When Autofix repairs a request, Manifest sends the failed request, including its message content, plus the provider's error response to the healing service. +When Autofix repairs a request, the gateway sends the failed request, including its message content, plus the provider's error response to the healing service. Provider API keys and OAuth tokens are never sent: credentials travel in request headers, which are not part of the payload sent to the healing service. The provider's error response is scrubbed for secrets before it is sent. @@ -132,9 +132,9 @@ See the [privacy policy](https://manifest.build/privacy/) and the [terms](https: Autofix is built to stay out of the way when it isn't working: - **One attempt.** A patched request is sent once. It never loops. -- **Failures degrade quietly.** If the healing service errors or times out, your agent gets the original provider error. Autofix never turns a provider's 400 into a Manifest 500. -- **A slow healer stops being asked.** After three consecutive transport failures, Manifest stops calling it for 30 seconds so a down service doesn't add latency to every failing request. One successful call clears the streak. -- **Startup never waits.** In production, Manifest pings the healing service once at boot, fire-and-forget, and logs a warning if it's unreachable. It never blocks startup, and it's skipped entirely when Autofix is disabled globally. +- **Failures degrade quietly.** If the healing service errors or times out, your agent gets the original provider error. Autofix never turns a provider's 400 into a gateway 500. +- **A slow healer stops being asked.** After three consecutive transport failures, the gateway stops calling it for 30 seconds so a down service doesn't add latency to every failing request. One successful call clears the streak. +- **Startup never waits.** In production, the gateway pings the healing service once at boot, fire-and-forget, and logs a warning if it's unreachable. It never blocks startup, and it's skipped entirely when Autofix is disabled globally. - **Fallback still runs.** Autofix sits in front of the chain, it doesn't replace it. ## Seeing what it did @@ -162,7 +162,7 @@ Both **Recovered** cards link through to the matching filter on the request log. ## Environment variables - Self-hosted only. On [Manifest Cloud](https://app.manifest.build) there is no + Self-hosted only. On [LLM Gateway Cloud](https://app.manifest.build) there is no server to configure: everything Autofix related lives in the dashboard. @@ -173,7 +173,7 @@ Autofix needs no setup. Every variable below is optional and adjusts behavior: | `AUTOFIX_GLOBAL_ENABLED` | `true` | Set `false` to turn Autofix off for the whole deployment. No call reaches the healing service, including the boot health check | | `AUTOFIX_TIMEOUT_MS` | `10000` | Timeout per healing call, in milliseconds | | `AUTOFIX_REPAIRABLE_STATUSES` | `400,404,422` | Which provider statuses are eligible for a repair | -| `AUTOFIX_HEALING_API_KEY` | unset | Sent as `x-api-key` to the healing service. Only relevant if Manifest issues you one | +| `AUTOFIX_HEALING_API_KEY` | unset | Sent as `x-api-key` to the healing service. Only relevant when provided by the hosted healing service | | `AUTOFIX_REPORT_ALL_4XX` | `false` | Opt-in. Also report request-side 4xx errors (except `401`/`402`/`403`/`429`) from harnesses with Autofix on, as diagnostic evidence with the secret-scrubbed request body. Bodies over 256 KB are dropped, not truncated. No fix comes back from it | You don't need `AUTOFIX_HEALING_API_KEY`. With no key set, the install announces its anonymous install id instead. The full list of variables lives in [Environment variables](/llm-gateway/docs/reference/environment-variables). diff --git a/deploy.mdx b/deploy.mdx index 93cf485..ec96add 100644 --- a/deploy.mdx +++ b/deploy.mdx @@ -1,11 +1,11 @@ --- -title: "Self-hosting Manifest" +title: "Deploy the LLM Gateway" sidebarTitle: "Overview" -description: "Every way to run your own Manifest instance: Docker on your machine, a managed platform like Railway, your own AWS or GCP account, or a server panel like Coolify." +description: "Run the Manifest LLM Gateway with Docker, on a managed platform, or in your cloud account." icon: "rocket" keywords: [ - "self-hosted Manifest", + "self-hosted gateway", "Manifest deployment", "Docker", "Railway", @@ -19,7 +19,7 @@ keywords: canonical: "https://manifest.build/llm-gateway/docs/deploy" --- -Everything on this page runs the same open-source Manifest image on infrastructure you control, with your own PostgreSQL database. Pick the path that matches where you want it to live. If you'd rather not run anything yourself, [Manifest Cloud](https://app.manifest.build) is the hosted alternative. +Everything on this page runs the same open-source gateway image on infrastructure you control, with your own PostgreSQL database. Pick the path that matches where you want it to live. If you'd rather not run anything yourself, [LLM Gateway Cloud](https://app.manifest.build) is the hosted alternative. ## Your own machine @@ -35,7 +35,7 @@ Someone else runs the infrastructure. Usually a one-click template or deploy but - Best one-click path for most users. The template provisions Manifest and + Best one-click path for most users. The template provisions the gateway and PostgreSQL. @@ -43,12 +43,12 @@ Someone else runs the infrastructure. Usually a one-click template or deploy but Render PostgreSQL. - App Platform deploy from the public repository with Manifest and a Dev + App Platform deploy from the public repository with the gateway and a Dev PostgreSQL database. CLI-based deploy using the public Docker image, Fly Postgres, and generated - Manifest secrets. + gateway secrets. Deploy-button guide with Heroku Postgres, generated secrets, and one diff --git a/deploy/aws.mdx b/deploy/aws.mdx index 9ae9176..aa59210 100644 --- a/deploy/aws.mdx +++ b/deploy/aws.mdx @@ -1,10 +1,10 @@ --- -title: "Deploy Manifest on AWS" +title: "Deploy the LLM Gateway on AWS" sidebarTitle: "AWS" -description: "Deploy Manifest on AWS with CloudFormation, ECS Fargate, RDS PostgreSQL, Secrets Manager, and an Application Load Balancer." +description: "Deploy the Manifest LLM Gateway with CloudFormation, ECS Fargate, RDS PostgreSQL, Secrets Manager, and an Application Load Balancer." icon: "/icons/deploy/aws.svg" keywords: - ["Manifest deployment", "AWS", "self-hosted Manifest", "PostgreSQL", "Docker"] + ["Manifest deployment", "AWS", "self-hosted gateway", "PostgreSQL", "Docker"] canonical: "https://manifest.build/llm-gateway/docs/deploy/aws" --- @@ -19,7 +19,7 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/aws" ## Overview -The AWS template deploys Manifest with ECS Fargate, RDS PostgreSQL, Secrets Manager, CloudWatch Logs, and an Application Load Balancer. It is a good fit when you want Manifest in your own AWS account. +The AWS template deploys the gateway with ECS Fargate, RDS PostgreSQL, Secrets Manager, CloudWatch Logs, and an Application Load Balancer. It is a good fit when you want the gateway in your own AWS account. ## Prerequisites @@ -65,11 +65,11 @@ The CloudFormation template generates: - `BETTER_AUTH_SECRET`. - `MANIFEST_ENCRYPTION_KEY`. -Manifest runs with `PORT=2099`, `BIND_ADDRESS=0.0.0.0`, and `MANIFEST_MODE=selfhosted`. +The gateway runs with `PORT=2099`, `BIND_ADDRESS=0.0.0.0`, and `MANIFEST_MODE=selfhosted`. The template also creates an S3 bucket for [request logs](/llm-gateway/docs/request-logs). It is retained by design, so deleting the CloudFormation stack leaves the bucket in place. Delete it by hand when you tear the deployment down. -## Open Manifest +## Open the gateway After deployment, the script prints `ServiceUrl` and `HealthCheckUrl`. @@ -83,7 +83,7 @@ Open `ServiceUrl` and create the first account. Fresh installs redirect to `/set ## Production notes -- The default template exposes Manifest over HTTP on the generated load balancer DNS name. +- The default template exposes the gateway over HTTP on the generated load balancer DNS name. - Configure TLS with your own domain and ACM certificate before using the deployment for production authentication traffic. - Set database deletion protection if you want to avoid accidental RDS deletion. diff --git a/deploy/coolify.mdx b/deploy/coolify.mdx index b6ad73d..85d2462 100644 --- a/deploy/coolify.mdx +++ b/deploy/coolify.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Coolify" +title: "Deploy the LLM Gateway on Coolify" sidebarTitle: "Coolify" -description: "Deploy Manifest on Coolify with a Docker Compose service stack, generated secrets, and a private PostgreSQL container." +description: "Deploy the Manifest LLM Gateway with a Docker Compose service stack, generated secrets, and a private PostgreSQL container." icon: "/icons/deploy/coolify.svg" keywords: [ "Manifest deployment", "Coolify", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -49,11 +49,11 @@ This stack runs on infrastructure you control through Coolify. Your server, stor - `SERVICE_HEX_64_ENCRYPTION` 7. Deploy. -The Compose file uses Coolify magic environment variables so the app URL, Postgres password, Better Auth secret, and Manifest encryption key are generated and stay stable between deployments. +The Compose file uses Coolify magic environment variables so the app URL, Postgres password, Better Auth secret, and gateway encryption key are generated and stay stable between deployments. -## Open Manifest +## Open the gateway -After deployment, open the generated Manifest URL and create the first admin account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. +After deployment, open the generated gateway URL and create the first admin account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. Verify the deployment: diff --git a/deploy/digitalocean.mdx b/deploy/digitalocean.mdx index cb71a2d..5a4a822 100644 --- a/deploy/digitalocean.mdx +++ b/deploy/digitalocean.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on DigitalOcean" +title: "Deploy the LLM Gateway on DigitalOcean" sidebarTitle: "DigitalOcean" -description: "Deploy Manifest on DigitalOcean App Platform with the Deploy to DigitalOcean flow and a Dev PostgreSQL database." +description: "Deploy the Manifest LLM Gateway on App Platform with the Deploy to DigitalOcean flow and a PostgreSQL database." icon: "/icons/deploy/digitalocean.svg" keywords: [ "Manifest deployment", "DigitalOcean", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -19,13 +19,13 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/digitalocean" icon="external-link" href="https://cloud.digitalocean.com/apps/new?repo=https://github.com/mnfst/manifest/tree/main" > - App Platform deploy from the public repository with Manifest and a Dev + App Platform deploy from the public repository with the gateway and a Dev PostgreSQL database. ## Overview -DigitalOcean App Platform can create a Manifest app from the public repository template. The template uses the public Manifest Docker image and provisions a Dev PostgreSQL database. +DigitalOcean App Platform can create a gateway app from the public repository template. The template uses the public Manifest Docker image and provisions a Dev PostgreSQL database. ## Prerequisites @@ -51,9 +51,9 @@ openssl rand -hex 32 6. Fill in the recording storage secrets with your Space's settings: `REQUEST_RECORDING_S3_BUCKET`, `REQUEST_RECORDING_S3_ENDPOINT`, `REQUEST_RECORDING_S3_ACCESS_KEY_ID`, and `REQUEST_RECORDING_S3_SECRET_ACCESS_KEY`. The region defaults to `us-east-1`, which DigitalOcean Spaces accepts as a signing region. 7. Create the app. -DigitalOcean reads `.do/deploy.template.yaml` from the Manifest repository. The template includes the Manifest service and a Dev PostgreSQL database. +DigitalOcean reads `.do/deploy.template.yaml` from the Manifest repository. The template includes the gateway service and a Dev PostgreSQL database. -## Open Manifest +## Open the gateway After App Platform finishes deploying, open the app URL and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. diff --git a/deploy/easypanel.mdx b/deploy/easypanel.mdx index 065366e..3e7c80e 100644 --- a/deploy/easypanel.mdx +++ b/deploy/easypanel.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Easypanel" +title: "Deploy the LLM Gateway on Easypanel" sidebarTitle: "Easypanel" -description: "Deploy Manifest on Easypanel with a Docker image app service, PostgreSQL, generated secrets, and HTTPS." +description: "Deploy the Manifest LLM Gateway with a Docker image app service, PostgreSQL, generated secrets, and HTTPS." icon: "/icons/deploy/easypanel.svg" keywords: [ "Manifest deployment", "Easypanel", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -25,7 +25,7 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/easypanel" ## Overview -Easypanel can run Manifest as a Docker image app service, proxy the dashboard over HTTPS, and provision PostgreSQL in the same project. The Manifest repository includes Easypanel template files that provision the app, database, generated secrets, and proxy port. Until that template is accepted into the Easypanel catalog, you can create the same services manually. +Easypanel can run the gateway as a Docker image app service, proxy the dashboard over HTTPS, and provision PostgreSQL in the same project. The Manifest repository includes Easypanel template files that provision the app, database, generated secrets, and proxy port. Until that template is accepted into the Easypanel catalog, you can create the same services manually. ## Prerequisites @@ -37,7 +37,7 @@ This stack runs on infrastructure you control through Easypanel. Your server, st ## Template deploy -If Manifest is available in your Easypanel template catalog: +If the gateway is available in your Easypanel template catalog: 1. Open your Easypanel project. 2. Choose **Templates**. @@ -45,13 +45,13 @@ If Manifest is available in your Easypanel template catalog: 4. Keep the default image `manifestdotbuild/manifest:6` unless you need another version. 5. Deploy the template. -The template provisions Manifest, PostgreSQL, generated secrets, and a domain proxy on port `2099`. +The template provisions the gateway, PostgreSQL, generated secrets, and a domain proxy on port `2099`. ## Manual deploy Create a PostgreSQL service named `manifest-db`, then create an App service from Docker image `manifestdotbuild/manifest:6`. Set the proxy port to `2099`, add a domain, and mark it as the primary domain. -Set the Manifest environment: +Set the gateway environment: ```env PORT=2099 @@ -74,9 +74,9 @@ Generate two different 64-character secrets with `openssl rand -hex 32`. Use an Add an app volume named `recordings` mounted at `/data/request-recordings` so [request logs](/llm-gateway/docs/request-logs) survive restarts and redeploys. This volume is single-instance; move to S3-compatible storage before running more than one replica. -`AUTH_DB_POOL_MAX` is read by Manifest's Better Auth database pool. Set `MANIFEST_TELEMETRY_DISABLED=1` if you want to disable anonymous self-hosted telemetry. +`AUTH_DB_POOL_MAX` is read by the gateway's Better Auth database pool. Set `MANIFEST_TELEMETRY_DISABLED=1` if you want to disable anonymous self-hosted telemetry. -## Open Manifest +## Open the gateway Open the Easypanel domain and create the first admin account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. @@ -103,7 +103,7 @@ Before submitting a template upstream, verify: - Back up the PostgreSQL service through Easypanel or your server provider. - Use a real HTTPS domain before enabling OAuth providers. - Keep `BETTER_AUTH_SECRET` and `MANIFEST_ENCRYPTION_KEY` as separate values. -- If you deploy more than one Manifest replica, lower database pool sizes or upgrade PostgreSQL connection capacity. +- If you deploy more than one gateway replica, lower database pool sizes or upgrade PostgreSQL connection capacity. Relevant Easypanel docs: diff --git a/deploy/fly.mdx b/deploy/fly.mdx index 76061ac..ab87d5d 100644 --- a/deploy/fly.mdx +++ b/deploy/fly.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Fly.io" +title: "Deploy the LLM Gateway on Fly.io" sidebarTitle: "Fly.io" -description: "Deploy Manifest on Fly.io from the public Docker image with Fly Postgres, generated secrets, and HTTPS." +description: "Deploy the Manifest LLM Gateway from the public Docker image with Fly Postgres, generated secrets, and HTTPS." icon: "/icons/deploy/fly.svg" keywords: [ "Manifest deployment", "Fly.io", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -20,12 +20,12 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/fly" href="https://fly.io/docs/flyctl/install/" > CLI-based deploy using the public Docker image, Fly Postgres, and generated - Manifest secrets. + gateway secrets. ## Overview -Fly.io runs Manifest as a long-running Docker container close to users, with a Fly Postgres database in the same private network. This is not a browser one-click button. The Manifest repository includes a `deploy/fly/deploy.sh` script that creates the app, creates Postgres, attaches `DATABASE_URL`, sets secrets, and deploys the public Docker image. +Fly.io runs the gateway as a long-running Docker container close to users, with a Fly Postgres database in the same private network. This is not a browser one-click button. The Manifest repository includes a `deploy/fly/deploy.sh` script that creates the app, creates Postgres, attaches `DATABASE_URL`, sets secrets, and deploys the public Docker image. ## Prerequisites @@ -48,7 +48,7 @@ FLY_ORG=personal \ The script deploys `docker.io/manifestdotbuild/manifest:6`, provisions Fly Postgres, attaches it as `DATABASE_URL`, and generates separate values for `BETTER_AUTH_SECRET` and `MANIFEST_ENCRYPTION_KEY`. Re-running the script keeps existing secrets instead of rotating sessions or encrypted provider credentials. -## Open Manifest +## Open the gateway Open the deployed app and create the first admin account: @@ -70,7 +70,7 @@ fly logs --app ## Production notes -- The template keeps one Machine running so Manifest is always available for agents. +- The template keeps one Machine running so the gateway is always available for agents. - For production, choose a larger Postgres configuration or Fly Managed Postgres instead of the small script default. - Add a custom domain before configuring OAuth callback URLs. - The deploy script creates a private Tigris bucket, stores its credentials as Fly secrets, and sets `REQUEST_RECORDING_STORAGE=s3` with the Tigris endpoint. [Request logs](/llm-gateway/docs/request-logs) persist across deploys and across scaled Machines. Nothing to attach manually. diff --git a/deploy/gcp.mdx b/deploy/gcp.mdx index 8c3390a..73b3e8c 100644 --- a/deploy/gcp.mdx +++ b/deploy/gcp.mdx @@ -1,10 +1,10 @@ --- -title: "Deploy Manifest on GCP" +title: "Deploy the LLM Gateway on Google Cloud" sidebarTitle: "GCP" -description: "Deploy Manifest on Google Cloud with Cloud Run, Cloud SQL for PostgreSQL, Secret Manager, and the Cloud Shell DeployStack flow." +description: "Deploy the Manifest LLM Gateway with Cloud Run, Cloud SQL for PostgreSQL, Secret Manager, and Cloud Shell." icon: "/icons/deploy/gcp.svg" keywords: - ["Manifest deployment", "GCP", "self-hosted Manifest", "PostgreSQL", "Docker"] + ["Manifest deployment", "GCP", "self-hosted gateway", "PostgreSQL", "Docker"] canonical: "https://manifest.build/llm-gateway/docs/deploy/gcp" --- @@ -19,7 +19,7 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/gcp" ## Overview -The Google Cloud deploy path uses Cloud Shell and DeployStack to run Terraform. It provisions Cloud Run for the Manifest container, Cloud SQL for PostgreSQL, and Secret Manager for runtime secrets. +The Google Cloud deploy path uses Cloud Shell and DeployStack to run Terraform. It provisions Cloud Run for the gateway container, Cloud SQL for PostgreSQL, and Secret Manager for runtime secrets. ## Prerequisites @@ -38,7 +38,7 @@ The walkthrough prompts for: - Google Cloud project. - Region. - Cloud Run service name. -- Manifest image URL. +- Gateway image URL. - Cloud SQL machine tier. - Maximum Cloud Run instances. - Cloud SQL deletion protection. @@ -61,7 +61,7 @@ deploystack install The first deploy usually takes 10-15 minutes because Cloud SQL needs time to provision. The Terraform config patches `BETTER_AUTH_URL` to the final Cloud Run URL after the service is created. -## Open Manifest +## Open the gateway Print the service URL: diff --git a/deploy/heroku.mdx b/deploy/heroku.mdx index 557c9fa..2ac0ae7 100644 --- a/deploy/heroku.mdx +++ b/deploy/heroku.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Heroku" +title: "Deploy the LLM Gateway on Heroku" sidebarTitle: "Heroku" -description: "Deploy Manifest on Heroku with a Deploy to Heroku button, a container-stack web dyno, and Heroku Postgres." +description: "Deploy the Manifest LLM Gateway with a container-stack web dyno and Heroku Postgres." icon: "/icons/deploy/heroku.svg" keywords: [ "Manifest deployment", "Heroku", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -25,7 +25,7 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/heroku" ## Overview -Heroku can deploy Manifest with the repository `app.json` and `heroku.yml`. The deploy button creates a Cedar container-stack app, provisions Heroku Postgres, generates Manifest secrets, and builds a tiny Heroku Dockerfile that uses the public Manifest image. +Heroku can deploy the gateway with the repository `app.json` and `heroku.yml`. The deploy button creates a Cedar container-stack app, provisions Heroku Postgres, generates gateway secrets, and builds a tiny Heroku Dockerfile that uses the public Manifest image. Heroku asks for a few values before it can deploy. Set `BETTER_AUTH_URL` to the public URL for the app name you choose. Heroku dynos have an ephemeral filesystem, so [request logs](/llm-gateway/docs/request-logs) write to external S3-compatible storage rather than local disk: set the bucket, region, and access key for that storage too. @@ -52,7 +52,7 @@ https://.herokuapp.com Heroku generates `BETTER_AUTH_SECRET` and `MANIFEST_ENCRYPTION_KEY`, provisions Postgres as `DATABASE_URL`, builds `Dockerfile.heroku`, and starts the web dyno. -## Open Manifest +## Open the gateway Open the deployed app and create the first admin account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. diff --git a/deploy/koyeb.mdx b/deploy/koyeb.mdx index 34ad2ba..530c67e 100644 --- a/deploy/koyeb.mdx +++ b/deploy/koyeb.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Koyeb" +title: "Deploy the LLM Gateway on Koyeb" sidebarTitle: "Koyeb" -description: "Deploy Manifest on Koyeb from the public Docker image with a Koyeb web service and PostgreSQL connection string." +description: "Deploy the Manifest LLM Gateway from the public Docker image with a Koyeb web service and PostgreSQL." icon: "/icons/deploy/koyeb.svg" keywords: [ "Manifest deployment", "Koyeb", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -25,22 +25,22 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/koyeb" ## Overview -Koyeb can deploy Manifest from the public Docker image with a Deploy to Koyeb button. The button pre-fills the image, HTTP port, and runtime settings, but you must create PostgreSQL separately and replace the placeholder secrets before deploying. +Koyeb can deploy the gateway from the public Docker image with a Deploy to Koyeb button. The button pre-fills the image, HTTP port, and runtime settings, but you must create PostgreSQL separately and replace the placeholder secrets before deploying. ## Prerequisites - A Koyeb account on a paid plan (Pro or higher) with a valid payment method. Since Koyeb joined Mistral AI, deploying requires a paid plan: on an account without one, the deploy link shows an announcement banner instead of the form. - A Koyeb PostgreSQL Database Service. -- Two random 32+ character secrets for Manifest. +- Two random 32+ character secrets for the gateway. - A private S3-compatible bucket and an access key for it. Koyeb instances have no persistent disk, so request logs need external storage. This deployment creates paid resources. ## Create PostgreSQL -In Koyeb, create a PostgreSQL Database Service in the same region you plan to use for Manifest. After it is ready, open the database connection details and copy the connection string. +In Koyeb, create a PostgreSQL Database Service in the same region you plan to use for the gateway. After it is ready, open the database connection details and copy the connection string. -Manifest uses TLS to connect to Koyeb Postgres, so include `sslmode=require` in the connection string. If the copied URL has no query string, append `?sslmode=require`. If it already has query parameters, append `&sslmode=require`. +The gateway uses TLS to connect to Koyeb Postgres, so include `sslmode=require` in the connection string. If the copied URL has no query string, append `?sslmode=require`. If it already has query parameters, append `&sslmode=require`. ## Generate secrets @@ -51,7 +51,7 @@ openssl rand -hex 32 openssl rand -hex 32 ``` -## Deploy Manifest +## Deploy the gateway 1. Open the [Koyeb deploy link](https://app.koyeb.com/deploy?type=docker&image=docker.io%2Fmanifestdotbuild%2Fmanifest%3A6&name=manifest&service_type=web&ports=2099%3Bhttp%3B%2F&env%5BPORT%5D=2099&env%5BDATABASE_URL%5D=postgres%3A%2F%2FUSER%3APASSWORD%40HOST%2FDB%3Fsslmode%3Drequire&env%5BBETTER_AUTH_SECRET%5D=replace-with-openssl-rand-hex-32&env%5BMANIFEST_ENCRYPTION_KEY%5D=replace-with-different-openssl-rand-hex-32&env%5BBETTER_AUTH_URL%5D=https%3A%2F%2F%7B%7B+KOYEB_PUBLIC_DOMAIN+%7D%7D&env%5BMANIFEST_MODE%5D=selfhosted&env%5BBIND_ADDRESS%5D=0.0.0.0&env%5BDB_POOL_MAX%5D=8&env%5BAUTH_DB_POOL_MAX%5D=4&env%5BREQUEST_RECORDING_STORAGE%5D=s3&env%5BREQUEST_RECORDING_S3_BUCKET%5D=replace-with-private-bucket&env%5BREQUEST_RECORDING_S3_ENDPOINT%5D=https%3A%2F%2Fs3.example.com&env%5BREQUEST_RECORDING_S3_REGION%5D=us-east-1&env%5BREQUEST_RECORDING_S3_ACCESS_KEY_ID%5D=replace-with-limited-access-key&env%5BREQUEST_RECORDING_S3_SECRET_ACCESS_KEY%5D=replace-with-secret-key&env%5BREQUEST_RECORDING_S3_FORCE_PATH_STYLE%5D=false). 2. Replace `DATABASE_URL` with your Koyeb Postgres connection string. @@ -64,7 +64,7 @@ openssl rand -hex 32 The button deploys `docker.io/manifestdotbuild/manifest:6` and exposes port `2099` over HTTP. -## Open Manifest +## Open the gateway After the deployment is live, open the public Koyeb domain and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. diff --git a/deploy/railway.mdx b/deploy/railway.mdx index 8e8aaa7..a5c9e10 100644 --- a/deploy/railway.mdx +++ b/deploy/railway.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Railway" +title: "Deploy the LLM Gateway on Railway" sidebarTitle: "Railway" -description: "Deploy Manifest on Railway with the published template, including the Manifest service and PostgreSQL." +description: "Deploy the Manifest LLM Gateway with the published Railway template, PostgreSQL, and request-log storage." icon: "/icons/deploy/railway.svg" keywords: [ "Manifest deployment", "Railway", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -19,13 +19,13 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/railway" icon="external-link" href="https://railway.com/deploy/wild-wild" > - Best one-click path for most users. The template provisions Manifest, + Best one-click path for most users. The template provisions the gateway, PostgreSQL, and a bucket for request logs. ## Overview -Railway is the fastest hosted path for Manifest. Use the published Manifest template to create a project with the Manifest app, a PostgreSQL database, and a storage bucket for [request logs](/llm-gateway/docs/request-logs), then open the generated domain and create the first admin account. +Railway is the fastest hosted path for the gateway. Use the published Manifest template to create a project with the gateway app, a PostgreSQL database, and a storage bucket for [request logs](/llm-gateway/docs/request-logs), then open the generated domain and create the first admin account. ## Prerequisites @@ -36,14 +36,14 @@ Railway is the fastest hosted path for Manifest. Use the published Manifest temp ## Deploy 1. Open the [Manifest Railway template](https://railway.com/deploy/wild-wild). -2. Choose the Railway workspace and project where you want to deploy Manifest. +2. Choose the Railway workspace and project where you want to deploy the gateway. 3. Review the services that will be created. 4. Deploy the template. -5. Wait for the Manifest service and PostgreSQL database to finish provisioning. +5. Wait for the gateway service and PostgreSQL database to finish provisioning. The template includes the app service, the database, and a `messages-recordings` bucket, so you do not need to create PostgreSQL or recording storage separately. -## Open Manifest +## Open the gateway Open the generated Railway domain and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. diff --git a/deploy/render.mdx b/deploy/render.mdx index bcbd4a8..23d1038 100644 --- a/deploy/render.mdx +++ b/deploy/render.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Render" +title: "Deploy the LLM Gateway on Render" sidebarTitle: "Render" -description: "Deploy Manifest on Render with the repository Blueprint, including a web service and Render PostgreSQL." +description: "Deploy the Manifest LLM Gateway with a repository Blueprint, a web service, and Render PostgreSQL." icon: "/icons/deploy/render.svg" keywords: [ "Manifest deployment", "Render", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -25,7 +25,7 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/render" ## Overview -Render can deploy Manifest from the repository Blueprint. The Blueprint creates a web service from the public Manifest Docker image and a Render PostgreSQL database, then wires the runtime environment variables. +Render can deploy the gateway from the repository Blueprint. The Blueprint creates a web service from the public Manifest Docker image and a Render PostgreSQL database, then wires the runtime environment variables. ## Prerequisites @@ -50,7 +50,7 @@ The repository `render.yaml` sets: - `BETTER_AUTH_URL` from the Render external service URL. - `MANIFEST_MODE=selfhosted`. -## Open Manifest +## Open the gateway Open the Render service URL and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. diff --git a/docs.json b/docs.json index 7e8aef7..6a38707 100644 --- a/docs.json +++ b/docs.json @@ -1,6 +1,6 @@ { "$schema": "https://mintlify.com/docs.json", - "name": "Manifest", + "name": "Manifest LLM Gateway", "theme": "maple", "logo": { "light": "/logo/light.svg", @@ -11,7 +11,7 @@ "css": "/style.css", "js": "/scripts/no-tab-jump.js", "banner": { - "content": "**NEW** Manifest is becoming the self-healing layer for APIs. [Read more](https://manifest.build/blog/manifest-is-taking-a-new-direction/)", + "content": "You are reading the **Manifest LLM Gateway documentation**. Looking for Manifest, the self-healing API? [Read the Manifest documentation](https://manifest.build/docs).", "dismissible": false }, "colors": { @@ -42,7 +42,7 @@ "seo": { "metatags": { "canonical": "https://manifest.build", - "og:site_name": "Manifest", + "og:site_name": "Manifest LLM Gateway", "og:type": "website", "twitter:card": "summary_large_image", "twitter:site": "@Manifestforai", @@ -73,12 +73,12 @@ } }, "search": { - "prompt": "Search Manifest docs..." + "prompt": "Search LLM Gateway docs..." }, "navbar": { "links": [ { - "label": "Dashboard", + "label": "Open dashboard", "href": "https://app.manifest.build" }, { @@ -99,7 +99,7 @@ "navigation": { "tabs": [ { - "tab": "Documentation", + "tab": "LLM Gateway", "groups": [ { "group": "Getting Started", diff --git a/errors.mdx b/errors.mdx index 05e5ef9..ed08a4e 100644 --- a/errors.mdx +++ b/errors.mdx @@ -1,14 +1,14 @@ --- -title: "Manifest error codes" +title: "LLM Gateway error codes" sidebarTitle: "Errors" -description: "Reference for every Manifest proxy error code (M001-M500). What you saw, why it happened, and how to fix it. Covers auth, providers, limits, validation." +description: "Reference for errors returned by the Manifest LLM Gateway, with causes and fixes." icon: "circle-alert" keywords: ["Manifest error codes", "M001", "M100", "M102", "M200", "M204", "M302", "M500", "OpenAI compatible errors", "401 Unauthorized", "402 Payment Required", "429 Too Many Requests", "Bearer token error", "chat completions error", "model not available", "troubleshooting", "proxy errors"] canonical: "https://manifest.build/llm-gateway/docs/errors" --- -When Manifest blocks or rejects a request, the response message starts with a code in square brackets: +When the gateway blocks or rejects a request, the response message starts with a code in square brackets: ```text [🦚 Manifest M100] No anthropic API key yet. Add one here: https://app.manifest.build/... @@ -33,7 +33,7 @@ These fire when the bearer token on `/v1/chat/completions` is missing or wrong. ## Providers (M100–M102) -Your Manifest key is fine, but provider credentials are missing or unusable. See [Routing](/llm-gateway/docs/llm-gateway), [API key providers](/llm-gateway/docs/providers/api-key-providers), and [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers). +Your harness key is fine, but provider credentials are missing or unusable. See [Routing](/llm-gateway/docs/llm-gateway), [API key providers](/llm-gateway/docs/providers/api-key-providers), and [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers). | Code | What | |------|------| @@ -55,13 +55,13 @@ You hit a usage cap, rate limit, or Free plan request quota. M200 comes back as ## Request validation and model selection (M300–M303) -M300 means the request body is malformed and surfaces as HTTP 400. M302 and M303 mean Manifest can't reach the model the request asked for. M302 comes back as an HTTP 200 assistant message for every client. M303 surfaces as HTTP 400 to non-chat clients. +M300 means the request body is malformed and surfaces as HTTP 400. M302 and M303 mean the gateway can't reach the model the request asked for. M302 comes back as an HTTP 200 assistant message for every client. M303 surfaces as HTTP 400 to non-chat clients. | Code | What | |------|------| | [M300: Missing messages array](/llm-gateway/docs/errors/M300) | Body has no `messages` array, or it's empty | | [M302: Model not available](/llm-gateway/docs/errors/M302) | Explicit model ID is not available for this harness | -| [M303: Local provider unavailable on Manifest Cloud](/llm-gateway/docs/errors/M303) | Routed to Ollama, LM Studio, or llama.cpp on Cloud | +| [M303: Local provider unavailable on LLM Gateway Cloud](/llm-gateway/docs/errors/M303) | Routed to Ollama, LM Studio, or llama.cpp on Cloud | M301 is missing from that list on purpose. It capped requests at 1000 @@ -71,7 +71,7 @@ M300 means the request body is malformed and surfaces as HTTP 400. M302 and M303 ## Server (M500) -Manifest itself broke. Surfaces as HTTP 500. +The gateway itself broke. Surfaces as HTTP 500. | Code | What | |------|------| diff --git a/errors/M001.mdx b/errors/M001.mdx index 67d828e..67b91d8 100644 --- a/errors/M001.mdx +++ b/errors/M001.mdx @@ -1,7 +1,7 @@ --- title: "M001: Missing Authorization header" sidebarTitle: "M001" -description: "Manifest error M001 fires when /v1/chat/completions is called without an Authorization header. Returns HTTP 401. Fix: send `Authorization: Bearer mnfst_`." +description: "M001 is returned when a request has no Authorization header. Send a Bearer harness key." icon: "key-round" keywords: ["M001", "Manifest M001", "Authorization header missing", "Bearer token", "401 Unauthorized", "OpenAI compatible 401", "Manifest auth error", "chat completions authorization"] @@ -17,7 +17,7 @@ See https://manifest.build/llm-gateway/docs/errors/M001 ## Why it happened -Your client called `/v1/chat/completions` without an `Authorization` header. Manifest checks a harness API key on every request, so the header has to be there. +Your client called `/v1/chat/completions` without an `Authorization` header. The gateway checks a harness API key on every request, so the header has to be there. ## How to fix it diff --git a/errors/M002.mdx b/errors/M002.mdx index 72d2239..bc0649f 100644 --- a/errors/M002.mdx +++ b/errors/M002.mdx @@ -1,7 +1,7 @@ --- title: "M002: Empty Bearer token" sidebarTitle: "M002" -description: "Manifest error M002 fires when the Authorization header is set but the Bearer token is blank. Often an unexpanded env var. Returns HTTP 401." +description: "M002 is returned when the Bearer token is empty, often because an environment variable was not expanded." icon: "key-round" keywords: ["M002", "Manifest M002", "empty Bearer token", "blank token", "401 Unauthorized", "MNFST_KEY env var", "Manifest auth error", "OpenAI compatible 401"] diff --git a/errors/M003.mdx b/errors/M003.mdx index e938a89..1e26629 100644 --- a/errors/M003.mdx +++ b/errors/M003.mdx @@ -1,7 +1,7 @@ --- title: "M003: Invalid key format" sidebarTitle: "M003" -description: "Manifest error M003 fires when the Bearer token doesn't start with mnfst_. Often a provider key (sk-...) pasted in the wrong field. Returns HTTP 401." +description: "M003 is returned when the Bearer token does not start with mnfst_. Use a harness key, not a provider key." icon: "key-round" keywords: ["M003", "Manifest M003", "invalid API key format", "mnfst_ prefix", "wrong API key", "OpenAI sk- key", "401 Unauthorized", "Manifest auth error"] @@ -17,7 +17,7 @@ See https://manifest.build/llm-gateway/docs/errors/M003 ## Why it happened -Manifest keys always start with `mnfst_`. Your token has a different prefix, which almost always means a provider key (OpenAI, Anthropic, etc.) got pasted into the wrong field. +harness keys always start with `mnfst_`. Your token has a different prefix, which almost always means a provider key (OpenAI, Anthropic, etc.) got pasted into the wrong field. ## How to fix it diff --git a/errors/M004.mdx b/errors/M004.mdx index 7734989..0b53349 100644 --- a/errors/M004.mdx +++ b/errors/M004.mdx @@ -1,7 +1,7 @@ --- title: "M004: Key expired" sidebarTitle: "M004" -description: "Manifest error M004 fires when the API key you sent is past its expiration date. Returns HTTP 401. Fix: rotate the key in the dashboard." +description: "M004 is returned when the harness key has expired. Rotate the key in the dashboard." icon: "key-round" keywords: ["M004", "Manifest M004", "API key expired", "key rotation", "401 Unauthorized", "Manifest auth error", "expired token"] @@ -17,7 +17,7 @@ See https://manifest.build/llm-gateway/docs/errors/M004 ## Why it happened -Manifest keys can carry an expiration date. Yours is past that date, so it no longer authenticates anything. +harness keys can carry an expiration date. Yours is past that date, so it no longer authenticates anything. ## How to fix it diff --git a/errors/M005.mdx b/errors/M005.mdx index 36811ef..8263a0b 100644 --- a/errors/M005.mdx +++ b/errors/M005.mdx @@ -1,7 +1,7 @@ --- title: "M005: Key not recognized" sidebarTitle: "M005" -description: "Manifest error M005 fires when the mnfst_ key has the right shape but no matching harness exists. Often a rotated or deleted key. Returns HTTP 401." +description: "M005 is returned when no harness matches a correctly shaped mnfst_ key." icon: "key-round" keywords: ["M005", "Manifest M005", "invalid API key", "rotated key", "deleted agent", "deleted harness", "401 Unauthorized", "key not found", "Manifest auth error"] diff --git a/errors/M100.mdx b/errors/M100.mdx index e299f13..1b15f47 100644 --- a/errors/M100.mdx +++ b/errors/M100.mdx @@ -1,7 +1,7 @@ --- title: "M100: Provider API key missing" sidebarTitle: "M100" -description: "Manifest error M100 fires when routing picks a provider that has no API key for this harness (OpenAI, Anthropic, Gemini, OpenRouter, etc). Fix: add the key from Providers → Usage-based." +description: "M100 is returned when routing selects a provider for which the harness has no API key." icon: "plug-zap" keywords: ["M100", "Manifest M100", "provider API key missing", "no OpenAI key", "no Anthropic key", "no Gemini key", "OpenRouter key", "Manifest routing", "provider credentials"] @@ -15,11 +15,11 @@ canonical: "https://manifest.build/llm-gateway/docs/errors/M100" See https://manifest.build/llm-gateway/docs/errors/M100 ``` -The provider name (`anthropic` in the example) varies based on which provider Manifest selected. +The provider name (`anthropic` in the example) varies based on which provider the gateway selected. ## Why it happened -Routing picked a provider that has no key on file for this harness. Manifest won't silently swap vendors on you, so the request stops here. +Routing picked a provider that has no key on file for this harness. The gateway won't silently swap vendors on you, so the request stops here. ## How to fix it diff --git a/errors/M101.mdx b/errors/M101.mdx index 98d238e..2308934 100644 --- a/errors/M101.mdx +++ b/errors/M101.mdx @@ -1,7 +1,7 @@ --- title: "M101: No providers configured" sidebarTitle: "M101" -description: "Manifest error M101 fires when an authenticated harness has zero providers connected. Common on first run. Fix: connect a provider from the Providers section of the dashboard." +description: "M101 is returned when the authenticated harness has no providers configured." icon: "plug-zap" keywords: ["M101", "Manifest M101", "no providers configured", "first-time setup", "empty agent", "empty harness", "Manifest routing", "connect provider", "OpenRouter setup"] diff --git a/errors/M102.mdx b/errors/M102.mdx index 675af13..90b2271 100644 --- a/errors/M102.mdx +++ b/errors/M102.mdx @@ -1,7 +1,7 @@ --- title: "M102: Provider subscription credentials unusable" sidebarTitle: "M102" -description: "Manifest error M102 fires when routing picks a subscription provider whose OAuth or refresh credentials cannot be used. Fix: reconnect the subscription from Providers → Subscriptions." +description: "M102 is returned when subscription credentials cannot be refreshed or used." icon: "plug-zap" keywords: [ @@ -25,11 +25,11 @@ canonical: "https://manifest.build/llm-gateway/docs/errors/M102" See https://manifest.build/llm-gateway/docs/errors/M102 ``` -The provider name (`openai` in the example) varies based on which subscription route Manifest selected. +The provider name (`openai` in the example) varies based on which subscription route the gateway selected. ## Why it happened -Routing elected a **subscription** connection (ChatGPT OAuth, Claude Max token, Copilot device code, and similar), and Manifest still has a stored credential blob for it — but that credential cannot produce a usable access token. +Routing elected a **subscription** connection (ChatGPT OAuth, Claude Max token, Copilot device code, and similar), and the gateway still has a stored credential blob for it — but that credential cannot produce a usable access token. Typical causes: @@ -39,7 +39,7 @@ Typical causes: This is different from [M100](/llm-gateway/docs/errors/M100): M100 means there is no key on file. M102 means a subscription row exists and was elected, but the unwrap/refresh step failed. -When fallback routes are configured, Manifest records the M102 hop and continues the chain. You only see a terminal M102 response when no usable fallback remains. +When fallback routes are configured, the gateway records the M102 hop and continues the chain. You only see a terminal M102 response when no usable fallback remains. ## How to fix it diff --git a/errors/M200.mdx b/errors/M200.mdx index 8af7aa1..474eb01 100644 --- a/errors/M200.mdx +++ b/errors/M200.mdx @@ -1,7 +1,7 @@ --- title: "M200: Usage limit exceeded" sidebarTitle: "M200" -description: "Manifest error M200 fires when a harness crosses a cost or token cap you configured on the Limits page. It comes back as an HTTP 200 assistant message, not a 429. Fix: raise the threshold or wait for the period to reset." +description: "M200 is returned when a harness exceeds a cost or token limit configured in the dashboard." icon: "gauge" keywords: ["M200", "Manifest M200", "usage limit exceeded", "cost limit", "token limit", "spending cap", "budget cap", "monthly budget LLM", "Manifest limits", "HTTP 200 assistant message"] @@ -21,9 +21,9 @@ The response arrives as HTTP 200 with this text in the assistant message content ## Why it happened -You set a limit on the harness's [Limits](/llm-gateway/docs/observability#spend-alerts) page (say, "$10/day" or "500k tokens/hour"), and the current period's usage just crossed it. Manifest blocks every following request until the period resets or you raise the cap. +You set a limit on the harness's [Limits](/llm-gateway/docs/observability#spend-alerts) page (say, "$10/day" or "500k tokens/hour"), and the current period's usage just crossed it. The gateway blocks every following request until the period resets or you raise the cap. -This is different from [M204](/llm-gateway/docs/errors/M204), which is the Manifest Cloud Free plan monthly request quota, and different from provider 402 errors such as "insufficient credits" from OpenRouter. +This is different from [M204](/llm-gateway/docs/errors/M204), which is the LLM Gateway Cloud Free plan monthly request quota, and different from provider 402 errors such as "insufficient credits" from OpenRouter. ## How to fix it diff --git a/errors/M201.mdx b/errors/M201.mdx index ccf9184..79f6ca9 100644 --- a/errors/M201.mdx +++ b/errors/M201.mdx @@ -1,7 +1,7 @@ --- title: "M201: Per-user rate limit exceeded" sidebarTitle: "M201" -description: "Manifest error M201 fires when one user sends more than 200 requests per minute. Returns HTTP 429. Fix: back off and retry." +description: "M201 is returned when one user exceeds 200 requests per minute." icon: "timer" keywords: ["M201", "Manifest M201", "rate limit", "429 Too Many Requests", "per-user rate limit", "throttling", "exponential backoff", "Manifest limits"] @@ -19,12 +19,12 @@ The HTTP status is `429`, which most SDKs treat as retryable and back off from o ## Why it happened -Manifest caps each user at 200 requests per minute. The cap is there mostly to keep runaway loops from melting the proxy. Your agent crossed it inside a 60-second window. +The gateway caps each user at 200 requests per minute. The cap is there mostly to keep runaway loops from melting the proxy. Your agent crossed it inside a 60-second window. ## How to fix it 1. Add a small delay or exponential backoff on retry. Most SDKs handle this automatically when they see a `429`. -2. If parallel workers are hitting Manifest, throttle them so they share the budget. +2. If parallel workers are hitting the gateway, throttle them so they share the budget. 3. The 200 requests/minute cap is a fixed guardrail. No env var or setting changes it, self-hosted included. Lean on the backoff in step 1. ## Related diff --git a/errors/M202.mdx b/errors/M202.mdx index e5a7e00..6f240cc 100644 --- a/errors/M202.mdx +++ b/errors/M202.mdx @@ -1,7 +1,7 @@ --- title: "M202: Per-IP rate limit exceeded" sidebarTitle: "M202" -description: "Manifest error M202 fires when one IP sends more than 500 requests per minute across all harnesses. Returns HTTP 429. Fix: back off and stagger workers." +description: "M202 is returned when one IP exceeds 500 requests per minute across all harnesses." icon: "timer" keywords: ["M202", "Manifest M202", "per-IP rate limit", "shared IP", "429 Too Many Requests", "Manifest limits", "abuse protection"] diff --git a/errors/M203.mdx b/errors/M203.mdx index 08aced8..5b6a647 100644 --- a/errors/M203.mdx +++ b/errors/M203.mdx @@ -1,7 +1,7 @@ --- title: "M203: Concurrency limit exceeded" sidebarTitle: "M203" -description: "Manifest error M203 fires when a workspace has more than 10 in-flight requests. Returns HTTP 429. Common with parallel streaming. Fix: reduce parallelism." +description: "M203 is returned when a workspace has more than 10 requests in flight." icon: "timer" keywords: ["M203", "Manifest M203", "concurrency limit", "in-flight requests", "429 Too Many Requests", "parallel streams", "Manifest limits"] @@ -17,7 +17,7 @@ See https://manifest.build/llm-gateway/docs/errors/M203 ## Why it happened -Manifest caps in-flight requests at 10 per workspace, shared by all its harnesses. Streaming completions hold a slot until the stream ends, so an agent that fires off parallel streams can hit this fast. +The gateway caps in-flight requests at 10 per workspace, shared by all its harnesses. Streaming completions hold a slot until the stream ends, so an agent that fires off parallel streams can hit this fast. ## How to fix it diff --git a/errors/M204.mdx b/errors/M204.mdx index 78e2b1e..82a784c 100644 --- a/errors/M204.mdx +++ b/errors/M204.mdx @@ -1,7 +1,7 @@ --- title: "M204: Monthly request limit reached" sidebarTitle: "M204" -description: "Manifest error M204 fires when a Manifest Cloud Free workspace uses all monthly requests. Returns HTTP 402 with PLAN_LIMIT_REQUESTS. Fix: upgrade to Pro or wait for the monthly reset." +description: "M204 is returned when a free LLM Gateway Cloud workspace reaches its monthly request allowance." icon: "badge-alert" keywords: ["M204", "Manifest M204", "PLAN_LIMIT_REQUESTS", "monthly request limit", "free plan request limit", "Manifest billing", "402 Payment Required", "upgrade to Pro"] @@ -19,7 +19,7 @@ SDK and tool callers receive an HTTP 402 JSON error with `error.code` set to `PL ## Why it happened -Your Manifest Cloud workspace reached the Free plan monthly request quota. This is a Manifest plan restriction, not a provider error and not a limit you configured on a harness. +Your LLM Gateway Cloud workspace reached the Free plan monthly request quota. This is a Cloud plan restriction, not a provider error and not a limit you configured on a harness. ## How to fix it diff --git a/errors/M300.mdx b/errors/M300.mdx index b5705a3..914cb1c 100644 --- a/errors/M300.mdx +++ b/errors/M300.mdx @@ -1,7 +1,7 @@ --- title: "M300: Missing messages array" sidebarTitle: "M300" -description: "Manifest error M300 fires when the /v1/chat/completions body has no messages array (missing, not an array, or empty). Returns HTTP 400." +description: "M300 is returned when /v1/chat/completions receives no usable messages array." icon: "message-square-warning" keywords: ["M300", "Manifest M300", "messages array required", "empty messages", "400 Bad Request", "OpenAI compatible API", "chat completions validation", "Manifest validation"] @@ -17,7 +17,7 @@ See https://manifest.build/llm-gateway/docs/errors/M300 ## Why it happened -The OpenAI-compatible chat completions API requires a `messages` array with at least one entry. Manifest got a body where `messages` was missing, not an array, or empty. +The OpenAI-compatible chat completions API requires a `messages` array with at least one entry. The gateway received a body where `messages` was missing, not an array, or empty. ## How to fix it @@ -36,6 +36,6 @@ The OpenAI SDK handles this for you. If you're hand-rolling the request, eyeball ## Related -- [M303: Local provider unavailable on Manifest Cloud](/llm-gateway/docs/errors/M303) +- [M303: Local provider unavailable on LLM Gateway Cloud](/llm-gateway/docs/errors/M303) - [API reference](/llm-gateway/docs/reference/api) - [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M302.mdx b/errors/M302.mdx index 9e3ac97..c64d54b 100644 --- a/errors/M302.mdx +++ b/errors/M302.mdx @@ -1,7 +1,7 @@ --- title: "M302: Model not available" sidebarTitle: "M302" -description: "Manifest error M302 fires when an explicit model ID is not available for the authenticated harness. Fix: use GET /v1/models or make the provider available for this harness." +description: "M302 is returned when the requested model is unavailable to the authenticated harness." icon: "message-square-warning" keywords: ["M302", "Manifest M302", "model not available", "model not found", "GET /v1/models", "direct model routing", "provider not connected", "provider not enabled", "agent-visible models", "harness-visible models", "Manifest validation"] @@ -25,13 +25,13 @@ This can happen when: 1. The model ID is misspelled or retired. 2. The provider is not connected yet, or it is connected but not enabled for this harness. -3. You used a provider-native name when Manifest expects the exact ID returned by `GET /v1/models`. -4. Manifest refuses the request instead of falling back to automatic routing. +3. You used a provider-native name when the gateway expects the exact ID returned by `GET /v1/models`. +4. The gateway refuses the request instead of falling back to automatic routing. ## How to fix it -1. Call `GET /v1/models` with the same Manifest harness key. -2. Use `auto` to let Manifest route, or copy one of the listed model IDs exactly. +1. Call `GET /v1/models` with the same harness key. +2. Use `auto` to let the gateway route, or copy one of the listed model IDs exactly. 3. If the model should be available, open the dashboard link in the error and enable or connect that provider for this harness. 4. Retry the request. @@ -40,5 +40,5 @@ This can happen when: - [Routing: Route a specific model](/llm-gateway/docs/llm-gateway#route-a-specific-model) - [API reference: Listing models](/llm-gateway/docs/reference/api#listing-models) - [M300: Missing messages array](/llm-gateway/docs/errors/M300) -- [M303: Local provider unavailable on Manifest Cloud](/llm-gateway/docs/errors/M303) +- [M303: Local provider unavailable on LLM Gateway Cloud](/llm-gateway/docs/errors/M303) - [All error codes](/llm-gateway/docs/errors) diff --git a/errors/M303.mdx b/errors/M303.mdx index b1cf4df..6ed95d0 100644 --- a/errors/M303.mdx +++ b/errors/M303.mdx @@ -1,7 +1,7 @@ --- -title: "M303: Local provider unavailable on Manifest Cloud" +title: "M303: Local provider unavailable on Cloud" sidebarTitle: "M303" -description: "Manifest error M303 fires when a Manifest Cloud request routes to Ollama, LM Studio, or llama.cpp. Returns HTTP 400. Fix: self-host, or expose the runtime as a custom provider." +description: "M303 is returned when LLM Gateway Cloud is asked to use Ollama, LM Studio, or llama.cpp." icon: "server-off" keywords: ["M303", "Manifest M303", "local provider cloud", "Ollama Manifest Cloud", "LM Studio", "llama.cpp", "400 Bad Request", "custom provider", "tunnel"] @@ -17,9 +17,9 @@ See https://manifest.build/llm-gateway/docs/errors/M303 ## Why it happened -The built-in [local provider](/llm-gateway/docs/providers/local-models) tiles (Ollama, LM Studio, llama.cpp) route to `localhost`. On a self-hosted install that's your machine, which is the whole point. On Manifest Cloud, `localhost` is our server, which has no model running on it and no route back to your laptop. +The built-in [local provider](/llm-gateway/docs/providers/local-models) tiles (Ollama, LM Studio, llama.cpp) route to `localhost`. On a self-hosted install that's your machine, which is the whole point. On LLM Gateway Cloud, `localhost` is our server, which has no model running on it and no route back to your laptop. -Rather than dial an address that can't be yours, Manifest stops the request here. +Rather than dial an address that can't be yours, the gateway stops the request here. ## How to fix it diff --git a/errors/M500.mdx b/errors/M500.mdx index 9a42763..defa66a 100644 --- a/errors/M500.mdx +++ b/errors/M500.mdx @@ -1,7 +1,7 @@ --- title: "M500: Internal server error" sidebarTitle: "M500" -description: "Manifest error M500 is a generic 500 fallback for unhandled errors in the proxy. Usually transient. Fix: retry, then check backend logs on self-hosted." +description: "M500 is returned when the gateway encounters an unhandled internal error." icon: "server-crash" keywords: ["M500", "Manifest M500", "internal server error", "500 Internal Server Error", "5xx", "proxy crash", "Manifest server error", "self-hosted debugging"] @@ -17,7 +17,7 @@ See https://manifest.build/llm-gateway/docs/errors/M500 ## Why it happened -Manifest hit an unexpected error while handling your request. Usually that's a database hiccup, a downstream provider returning something unparseable, or a real bug. The actual stack trace is logged server-side, but masked in the response so we don't leak internals. +The gateway hit an unexpected error while handling your request. Usually that's a database hiccup, a downstream provider returning something unparseable, or a real bug. The actual stack trace is logged server-side, but masked in the response so we don't leak internals. ## How to fix it diff --git a/integrations/n8n.mdx b/integrations/n8n.mdx index 21148f0..b8f28dc 100644 --- a/integrations/n8n.mdx +++ b/integrations/n8n.mdx @@ -1,6 +1,6 @@ --- title: "n8n" -description: "Use Manifest as the chat model behind n8n AI Agents, or call it directly from a workflow, with the n8n-nodes-manifest community node." +description: "Use the Manifest LLM Gateway as the model endpoint for an n8n AI Agent or call it directly from a workflow." icon: "workflow" keywords: ["n8n", "n8n community node", "n8n-nodes-manifest", "n8n AI Agent", "Manifest Chat Model", "n8n self-hosted", "n8n LLM router", "n8n OpenAI alternative"] @@ -12,14 +12,14 @@ canonical: "https://manifest.build/llm-gateway/docs/integrations/n8n" -Manifest ships as a verified n8n community node, [`n8n-nodes-manifest`](https://www.npmjs.com/package/n8n-nodes-manifest). It installs from n8n's own **Community Nodes** screen on n8n Cloud and on self-hosted n8n alike, and gives you two nodes: +The gateway ships as a verified n8n community node, [`n8n-nodes-manifest`](https://www.npmjs.com/package/n8n-nodes-manifest). It installs from n8n's own **Community Nodes** screen on n8n Cloud and on self-hosted n8n alike, and gives you two nodes: - A language-model sub-node. Plug it into the **AI Agent** or **Basic LLM Chain** node in place of the OpenAI Chat Model, and Manifest routes every request the agent makes. + A language-model sub-node. Plug it into the **AI Agent** or **Basic LLM Chain** node in place of the OpenAI Chat Model, and the gateway routes every request the agent makes. - An action node for calling Manifest directly from a workflow: list models, create a chat completion, or create a Responses API call. + An action node for calling the gateway directly from a workflow: list models, create a chat completion, or create a Responses API call. @@ -71,24 +71,24 @@ Select **Test** on the credential. It calls `GET /v1/models`, so a passing test ### Reaching a self-hosted instance from n8n -The credential test fails with a connection error when n8n cannot reach Manifest. Where the two run decides the address: +The credential test fails with a connection error when n8n cannot reach the gateway. Where the two run decides the address: -- **n8n and Manifest in Docker on the same host.** `localhost` inside the n8n container is n8n itself. Put both containers on a shared Docker network and use the Manifest service name as the host, or use `http://host.docker.internal:2099` on Docker Desktop. -- **n8n on another machine.** The bundled compose file binds Manifest to `127.0.0.1` only. Set `HOST_BIND_ADDRESS=0.0.0.0` in Manifest's `.env`, as described in [Exposing on the LAN](/llm-gateway/docs/self-hosted#exposing-on-the-lan), so the port answers on the network. +- **n8n and the gateway in Docker on the same host.** `localhost` inside the n8n container is n8n itself. Put both containers on a shared Docker network and use the gateway service name as the host, or use `http://host.docker.internal:2099` on Docker Desktop. +- **n8n on another machine.** The bundled compose file binds the gateway to `127.0.0.1` only. Set `HOST_BIND_ADDRESS=0.0.0.0` in the gateway's `.env`, as described in [Exposing on the LAN](/llm-gateway/docs/self-hosted#exposing-on-the-lan), so the port answers on the network. - **Plain `http://` is fine.** The calls are server to server. The dashboard's browser security rules do not apply to them. -The keyless loopback shortcut that a development build of Manifest accepts does not exist in the production image. n8n always needs a real `mnfst_` key. +The keyless loopback shortcut that a development build of the gateway accepts does not exist in the production image. n8n always needs a real `mnfst_` key. -## Use Manifest behind an AI Agent +## Use the gateway behind an AI Agent On an **AI Agent** or **Basic LLM Chain** node, select the **Model** connector and choose **Manifest Chat Model**. - Select the Manifest credential. The model list is loaded from your instance. Leave it on `auto` to let your [routing rules](/llm-gateway/docs/llm-gateway#routing) pick the model, or choose a model ID to pin one. + Select the **Manifest API** credential. The model list is loaded from your instance. Leave it on `auto` to let your [routing rules](/llm-gateway/docs/llm-gateway#routing) pick the model, or choose a model ID to pin one. Streaming and tool calling work as they do with the OpenAI Chat Model, so agent tools run unchanged. Each call appears on the harness's **Requests** page with the model that served it and its cost. @@ -97,7 +97,7 @@ The keyless loopback shortcut that a development build of Manifest accepts does An option on the sub-node switches it from Chat Completions to the [Responses API](/llm-gateway/docs/reference/api#endpoints). Leave it off unless a model you route to needs Responses. -## Call Manifest from a workflow +## Call the gateway from a workflow The **Manifest** action node has three operations: @@ -111,7 +111,7 @@ For a first test, choose **Create Chat Completion**, set **Model** to `auto`, an ```json [ - { "role": "user", "content": "Reply with one sentence confirming that Manifest is connected." } + { "role": "user", "content": "Reply with one sentence confirming that the gateway is connected." } ] ``` diff --git a/introduction.mdx b/introduction.mdx index 216e522..a05b54e 100644 --- a/introduction.mdx +++ b/introduction.mdx @@ -1,7 +1,7 @@ --- -title: "Manifest documentation" +title: "Manifest LLM Gateway documentation" sidebarTitle: "Introduction" -description: "Open-source LLM router for agents. Connect your subscriptions, API keys, and local models to any harness through one endpoint, with fallbacks and spend limits." +description: "Connect models and subscriptions behind one OpenAI- and Anthropic-compatible endpoint, with routing, fallbacks, limits, and Autofix." icon: "house" keywords: ["LLM router", "AI subscriptions", "model routing", "open source", "LLM gateway", "quickstart", "first request", "create a harness", "gateway base URL"] @@ -13,9 +13,9 @@ canonical: "https://manifest.build/llm-gateway/docs/introduction" -Manifest is an open-source LLM router for agents and AI apps. Connect the providers you already use — subscriptions like ChatGPT or Claude, pay-per-token API keys, local models, and custom endpoints — and route every request through one OpenAI- and Anthropic-compatible URL. +The Manifest LLM Gateway connects the models and subscriptions you already use behind one OpenAI- and Anthropic-compatible endpoint. -Send `auto` as the model and Manifest routes each request by the rules you set, with fallbacks when one fails. Pin a real model ID instead, and the request goes straight to that model, with no routing and no fallback. Either way, you cap what each harness spends and see the cost of every request. +Send `auto` to apply your routing rules and fallbacks, or name a model to send the request there directly. Either way, you can cap what each harness spends and see the cost of every request. ## Key features @@ -39,10 +39,10 @@ Send `auto` as the model and Manifest routes each request by the rules you set, ## Cloud or self-hosted? -Manifest is open source and runs two ways. Both speak the same API, so pointing a client at one or the other is a URL change. +The gateway is open source and runs two ways. Both speak the same API, so pointing a client at one or the other is a URL change. - + Runs on our servers. Sign up, connect a provider, and you're routing. Recommended if you're starting out. @@ -52,7 +52,7 @@ Manifest is open source and runs two ways. Both speak the same API, so pointing -On the cloud there is nothing to install and nothing to configure on a server. You sign up, connect your providers, point your harness at the gateway URL, and everything else — routing, limits, [Autofix](/llm-gateway/docs/autofix), alerts — is a setting in the dashboard. The [environment variables](/llm-gateway/docs/reference/environment-variables) throughout these docs apply only to a Manifest instance you run yourself. +On Cloud there is nothing to install or configure on a server. You sign up, connect your providers, and point your harness at the gateway URL. Routing, limits, [Autofix](/llm-gateway/docs/autofix), and alerts are settings in the dashboard. The [environment variables](/llm-gateway/docs/reference/environment-variables) throughout these docs apply only to a self-hosted gateway. Cloud is also subject to [plan limits](/llm-gateway/docs/errors/M204): the free plan includes 10,000 routed requests per calendar month, shown by the usage meter in the dashboard sidebar and reset at 00:00 UTC on the 1st. And it can't reach [local models](/llm-gateway/docs/providers/local-models) running on your machine. A self-hosted instance has neither restriction. @@ -63,7 +63,7 @@ Cloud is also subject to [plan limits](/llm-gateway/docs/errors/M204): the free Sign up at [app.manifest.build](https://app.manifest.build) and a guided setup walks you through it. Step one creates a harness. Then connect a provider, and give your default tier a model to route to. The last step shows the gateway address and the harness key, which starts with `mnfst_`. A harness you create later reuses the providers your workspace already has. - The Manifest guided setup on step one, Create your first harness, with the five setup steps listed in the sidebar + The LLM Gateway guided setup on step one, Create your first harness, with the five setup steps listed in the sidebar ```bash @@ -73,12 +73,12 @@ curl -X POST https://app.manifest.build/v1/chat/completions \ -d '{"model": "auto", "messages": [{"role": "user", "content": "Hello"}]}' ``` -`auto` asks Manifest to route the request. The request appears on the Requests page with the model that served it and its cost. +`auto` asks the gateway to route the request. The request appears on the **Requests** page with the model that served it and its cost.
-Install Manifest first: [Self-hosted](/llm-gateway/docs/self-hosted) covers Docker and the managed platforms. Your instance starts empty. [First request](/llm-gateway/docs/self-hosted#first-request) takes you from there to a routed request, against `http://localhost:2099/v1`. +Install the gateway first. [Self-hosted](/llm-gateway/docs/self-hosted) covers Docker and the managed platforms. Your instance starts empty. [First request](/llm-gateway/docs/self-hosted#first-request) takes you from there to a routed request against `http://localhost:2099/v1`.
diff --git a/llm-gateway.mdx b/llm-gateway.mdx index 6fa8c30..0d2b215 100644 --- a/llm-gateway.mdx +++ b/llm-gateway.mdx @@ -1,13 +1,13 @@ --- title: "LLM Gateway" -description: "Every request passes through the gateway: limits check it, routing picks the model, and fallback catches it when that model fails." +description: "Route requests across models, enforce limits, and use fallbacks when a provider fails." icon: "split" keywords: ["LLM gateway", "model routing", "LLM router", "fallback chain", "hard limit", "custom tier", "default model", "fallback exhausted", "429 too many requests"] canonical: "https://manifest.build/llm-gateway/docs/llm-gateway" --- -Point your client at Manifest instead of a provider, and every request goes through the gateway. Three things happen there: limits decide whether the request runs at all, routing picks the model, and fallback steps in when that model fails. +Point your client at the gateway instead of a provider, and every request goes through the gateway. Three things happen there: limits decide whether the request runs at all, routing picks the model, and fallback steps in when that model fails. ## How a request flows @@ -22,7 +22,7 @@ Point your client at Manifest instead of a provider, and every request goes thro A custom tier matches on a request header, or the request goes to your default tier. Send a real model ID instead of `auto` to skip routing. A matching custom tier header still wins. - Manifest calls the resolved provider with your credentials and streams the response back. + The gateway calls the resolved provider with your credentials and streams the response back. If the provider returns an error, [Autofix](/llm-gateway/docs/autofix) may repair and resend the request once. Anything still failing moves down the fallback chain. @@ -33,7 +33,7 @@ Routing runs in-process. There's no extra network call and no added latency. ## Routing -Instead of hard-coding one model into every client, you point your client at Manifest and let it pick. Send `auto` as the model, and Manifest resolves the real model based on the rules you set on the dashboard **Routing** page. +Instead of hard-coding one model into every client, you point your client at the gateway and let it pick. Send `auto` as the model, and the gateway resolves the real model based on the rules you set on the dashboard **Routing** page. @@ -50,7 +50,7 @@ Every harness has a default tier: one model plus up to five fallbacks. You set i ### Custom tiers -Custom tiers route by request header. You create a tier on the dashboard, give it a header key and value, and pin it to a model with its own fallbacks. When an incoming request carries that header, Manifest sends it to that tier's model instead of the default. The header wins even when the body names an explicit model ID. +Custom tiers route by request header. You create a tier on the dashboard, give it a header key and value, and pin it to a model with its own fallbacks. When an incoming request carries that header, the gateway sends it to that tier's model instead of the default. The header wins even when the body names an explicit model ID. The header key is yours to choose (lowercase letters, numbers, and hyphens). A few names are reserved and rejected, including `authorization`, `cookie`, and `x-api-key`. Send the header from your client like any other: @@ -73,7 +73,7 @@ Create as many tiers as you need, each with its own model and parameters. This i ### Route a specific model -To skip routing for a single request, send a real model ID instead of `auto`. Manifest forwards it straight to that model's provider, with no tier lookup and no fallbacks. A matching [custom tier](#custom-tiers) header is the exception. The request then runs on the tier's model and the tier's fallbacks. Call [`GET /v1/models`](/llm-gateway/docs/reference/api#listing-models) to list the model IDs your harness can reach. +To skip routing for a single request, send a real model ID instead of `auto`. The gateway forwards it straight to that model's provider, with no tier lookup and no fallbacks. A matching [custom tier](#custom-tiers) header is the exception. The request then runs on the tier's model and the tier's fallbacks. Call [`GET /v1/models`](/llm-gateway/docs/reference/api#listing-models) to list the model IDs your harness can reach. ```bash curl -X POST https://app.manifest.build/v1/chat/completions \ @@ -87,11 +87,11 @@ curl -X POST https://app.manifest.build/v1/chat/completions \ The response comes back with `X-Manifest-Tier: direct`, so you can tell a direct call from a routed one. A direct model ID works on all three endpoints: `/v1/chat/completions`, `/v1/responses`, and the Anthropic `/v1/messages`. -The model list is scoped to the harness key. A model may be missing because its provider is not connected yet, or because the provider exists in your workspace but is not enabled for this harness. If you send a model ID that has no connected provider route, Manifest returns [M302: Model not available](/llm-gateway/docs/errors/M302). Send `auto` to use routing. +The model list is scoped to the harness key. A model may be missing because its provider is not connected yet, or because the provider exists in your workspace but is not enabled for this harness. If you send a model ID that has no connected provider route, the gateway returns [M302: Model not available](/llm-gateway/docs/errors/M302). Send `auto` to use routing. ## Fallback -When a model fails (provider outage, rate limit, bad request), Manifest retries with a backup model from the same tier. Your agent gets a response instead of an error. +When a model fails (provider outage, rate limit, bad request), the gateway retries with a backup model from the same tier. Your agent gets a response instead of an error. ### What triggers a fallback @@ -132,9 +132,9 @@ Fallback models are configured **per tier**. Each tier holds up to **5 fallback ### Hung providers and the per-attempt timeout -A provider that opens a connection but never returns eventually triggers a fallback via Manifest's per-attempt timeout (default 180 seconds), which surfaces as a synthetic `504 Gateway Timeout` and moves to the next model in the chain. +A provider that opens a connection but never returns eventually triggers a fallback via the gateway's per-attempt timeout (default 180 seconds), which surfaces as a synthetic `504 Gateway Timeout` and moves to the next model in the chain. -If your upstream client has its own timeout that fires at the same time, the client may disconnect first and Manifest will give up before reaching a healthy fallback. On self-hosted installs, lower [`PROVIDER_TIMEOUT_MS`](/llm-gateway/docs/reference/environment-variables) strictly below your client's timeout so the chain has room to run inside the client's window. +If your upstream client has its own timeout that fires at the same time, the client may disconnect first and the gateway will give up before reaching a healthy fallback. On self-hosted installs, lower [`PROVIDER_TIMEOUT_MS`](/llm-gateway/docs/reference/environment-variables) strictly below your client's timeout so the chain has room to run inside the client's window. ### Response headers @@ -151,7 +151,7 @@ A limit rule can block requests as well as email you. When a rule's action inclu The check runs before any provider is contacted, so a blocked request costs nothing. The block clears on its own at the start of the next period, or immediately if you raise the threshold. - On Manifest Cloud, running out of the Free plan's monthly requests is a + On LLM Gateway Cloud, running out of the Free plan's monthly requests is a different thing entirely: that returns [M204](/llm-gateway/docs/errors/M204) with HTTP `402` and isn't something you configured. diff --git a/llm-gateway/docs/autofix.mdx b/llm-gateway/docs/autofix.mdx index 1a34e7e..ef42075 100644 --- a/llm-gateway/docs/autofix.mdx +++ b/llm-gateway/docs/autofix.mdx @@ -1,6 +1,6 @@ --- title: "Autofix" -description: "Repair a malformed request and resend it once, before the fallback chain runs. Turn it on per harness." +description: "Use Manifest Autofix through the LLM Gateway to repair a malformed request and retry it once before fallback." icon: "wrench" keywords: ["auto-fix", "autofix", "self-healing", "LLM request repair", "malformed request", "400 bad request", "422 unprocessable", "model not found", "agent debugging", "harness debugging"] @@ -14,7 +14,7 @@ canonical: "https://manifest.build/llm-gateway/docs/autofix" Some requests fail because of the request, not the model. A parameter the provider doesn't accept, a tool schema in the wrong shape, a model name that moved. Switching to a fallback model doesn't help, because the next model rejects the same body. -Autofix handles that case. When a request fails with a repairable error, Manifest sends the failed request and the provider's error to a healing service, gets a corrected request back, and sends it once more. It runs **before** the fallback chain, so fallback is still the safety net if the repair doesn't land. +Autofix handles that case. When a request fails with a repairable error, the gateway sends the failed request and the provider's error to a healing service, gets a corrected request back, and sends it once more. It runs **before** the fallback chain, so fallback is still the safety net if the repair doesn't land. ## When it runs @@ -28,25 +28,25 @@ Autofix only touches request-side 4xx errors, where the provider is telling you It deliberately skips `401`, `403`, `429`, and every `5xx`. A bad key, a rate limit, or a provider outage isn't something a rewritten body can fix, so those go straight to [fallback](/llm-gateway/docs/llm-gateway#fallback). -Renamed models are covered through the provider itself. When a request names a model Manifest hasn't catalogued but the provider and credentials are unambiguous, Manifest forwards it anyway. The provider answers with its real error, typically a `404` for a model that no longer exists, and that error goes through the normal Autofix path. A patch that corrects the model name serves the request. +Renamed models are covered through the provider itself. When a request names a model the gateway hasn't catalogued but the provider and credentials are unambiguous, the gateway forwards it anyway. The provider answers with its real error, typically a `404` for a model that no longer exists, and that error goes through the normal Autofix path. A patch that corrects the model name serves the request. ## How it works - A provider returns a repairable 4xx. Manifest normalizes the error into a + A provider returns a repairable 4xx. The gateway normalizes the error into a message, type, and parameter. - + The failed request and the normalized error go to the hosted healing service, which returns a corrected request body. - Manifest sends the patched request. There is no retry budget: it tries + The gateway sends the patched request. There is no retry budget: it tries exactly once. - If the patched request fails too, Manifest reports the outcome and hands + If the patched request fails too, the gateway reports the outcome and hands off to your fallback chain as normal. @@ -59,7 +59,7 @@ Autofix is a per-harness toggle, and you meet it at two moments.
-Nothing to deploy or configure: your install talks to Manifest's hosted healing service directly. +Nothing to deploy or configure: your install talks to the hosted Manifest Autofix service directly.
@@ -73,11 +73,11 @@ The Connect Harness modal shows an Autofix toggle under the name field, pre-set ### Harnesses you already have -A harness that existed before Autofix keeps its state: Manifest never flips the toggle for you. +A harness that existed before Autofix keeps its state: the gateway never flips the toggle for you.
-On Manifest Cloud, those harnesses are already on. You can switch any of them off on its **Settings** page. +On LLM Gateway Cloud, those harnesses are already on. You can switch any of them off on its **Settings** page.
@@ -113,7 +113,7 @@ The first time you enable Autofix, your install records a one-time consent: enab -When Autofix repairs a request, Manifest sends the failed request, including its message content, plus the provider's error response to the healing service. +When Autofix repairs a request, the gateway sends the failed request, including its message content, plus the provider's error response to the healing service. Provider API keys and OAuth tokens are never sent: credentials travel in request headers, which are not part of the payload sent to the healing service. The provider's error response is scrubbed for secrets before it is sent. @@ -132,9 +132,9 @@ See the [privacy policy](https://manifest.build/privacy/) and the [terms](https: Autofix is built to stay out of the way when it isn't working: - **One attempt.** A patched request is sent once. It never loops. -- **Failures degrade quietly.** If the healing service errors or times out, your agent gets the original provider error. Autofix never turns a provider's 400 into a Manifest 500. -- **A slow healer stops being asked.** After three consecutive transport failures, Manifest stops calling it for 30 seconds so a down service doesn't add latency to every failing request. One successful call clears the streak. -- **Startup never waits.** In production, Manifest pings the healing service once at boot, fire-and-forget, and logs a warning if it's unreachable. It never blocks startup, and it's skipped entirely when Autofix is disabled globally. +- **Failures degrade quietly.** If the healing service errors or times out, your agent gets the original provider error. Autofix never turns a provider's 400 into a gateway 500. +- **A slow healer stops being asked.** After three consecutive transport failures, the gateway stops calling it for 30 seconds so a down service doesn't add latency to every failing request. One successful call clears the streak. +- **Startup never waits.** In production, the gateway pings the healing service once at boot, fire-and-forget, and logs a warning if it's unreachable. It never blocks startup, and it's skipped entirely when Autofix is disabled globally. - **Fallback still runs.** Autofix sits in front of the chain, it doesn't replace it. ## Seeing what it did @@ -162,7 +162,7 @@ Both **Recovered** cards link through to the matching filter on the request log. ## Environment variables - Self-hosted only. On [Manifest Cloud](https://app.manifest.build) there is no + Self-hosted only. On [LLM Gateway Cloud](https://app.manifest.build) there is no server to configure: everything Autofix related lives in the dashboard. @@ -173,7 +173,7 @@ Autofix needs no setup. Every variable below is optional and adjusts behavior: | `AUTOFIX_GLOBAL_ENABLED` | `true` | Set `false` to turn Autofix off for the whole deployment. No call reaches the healing service, including the boot health check | | `AUTOFIX_TIMEOUT_MS` | `10000` | Timeout per healing call, in milliseconds | | `AUTOFIX_REPAIRABLE_STATUSES` | `400,404,422` | Which provider statuses are eligible for a repair | -| `AUTOFIX_HEALING_API_KEY` | unset | Sent as `x-api-key` to the healing service. Only relevant if Manifest issues you one | +| `AUTOFIX_HEALING_API_KEY` | unset | Sent as `x-api-key` to the healing service. Only relevant when provided by the hosted healing service | | `AUTOFIX_REPORT_ALL_4XX` | `false` | Opt-in. Also report request-side 4xx errors (except `401`/`402`/`403`/`429`) from harnesses with Autofix on, as diagnostic evidence with the secret-scrubbed request body. Bodies over 256 KB are dropped, not truncated. No fix comes back from it | You don't need `AUTOFIX_HEALING_API_KEY`. With no key set, the install announces its anonymous install id instead. The full list of variables lives in [Environment variables](/llm-gateway/docs/reference/environment-variables). diff --git a/llm-gateway/docs/deploy.mdx b/llm-gateway/docs/deploy.mdx index 93cf485..ec96add 100644 --- a/llm-gateway/docs/deploy.mdx +++ b/llm-gateway/docs/deploy.mdx @@ -1,11 +1,11 @@ --- -title: "Self-hosting Manifest" +title: "Deploy the LLM Gateway" sidebarTitle: "Overview" -description: "Every way to run your own Manifest instance: Docker on your machine, a managed platform like Railway, your own AWS or GCP account, or a server panel like Coolify." +description: "Run the Manifest LLM Gateway with Docker, on a managed platform, or in your cloud account." icon: "rocket" keywords: [ - "self-hosted Manifest", + "self-hosted gateway", "Manifest deployment", "Docker", "Railway", @@ -19,7 +19,7 @@ keywords: canonical: "https://manifest.build/llm-gateway/docs/deploy" --- -Everything on this page runs the same open-source Manifest image on infrastructure you control, with your own PostgreSQL database. Pick the path that matches where you want it to live. If you'd rather not run anything yourself, [Manifest Cloud](https://app.manifest.build) is the hosted alternative. +Everything on this page runs the same open-source gateway image on infrastructure you control, with your own PostgreSQL database. Pick the path that matches where you want it to live. If you'd rather not run anything yourself, [LLM Gateway Cloud](https://app.manifest.build) is the hosted alternative. ## Your own machine @@ -35,7 +35,7 @@ Someone else runs the infrastructure. Usually a one-click template or deploy but - Best one-click path for most users. The template provisions Manifest and + Best one-click path for most users. The template provisions the gateway and PostgreSQL. @@ -43,12 +43,12 @@ Someone else runs the infrastructure. Usually a one-click template or deploy but Render PostgreSQL. - App Platform deploy from the public repository with Manifest and a Dev + App Platform deploy from the public repository with the gateway and a Dev PostgreSQL database. CLI-based deploy using the public Docker image, Fly Postgres, and generated - Manifest secrets. + gateway secrets. Deploy-button guide with Heroku Postgres, generated secrets, and one diff --git a/llm-gateway/docs/deploy/aws.mdx b/llm-gateway/docs/deploy/aws.mdx index 9ae9176..aa59210 100644 --- a/llm-gateway/docs/deploy/aws.mdx +++ b/llm-gateway/docs/deploy/aws.mdx @@ -1,10 +1,10 @@ --- -title: "Deploy Manifest on AWS" +title: "Deploy the LLM Gateway on AWS" sidebarTitle: "AWS" -description: "Deploy Manifest on AWS with CloudFormation, ECS Fargate, RDS PostgreSQL, Secrets Manager, and an Application Load Balancer." +description: "Deploy the Manifest LLM Gateway with CloudFormation, ECS Fargate, RDS PostgreSQL, Secrets Manager, and an Application Load Balancer." icon: "/icons/deploy/aws.svg" keywords: - ["Manifest deployment", "AWS", "self-hosted Manifest", "PostgreSQL", "Docker"] + ["Manifest deployment", "AWS", "self-hosted gateway", "PostgreSQL", "Docker"] canonical: "https://manifest.build/llm-gateway/docs/deploy/aws" --- @@ -19,7 +19,7 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/aws" ## Overview -The AWS template deploys Manifest with ECS Fargate, RDS PostgreSQL, Secrets Manager, CloudWatch Logs, and an Application Load Balancer. It is a good fit when you want Manifest in your own AWS account. +The AWS template deploys the gateway with ECS Fargate, RDS PostgreSQL, Secrets Manager, CloudWatch Logs, and an Application Load Balancer. It is a good fit when you want the gateway in your own AWS account. ## Prerequisites @@ -65,11 +65,11 @@ The CloudFormation template generates: - `BETTER_AUTH_SECRET`. - `MANIFEST_ENCRYPTION_KEY`. -Manifest runs with `PORT=2099`, `BIND_ADDRESS=0.0.0.0`, and `MANIFEST_MODE=selfhosted`. +The gateway runs with `PORT=2099`, `BIND_ADDRESS=0.0.0.0`, and `MANIFEST_MODE=selfhosted`. The template also creates an S3 bucket for [request logs](/llm-gateway/docs/request-logs). It is retained by design, so deleting the CloudFormation stack leaves the bucket in place. Delete it by hand when you tear the deployment down. -## Open Manifest +## Open the gateway After deployment, the script prints `ServiceUrl` and `HealthCheckUrl`. @@ -83,7 +83,7 @@ Open `ServiceUrl` and create the first account. Fresh installs redirect to `/set ## Production notes -- The default template exposes Manifest over HTTP on the generated load balancer DNS name. +- The default template exposes the gateway over HTTP on the generated load balancer DNS name. - Configure TLS with your own domain and ACM certificate before using the deployment for production authentication traffic. - Set database deletion protection if you want to avoid accidental RDS deletion. diff --git a/llm-gateway/docs/deploy/coolify.mdx b/llm-gateway/docs/deploy/coolify.mdx index b6ad73d..85d2462 100644 --- a/llm-gateway/docs/deploy/coolify.mdx +++ b/llm-gateway/docs/deploy/coolify.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Coolify" +title: "Deploy the LLM Gateway on Coolify" sidebarTitle: "Coolify" -description: "Deploy Manifest on Coolify with a Docker Compose service stack, generated secrets, and a private PostgreSQL container." +description: "Deploy the Manifest LLM Gateway with a Docker Compose service stack, generated secrets, and a private PostgreSQL container." icon: "/icons/deploy/coolify.svg" keywords: [ "Manifest deployment", "Coolify", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -49,11 +49,11 @@ This stack runs on infrastructure you control through Coolify. Your server, stor - `SERVICE_HEX_64_ENCRYPTION` 7. Deploy. -The Compose file uses Coolify magic environment variables so the app URL, Postgres password, Better Auth secret, and Manifest encryption key are generated and stay stable between deployments. +The Compose file uses Coolify magic environment variables so the app URL, Postgres password, Better Auth secret, and gateway encryption key are generated and stay stable between deployments. -## Open Manifest +## Open the gateway -After deployment, open the generated Manifest URL and create the first admin account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. +After deployment, open the generated gateway URL and create the first admin account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. Verify the deployment: diff --git a/llm-gateway/docs/deploy/digitalocean.mdx b/llm-gateway/docs/deploy/digitalocean.mdx index cb71a2d..5a4a822 100644 --- a/llm-gateway/docs/deploy/digitalocean.mdx +++ b/llm-gateway/docs/deploy/digitalocean.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on DigitalOcean" +title: "Deploy the LLM Gateway on DigitalOcean" sidebarTitle: "DigitalOcean" -description: "Deploy Manifest on DigitalOcean App Platform with the Deploy to DigitalOcean flow and a Dev PostgreSQL database." +description: "Deploy the Manifest LLM Gateway on App Platform with the Deploy to DigitalOcean flow and a PostgreSQL database." icon: "/icons/deploy/digitalocean.svg" keywords: [ "Manifest deployment", "DigitalOcean", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -19,13 +19,13 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/digitalocean" icon="external-link" href="https://cloud.digitalocean.com/apps/new?repo=https://github.com/mnfst/manifest/tree/main" > - App Platform deploy from the public repository with Manifest and a Dev + App Platform deploy from the public repository with the gateway and a Dev PostgreSQL database. ## Overview -DigitalOcean App Platform can create a Manifest app from the public repository template. The template uses the public Manifest Docker image and provisions a Dev PostgreSQL database. +DigitalOcean App Platform can create a gateway app from the public repository template. The template uses the public Manifest Docker image and provisions a Dev PostgreSQL database. ## Prerequisites @@ -51,9 +51,9 @@ openssl rand -hex 32 6. Fill in the recording storage secrets with your Space's settings: `REQUEST_RECORDING_S3_BUCKET`, `REQUEST_RECORDING_S3_ENDPOINT`, `REQUEST_RECORDING_S3_ACCESS_KEY_ID`, and `REQUEST_RECORDING_S3_SECRET_ACCESS_KEY`. The region defaults to `us-east-1`, which DigitalOcean Spaces accepts as a signing region. 7. Create the app. -DigitalOcean reads `.do/deploy.template.yaml` from the Manifest repository. The template includes the Manifest service and a Dev PostgreSQL database. +DigitalOcean reads `.do/deploy.template.yaml` from the Manifest repository. The template includes the gateway service and a Dev PostgreSQL database. -## Open Manifest +## Open the gateway After App Platform finishes deploying, open the app URL and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. diff --git a/llm-gateway/docs/deploy/easypanel.mdx b/llm-gateway/docs/deploy/easypanel.mdx index 065366e..3e7c80e 100644 --- a/llm-gateway/docs/deploy/easypanel.mdx +++ b/llm-gateway/docs/deploy/easypanel.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Easypanel" +title: "Deploy the LLM Gateway on Easypanel" sidebarTitle: "Easypanel" -description: "Deploy Manifest on Easypanel with a Docker image app service, PostgreSQL, generated secrets, and HTTPS." +description: "Deploy the Manifest LLM Gateway with a Docker image app service, PostgreSQL, generated secrets, and HTTPS." icon: "/icons/deploy/easypanel.svg" keywords: [ "Manifest deployment", "Easypanel", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -25,7 +25,7 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/easypanel" ## Overview -Easypanel can run Manifest as a Docker image app service, proxy the dashboard over HTTPS, and provision PostgreSQL in the same project. The Manifest repository includes Easypanel template files that provision the app, database, generated secrets, and proxy port. Until that template is accepted into the Easypanel catalog, you can create the same services manually. +Easypanel can run the gateway as a Docker image app service, proxy the dashboard over HTTPS, and provision PostgreSQL in the same project. The Manifest repository includes Easypanel template files that provision the app, database, generated secrets, and proxy port. Until that template is accepted into the Easypanel catalog, you can create the same services manually. ## Prerequisites @@ -37,7 +37,7 @@ This stack runs on infrastructure you control through Easypanel. Your server, st ## Template deploy -If Manifest is available in your Easypanel template catalog: +If the gateway is available in your Easypanel template catalog: 1. Open your Easypanel project. 2. Choose **Templates**. @@ -45,13 +45,13 @@ If Manifest is available in your Easypanel template catalog: 4. Keep the default image `manifestdotbuild/manifest:6` unless you need another version. 5. Deploy the template. -The template provisions Manifest, PostgreSQL, generated secrets, and a domain proxy on port `2099`. +The template provisions the gateway, PostgreSQL, generated secrets, and a domain proxy on port `2099`. ## Manual deploy Create a PostgreSQL service named `manifest-db`, then create an App service from Docker image `manifestdotbuild/manifest:6`. Set the proxy port to `2099`, add a domain, and mark it as the primary domain. -Set the Manifest environment: +Set the gateway environment: ```env PORT=2099 @@ -74,9 +74,9 @@ Generate two different 64-character secrets with `openssl rand -hex 32`. Use an Add an app volume named `recordings` mounted at `/data/request-recordings` so [request logs](/llm-gateway/docs/request-logs) survive restarts and redeploys. This volume is single-instance; move to S3-compatible storage before running more than one replica. -`AUTH_DB_POOL_MAX` is read by Manifest's Better Auth database pool. Set `MANIFEST_TELEMETRY_DISABLED=1` if you want to disable anonymous self-hosted telemetry. +`AUTH_DB_POOL_MAX` is read by the gateway's Better Auth database pool. Set `MANIFEST_TELEMETRY_DISABLED=1` if you want to disable anonymous self-hosted telemetry. -## Open Manifest +## Open the gateway Open the Easypanel domain and create the first admin account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. @@ -103,7 +103,7 @@ Before submitting a template upstream, verify: - Back up the PostgreSQL service through Easypanel or your server provider. - Use a real HTTPS domain before enabling OAuth providers. - Keep `BETTER_AUTH_SECRET` and `MANIFEST_ENCRYPTION_KEY` as separate values. -- If you deploy more than one Manifest replica, lower database pool sizes or upgrade PostgreSQL connection capacity. +- If you deploy more than one gateway replica, lower database pool sizes or upgrade PostgreSQL connection capacity. Relevant Easypanel docs: diff --git a/llm-gateway/docs/deploy/fly.mdx b/llm-gateway/docs/deploy/fly.mdx index 76061ac..ab87d5d 100644 --- a/llm-gateway/docs/deploy/fly.mdx +++ b/llm-gateway/docs/deploy/fly.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Fly.io" +title: "Deploy the LLM Gateway on Fly.io" sidebarTitle: "Fly.io" -description: "Deploy Manifest on Fly.io from the public Docker image with Fly Postgres, generated secrets, and HTTPS." +description: "Deploy the Manifest LLM Gateway from the public Docker image with Fly Postgres, generated secrets, and HTTPS." icon: "/icons/deploy/fly.svg" keywords: [ "Manifest deployment", "Fly.io", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -20,12 +20,12 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/fly" href="https://fly.io/docs/flyctl/install/" > CLI-based deploy using the public Docker image, Fly Postgres, and generated - Manifest secrets. + gateway secrets. ## Overview -Fly.io runs Manifest as a long-running Docker container close to users, with a Fly Postgres database in the same private network. This is not a browser one-click button. The Manifest repository includes a `deploy/fly/deploy.sh` script that creates the app, creates Postgres, attaches `DATABASE_URL`, sets secrets, and deploys the public Docker image. +Fly.io runs the gateway as a long-running Docker container close to users, with a Fly Postgres database in the same private network. This is not a browser one-click button. The Manifest repository includes a `deploy/fly/deploy.sh` script that creates the app, creates Postgres, attaches `DATABASE_URL`, sets secrets, and deploys the public Docker image. ## Prerequisites @@ -48,7 +48,7 @@ FLY_ORG=personal \ The script deploys `docker.io/manifestdotbuild/manifest:6`, provisions Fly Postgres, attaches it as `DATABASE_URL`, and generates separate values for `BETTER_AUTH_SECRET` and `MANIFEST_ENCRYPTION_KEY`. Re-running the script keeps existing secrets instead of rotating sessions or encrypted provider credentials. -## Open Manifest +## Open the gateway Open the deployed app and create the first admin account: @@ -70,7 +70,7 @@ fly logs --app ## Production notes -- The template keeps one Machine running so Manifest is always available for agents. +- The template keeps one Machine running so the gateway is always available for agents. - For production, choose a larger Postgres configuration or Fly Managed Postgres instead of the small script default. - Add a custom domain before configuring OAuth callback URLs. - The deploy script creates a private Tigris bucket, stores its credentials as Fly secrets, and sets `REQUEST_RECORDING_STORAGE=s3` with the Tigris endpoint. [Request logs](/llm-gateway/docs/request-logs) persist across deploys and across scaled Machines. Nothing to attach manually. diff --git a/llm-gateway/docs/deploy/gcp.mdx b/llm-gateway/docs/deploy/gcp.mdx index 8c3390a..73b3e8c 100644 --- a/llm-gateway/docs/deploy/gcp.mdx +++ b/llm-gateway/docs/deploy/gcp.mdx @@ -1,10 +1,10 @@ --- -title: "Deploy Manifest on GCP" +title: "Deploy the LLM Gateway on Google Cloud" sidebarTitle: "GCP" -description: "Deploy Manifest on Google Cloud with Cloud Run, Cloud SQL for PostgreSQL, Secret Manager, and the Cloud Shell DeployStack flow." +description: "Deploy the Manifest LLM Gateway with Cloud Run, Cloud SQL for PostgreSQL, Secret Manager, and Cloud Shell." icon: "/icons/deploy/gcp.svg" keywords: - ["Manifest deployment", "GCP", "self-hosted Manifest", "PostgreSQL", "Docker"] + ["Manifest deployment", "GCP", "self-hosted gateway", "PostgreSQL", "Docker"] canonical: "https://manifest.build/llm-gateway/docs/deploy/gcp" --- @@ -19,7 +19,7 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/gcp" ## Overview -The Google Cloud deploy path uses Cloud Shell and DeployStack to run Terraform. It provisions Cloud Run for the Manifest container, Cloud SQL for PostgreSQL, and Secret Manager for runtime secrets. +The Google Cloud deploy path uses Cloud Shell and DeployStack to run Terraform. It provisions Cloud Run for the gateway container, Cloud SQL for PostgreSQL, and Secret Manager for runtime secrets. ## Prerequisites @@ -38,7 +38,7 @@ The walkthrough prompts for: - Google Cloud project. - Region. - Cloud Run service name. -- Manifest image URL. +- Gateway image URL. - Cloud SQL machine tier. - Maximum Cloud Run instances. - Cloud SQL deletion protection. @@ -61,7 +61,7 @@ deploystack install The first deploy usually takes 10-15 minutes because Cloud SQL needs time to provision. The Terraform config patches `BETTER_AUTH_URL` to the final Cloud Run URL after the service is created. -## Open Manifest +## Open the gateway Print the service URL: diff --git a/llm-gateway/docs/deploy/heroku.mdx b/llm-gateway/docs/deploy/heroku.mdx index 557c9fa..2ac0ae7 100644 --- a/llm-gateway/docs/deploy/heroku.mdx +++ b/llm-gateway/docs/deploy/heroku.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Heroku" +title: "Deploy the LLM Gateway on Heroku" sidebarTitle: "Heroku" -description: "Deploy Manifest on Heroku with a Deploy to Heroku button, a container-stack web dyno, and Heroku Postgres." +description: "Deploy the Manifest LLM Gateway with a container-stack web dyno and Heroku Postgres." icon: "/icons/deploy/heroku.svg" keywords: [ "Manifest deployment", "Heroku", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -25,7 +25,7 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/heroku" ## Overview -Heroku can deploy Manifest with the repository `app.json` and `heroku.yml`. The deploy button creates a Cedar container-stack app, provisions Heroku Postgres, generates Manifest secrets, and builds a tiny Heroku Dockerfile that uses the public Manifest image. +Heroku can deploy the gateway with the repository `app.json` and `heroku.yml`. The deploy button creates a Cedar container-stack app, provisions Heroku Postgres, generates gateway secrets, and builds a tiny Heroku Dockerfile that uses the public Manifest image. Heroku asks for a few values before it can deploy. Set `BETTER_AUTH_URL` to the public URL for the app name you choose. Heroku dynos have an ephemeral filesystem, so [request logs](/llm-gateway/docs/request-logs) write to external S3-compatible storage rather than local disk: set the bucket, region, and access key for that storage too. @@ -52,7 +52,7 @@ https://.herokuapp.com Heroku generates `BETTER_AUTH_SECRET` and `MANIFEST_ENCRYPTION_KEY`, provisions Postgres as `DATABASE_URL`, builds `Dockerfile.heroku`, and starts the web dyno. -## Open Manifest +## Open the gateway Open the deployed app and create the first admin account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. diff --git a/llm-gateway/docs/deploy/koyeb.mdx b/llm-gateway/docs/deploy/koyeb.mdx index 34ad2ba..530c67e 100644 --- a/llm-gateway/docs/deploy/koyeb.mdx +++ b/llm-gateway/docs/deploy/koyeb.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Koyeb" +title: "Deploy the LLM Gateway on Koyeb" sidebarTitle: "Koyeb" -description: "Deploy Manifest on Koyeb from the public Docker image with a Koyeb web service and PostgreSQL connection string." +description: "Deploy the Manifest LLM Gateway from the public Docker image with a Koyeb web service and PostgreSQL." icon: "/icons/deploy/koyeb.svg" keywords: [ "Manifest deployment", "Koyeb", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -25,22 +25,22 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/koyeb" ## Overview -Koyeb can deploy Manifest from the public Docker image with a Deploy to Koyeb button. The button pre-fills the image, HTTP port, and runtime settings, but you must create PostgreSQL separately and replace the placeholder secrets before deploying. +Koyeb can deploy the gateway from the public Docker image with a Deploy to Koyeb button. The button pre-fills the image, HTTP port, and runtime settings, but you must create PostgreSQL separately and replace the placeholder secrets before deploying. ## Prerequisites - A Koyeb account on a paid plan (Pro or higher) with a valid payment method. Since Koyeb joined Mistral AI, deploying requires a paid plan: on an account without one, the deploy link shows an announcement banner instead of the form. - A Koyeb PostgreSQL Database Service. -- Two random 32+ character secrets for Manifest. +- Two random 32+ character secrets for the gateway. - A private S3-compatible bucket and an access key for it. Koyeb instances have no persistent disk, so request logs need external storage. This deployment creates paid resources. ## Create PostgreSQL -In Koyeb, create a PostgreSQL Database Service in the same region you plan to use for Manifest. After it is ready, open the database connection details and copy the connection string. +In Koyeb, create a PostgreSQL Database Service in the same region you plan to use for the gateway. After it is ready, open the database connection details and copy the connection string. -Manifest uses TLS to connect to Koyeb Postgres, so include `sslmode=require` in the connection string. If the copied URL has no query string, append `?sslmode=require`. If it already has query parameters, append `&sslmode=require`. +The gateway uses TLS to connect to Koyeb Postgres, so include `sslmode=require` in the connection string. If the copied URL has no query string, append `?sslmode=require`. If it already has query parameters, append `&sslmode=require`. ## Generate secrets @@ -51,7 +51,7 @@ openssl rand -hex 32 openssl rand -hex 32 ``` -## Deploy Manifest +## Deploy the gateway 1. Open the [Koyeb deploy link](https://app.koyeb.com/deploy?type=docker&image=docker.io%2Fmanifestdotbuild%2Fmanifest%3A6&name=manifest&service_type=web&ports=2099%3Bhttp%3B%2F&env%5BPORT%5D=2099&env%5BDATABASE_URL%5D=postgres%3A%2F%2FUSER%3APASSWORD%40HOST%2FDB%3Fsslmode%3Drequire&env%5BBETTER_AUTH_SECRET%5D=replace-with-openssl-rand-hex-32&env%5BMANIFEST_ENCRYPTION_KEY%5D=replace-with-different-openssl-rand-hex-32&env%5BBETTER_AUTH_URL%5D=https%3A%2F%2F%7B%7B+KOYEB_PUBLIC_DOMAIN+%7D%7D&env%5BMANIFEST_MODE%5D=selfhosted&env%5BBIND_ADDRESS%5D=0.0.0.0&env%5BDB_POOL_MAX%5D=8&env%5BAUTH_DB_POOL_MAX%5D=4&env%5BREQUEST_RECORDING_STORAGE%5D=s3&env%5BREQUEST_RECORDING_S3_BUCKET%5D=replace-with-private-bucket&env%5BREQUEST_RECORDING_S3_ENDPOINT%5D=https%3A%2F%2Fs3.example.com&env%5BREQUEST_RECORDING_S3_REGION%5D=us-east-1&env%5BREQUEST_RECORDING_S3_ACCESS_KEY_ID%5D=replace-with-limited-access-key&env%5BREQUEST_RECORDING_S3_SECRET_ACCESS_KEY%5D=replace-with-secret-key&env%5BREQUEST_RECORDING_S3_FORCE_PATH_STYLE%5D=false). 2. Replace `DATABASE_URL` with your Koyeb Postgres connection string. @@ -64,7 +64,7 @@ openssl rand -hex 32 The button deploys `docker.io/manifestdotbuild/manifest:6` and exposes port `2099` over HTTP. -## Open Manifest +## Open the gateway After the deployment is live, open the public Koyeb domain and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. diff --git a/llm-gateway/docs/deploy/railway.mdx b/llm-gateway/docs/deploy/railway.mdx index 8e8aaa7..a5c9e10 100644 --- a/llm-gateway/docs/deploy/railway.mdx +++ b/llm-gateway/docs/deploy/railway.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Railway" +title: "Deploy the LLM Gateway on Railway" sidebarTitle: "Railway" -description: "Deploy Manifest on Railway with the published template, including the Manifest service and PostgreSQL." +description: "Deploy the Manifest LLM Gateway with the published Railway template, PostgreSQL, and request-log storage." icon: "/icons/deploy/railway.svg" keywords: [ "Manifest deployment", "Railway", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -19,13 +19,13 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/railway" icon="external-link" href="https://railway.com/deploy/wild-wild" > - Best one-click path for most users. The template provisions Manifest, + Best one-click path for most users. The template provisions the gateway, PostgreSQL, and a bucket for request logs. ## Overview -Railway is the fastest hosted path for Manifest. Use the published Manifest template to create a project with the Manifest app, a PostgreSQL database, and a storage bucket for [request logs](/llm-gateway/docs/request-logs), then open the generated domain and create the first admin account. +Railway is the fastest hosted path for the gateway. Use the published Manifest template to create a project with the gateway app, a PostgreSQL database, and a storage bucket for [request logs](/llm-gateway/docs/request-logs), then open the generated domain and create the first admin account. ## Prerequisites @@ -36,14 +36,14 @@ Railway is the fastest hosted path for Manifest. Use the published Manifest temp ## Deploy 1. Open the [Manifest Railway template](https://railway.com/deploy/wild-wild). -2. Choose the Railway workspace and project where you want to deploy Manifest. +2. Choose the Railway workspace and project where you want to deploy the gateway. 3. Review the services that will be created. 4. Deploy the template. -5. Wait for the Manifest service and PostgreSQL database to finish provisioning. +5. Wait for the gateway service and PostgreSQL database to finish provisioning. The template includes the app service, the database, and a `messages-recordings` bucket, so you do not need to create PostgreSQL or recording storage separately. -## Open Manifest +## Open the gateway Open the generated Railway domain and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. diff --git a/llm-gateway/docs/deploy/render.mdx b/llm-gateway/docs/deploy/render.mdx index bcbd4a8..23d1038 100644 --- a/llm-gateway/docs/deploy/render.mdx +++ b/llm-gateway/docs/deploy/render.mdx @@ -1,13 +1,13 @@ --- -title: "Deploy Manifest on Render" +title: "Deploy the LLM Gateway on Render" sidebarTitle: "Render" -description: "Deploy Manifest on Render with the repository Blueprint, including a web service and Render PostgreSQL." +description: "Deploy the Manifest LLM Gateway with a repository Blueprint, a web service, and Render PostgreSQL." icon: "/icons/deploy/render.svg" keywords: [ "Manifest deployment", "Render", - "self-hosted Manifest", + "self-hosted gateway", "PostgreSQL", "Docker", ] @@ -25,7 +25,7 @@ canonical: "https://manifest.build/llm-gateway/docs/deploy/render" ## Overview -Render can deploy Manifest from the repository Blueprint. The Blueprint creates a web service from the public Manifest Docker image and a Render PostgreSQL database, then wires the runtime environment variables. +Render can deploy the gateway from the repository Blueprint. The Blueprint creates a web service from the public Manifest Docker image and a Render PostgreSQL database, then wires the runtime environment variables. ## Prerequisites @@ -50,7 +50,7 @@ The repository `render.yaml` sets: - `BETTER_AUTH_URL` from the Render external service URL. - `MANIFEST_MODE=selfhosted`. -## Open Manifest +## Open the gateway Open the Render service URL and create the first account. Fresh installs redirect to `/setup`; the first account you create becomes the admin. diff --git a/llm-gateway/docs/errors.mdx b/llm-gateway/docs/errors.mdx index 05e5ef9..ed08a4e 100644 --- a/llm-gateway/docs/errors.mdx +++ b/llm-gateway/docs/errors.mdx @@ -1,14 +1,14 @@ --- -title: "Manifest error codes" +title: "LLM Gateway error codes" sidebarTitle: "Errors" -description: "Reference for every Manifest proxy error code (M001-M500). What you saw, why it happened, and how to fix it. Covers auth, providers, limits, validation." +description: "Reference for errors returned by the Manifest LLM Gateway, with causes and fixes." icon: "circle-alert" keywords: ["Manifest error codes", "M001", "M100", "M102", "M200", "M204", "M302", "M500", "OpenAI compatible errors", "401 Unauthorized", "402 Payment Required", "429 Too Many Requests", "Bearer token error", "chat completions error", "model not available", "troubleshooting", "proxy errors"] canonical: "https://manifest.build/llm-gateway/docs/errors" --- -When Manifest blocks or rejects a request, the response message starts with a code in square brackets: +When the gateway blocks or rejects a request, the response message starts with a code in square brackets: ```text [🦚 Manifest M100] No anthropic API key yet. Add one here: https://app.manifest.build/... @@ -33,7 +33,7 @@ These fire when the bearer token on `/v1/chat/completions` is missing or wrong. ## Providers (M100–M102) -Your Manifest key is fine, but provider credentials are missing or unusable. See [Routing](/llm-gateway/docs/llm-gateway), [API key providers](/llm-gateway/docs/providers/api-key-providers), and [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers). +Your harness key is fine, but provider credentials are missing or unusable. See [Routing](/llm-gateway/docs/llm-gateway), [API key providers](/llm-gateway/docs/providers/api-key-providers), and [Subscription-based providers](/llm-gateway/docs/providers/subscription-based-providers). | Code | What | |------|------| @@ -55,13 +55,13 @@ You hit a usage cap, rate limit, or Free plan request quota. M200 comes back as ## Request validation and model selection (M300–M303) -M300 means the request body is malformed and surfaces as HTTP 400. M302 and M303 mean Manifest can't reach the model the request asked for. M302 comes back as an HTTP 200 assistant message for every client. M303 surfaces as HTTP 400 to non-chat clients. +M300 means the request body is malformed and surfaces as HTTP 400. M302 and M303 mean the gateway can't reach the model the request asked for. M302 comes back as an HTTP 200 assistant message for every client. M303 surfaces as HTTP 400 to non-chat clients. | Code | What | |------|------| | [M300: Missing messages array](/llm-gateway/docs/errors/M300) | Body has no `messages` array, or it's empty | | [M302: Model not available](/llm-gateway/docs/errors/M302) | Explicit model ID is not available for this harness | -| [M303: Local provider unavailable on Manifest Cloud](/llm-gateway/docs/errors/M303) | Routed to Ollama, LM Studio, or llama.cpp on Cloud | +| [M303: Local provider unavailable on LLM Gateway Cloud](/llm-gateway/docs/errors/M303) | Routed to Ollama, LM Studio, or llama.cpp on Cloud | M301 is missing from that list on purpose. It capped requests at 1000 @@ -71,7 +71,7 @@ M300 means the request body is malformed and surfaces as HTTP 400. M302 and M303 ## Server (M500) -Manifest itself broke. Surfaces as HTTP 500. +The gateway itself broke. Surfaces as HTTP 500. | Code | What | |------|------| diff --git a/llm-gateway/docs/errors/M001.mdx b/llm-gateway/docs/errors/M001.mdx index 67d828e..67b91d8 100644 --- a/llm-gateway/docs/errors/M001.mdx +++ b/llm-gateway/docs/errors/M001.mdx @@ -1,7 +1,7 @@ --- title: "M001: Missing Authorization header" sidebarTitle: "M001" -description: "Manifest error M001 fires when /v1/chat/completions is called without an Authorization header. Returns HTTP 401. Fix: send `Authorization: Bearer mnfst_`." +description: "M001 is returned when a request has no Authorization header. Send a Bearer harness key." icon: "key-round" keywords: ["M001", "Manifest M001", "Authorization header missing", "Bearer token", "401 Unauthorized", "OpenAI compatible 401", "Manifest auth error", "chat completions authorization"] @@ -17,7 +17,7 @@ See https://manifest.build/llm-gateway/docs/errors/M001 ## Why it happened -Your client called `/v1/chat/completions` without an `Authorization` header. Manifest checks a harness API key on every request, so the header has to be there. +Your client called `/v1/chat/completions` without an `Authorization` header. The gateway checks a harness API key on every request, so the header has to be there. ## How to fix it diff --git a/llm-gateway/docs/errors/M002.mdx b/llm-gateway/docs/errors/M002.mdx index 72d2239..bc0649f 100644 --- a/llm-gateway/docs/errors/M002.mdx +++ b/llm-gateway/docs/errors/M002.mdx @@ -1,7 +1,7 @@ --- title: "M002: Empty Bearer token" sidebarTitle: "M002" -description: "Manifest error M002 fires when the Authorization header is set but the Bearer token is blank. Often an unexpanded env var. Returns HTTP 401." +description: "M002 is returned when the Bearer token is empty, often because an environment variable was not expanded." icon: "key-round" keywords: ["M002", "Manifest M002", "empty Bearer token", "blank token", "401 Unauthorized", "MNFST_KEY env var", "Manifest auth error", "OpenAI compatible 401"] diff --git a/llm-gateway/docs/errors/M003.mdx b/llm-gateway/docs/errors/M003.mdx index e938a89..1e26629 100644 --- a/llm-gateway/docs/errors/M003.mdx +++ b/llm-gateway/docs/errors/M003.mdx @@ -1,7 +1,7 @@ --- title: "M003: Invalid key format" sidebarTitle: "M003" -description: "Manifest error M003 fires when the Bearer token doesn't start with mnfst_. Often a provider key (sk-...) pasted in the wrong field. Returns HTTP 401." +description: "M003 is returned when the Bearer token does not start with mnfst_. Use a harness key, not a provider key." icon: "key-round" keywords: ["M003", "Manifest M003", "invalid API key format", "mnfst_ prefix", "wrong API key", "OpenAI sk- key", "401 Unauthorized", "Manifest auth error"] @@ -17,7 +17,7 @@ See https://manifest.build/llm-gateway/docs/errors/M003 ## Why it happened -Manifest keys always start with `mnfst_`. Your token has a different prefix, which almost always means a provider key (OpenAI, Anthropic, etc.) got pasted into the wrong field. +harness keys always start with `mnfst_`. Your token has a different prefix, which almost always means a provider key (OpenAI, Anthropic, etc.) got pasted into the wrong field. ## How to fix it diff --git a/llm-gateway/docs/errors/M004.mdx b/llm-gateway/docs/errors/M004.mdx index 7734989..0b53349 100644 --- a/llm-gateway/docs/errors/M004.mdx +++ b/llm-gateway/docs/errors/M004.mdx @@ -1,7 +1,7 @@ --- title: "M004: Key expired" sidebarTitle: "M004" -description: "Manifest error M004 fires when the API key you sent is past its expiration date. Returns HTTP 401. Fix: rotate the key in the dashboard." +description: "M004 is returned when the harness key has expired. Rotate the key in the dashboard." icon: "key-round" keywords: ["M004", "Manifest M004", "API key expired", "key rotation", "401 Unauthorized", "Manifest auth error", "expired token"] @@ -17,7 +17,7 @@ See https://manifest.build/llm-gateway/docs/errors/M004 ## Why it happened -Manifest keys can carry an expiration date. Yours is past that date, so it no longer authenticates anything. +harness keys can carry an expiration date. Yours is past that date, so it no longer authenticates anything. ## How to fix it diff --git a/llm-gateway/docs/errors/M005.mdx b/llm-gateway/docs/errors/M005.mdx index 36811ef..8263a0b 100644 --- a/llm-gateway/docs/errors/M005.mdx +++ b/llm-gateway/docs/errors/M005.mdx @@ -1,7 +1,7 @@ --- title: "M005: Key not recognized" sidebarTitle: "M005" -description: "Manifest error M005 fires when the mnfst_ key has the right shape but no matching harness exists. Often a rotated or deleted key. Returns HTTP 401." +description: "M005 is returned when no harness matches a correctly shaped mnfst_ key." icon: "key-round" keywords: ["M005", "Manifest M005", "invalid API key", "rotated key", "deleted agent", "deleted harness", "401 Unauthorized", "key not found", "Manifest auth error"] diff --git a/llm-gateway/docs/errors/M100.mdx b/llm-gateway/docs/errors/M100.mdx index e299f13..1b15f47 100644 --- a/llm-gateway/docs/errors/M100.mdx +++ b/llm-gateway/docs/errors/M100.mdx @@ -1,7 +1,7 @@ --- title: "M100: Provider API key missing" sidebarTitle: "M100" -description: "Manifest error M100 fires when routing picks a provider that has no API key for this harness (OpenAI, Anthropic, Gemini, OpenRouter, etc). Fix: add the key from Providers → Usage-based." +description: "M100 is returned when routing selects a provider for which the harness has no API key." icon: "plug-zap" keywords: ["M100", "Manifest M100", "provider API key missing", "no OpenAI key", "no Anthropic key", "no Gemini key", "OpenRouter key", "Manifest routing", "provider credentials"] @@ -15,11 +15,11 @@ canonical: "https://manifest.build/llm-gateway/docs/errors/M100" See https://manifest.build/llm-gateway/docs/errors/M100 ``` -The provider name (`anthropic` in the example) varies based on which provider Manifest selected. +The provider name (`anthropic` in the example) varies based on which provider the gateway selected. ## Why it happened -Routing picked a provider that has no key on file for this harness. Manifest won't silently swap vendors on you, so the request stops here. +Routing picked a provider that has no key on file for this harness. The gateway won't silently swap vendors on you, so the request stops here. ## How to fix it diff --git a/llm-gateway/docs/errors/M101.mdx b/llm-gateway/docs/errors/M101.mdx index 98d238e..2308934 100644 --- a/llm-gateway/docs/errors/M101.mdx +++ b/llm-gateway/docs/errors/M101.mdx @@ -1,7 +1,7 @@ --- title: "M101: No providers configured" sidebarTitle: "M101" -description: "Manifest error M101 fires when an authenticated harness has zero providers connected. Common on first run. Fix: connect a provider from the Providers section of the dashboard." +description: "M101 is returned when the authenticated harness has no providers configured." icon: "plug-zap" keywords: ["M101", "Manifest M101", "no providers configured", "first-time setup", "empty agent", "empty harness", "Manifest routing", "connect provider", "OpenRouter setup"] diff --git a/llm-gateway/docs/errors/M102.mdx b/llm-gateway/docs/errors/M102.mdx index 675af13..90b2271 100644 --- a/llm-gateway/docs/errors/M102.mdx +++ b/llm-gateway/docs/errors/M102.mdx @@ -1,7 +1,7 @@ --- title: "M102: Provider subscription credentials unusable" sidebarTitle: "M102" -description: "Manifest error M102 fires when routing picks a subscription provider whose OAuth or refresh credentials cannot be used. Fix: reconnect the subscription from Providers → Subscriptions." +description: "M102 is returned when subscription credentials cannot be refreshed or used." icon: "plug-zap" keywords: [ @@ -25,11 +25,11 @@ canonical: "https://manifest.build/llm-gateway/docs/errors/M102" See https://manifest.build/llm-gateway/docs/errors/M102 ``` -The provider name (`openai` in the example) varies based on which subscription route Manifest selected. +The provider name (`openai` in the example) varies based on which subscription route the gateway selected. ## Why it happened -Routing elected a **subscription** connection (ChatGPT OAuth, Claude Max token, Copilot device code, and similar), and Manifest still has a stored credential blob for it — but that credential cannot produce a usable access token. +Routing elected a **subscription** connection (ChatGPT OAuth, Claude Max token, Copilot device code, and similar), and the gateway still has a stored credential blob for it — but that credential cannot produce a usable access token. Typical causes: @@ -39,7 +39,7 @@ Typical causes: This is different from [M100](/llm-gateway/docs/errors/M100): M100 means there is no key on file. M102 means a subscription row exists and was elected, but the unwrap/refresh step failed. -When fallback routes are configured, Manifest records the M102 hop and continues the chain. You only see a terminal M102 response when no usable fallback remains. +When fallback routes are configured, the gateway records the M102 hop and continues the chain. You only see a terminal M102 response when no usable fallback remains. ## How to fix it diff --git a/llm-gateway/docs/errors/M200.mdx b/llm-gateway/docs/errors/M200.mdx index 8af7aa1..474eb01 100644 --- a/llm-gateway/docs/errors/M200.mdx +++ b/llm-gateway/docs/errors/M200.mdx @@ -1,7 +1,7 @@ --- title: "M200: Usage limit exceeded" sidebarTitle: "M200" -description: "Manifest error M200 fires when a harness crosses a cost or token cap you configured on the Limits page. It comes back as an HTTP 200 assistant message, not a 429. Fix: raise the threshold or wait for the period to reset." +description: "M200 is returned when a harness exceeds a cost or token limit configured in the dashboard." icon: "gauge" keywords: ["M200", "Manifest M200", "usage limit exceeded", "cost limit", "token limit", "spending cap", "budget cap", "monthly budget LLM", "Manifest limits", "HTTP 200 assistant message"] @@ -21,9 +21,9 @@ The response arrives as HTTP 200 with this text in the assistant message content ## Why it happened -You set a limit on the harness's [Limits](/llm-gateway/docs/observability#spend-alerts) page (say, "$10/day" or "500k tokens/hour"), and the current period's usage just crossed it. Manifest blocks every following request until the period resets or you raise the cap. +You set a limit on the harness's [Limits](/llm-gateway/docs/observability#spend-alerts) page (say, "$10/day" or "500k tokens/hour"), and the current period's usage just crossed it. The gateway blocks every following request until the period resets or you raise the cap. -This is different from [M204](/llm-gateway/docs/errors/M204), which is the Manifest Cloud Free plan monthly request quota, and different from provider 402 errors such as "insufficient credits" from OpenRouter. +This is different from [M204](/llm-gateway/docs/errors/M204), which is the LLM Gateway Cloud Free plan monthly request quota, and different from provider 402 errors such as "insufficient credits" from OpenRouter. ## How to fix it diff --git a/llm-gateway/docs/errors/M201.mdx b/llm-gateway/docs/errors/M201.mdx index ccf9184..79f6ca9 100644 --- a/llm-gateway/docs/errors/M201.mdx +++ b/llm-gateway/docs/errors/M201.mdx @@ -1,7 +1,7 @@ --- title: "M201: Per-user rate limit exceeded" sidebarTitle: "M201" -description: "Manifest error M201 fires when one user sends more than 200 requests per minute. Returns HTTP 429. Fix: back off and retry." +description: "M201 is returned when one user exceeds 200 requests per minute." icon: "timer" keywords: ["M201", "Manifest M201", "rate limit", "429 Too Many Requests", "per-user rate limit", "throttling", "exponential backoff", "Manifest limits"] @@ -19,12 +19,12 @@ The HTTP status is `429`, which most SDKs treat as retryable and back off from o ## Why it happened -Manifest caps each user at 200 requests per minute. The cap is there mostly to keep runaway loops from melting the proxy. Your agent crossed it inside a 60-second window. +The gateway caps each user at 200 requests per minute. The cap is there mostly to keep runaway loops from melting the proxy. Your agent crossed it inside a 60-second window. ## How to fix it 1. Add a small delay or exponential backoff on retry. Most SDKs handle this automatically when they see a `429`. -2. If parallel workers are hitting Manifest, throttle them so they share the budget. +2. If parallel workers are hitting the gateway, throttle them so they share the budget. 3. The 200 requests/minute cap is a fixed guardrail. No env var or setting changes it, self-hosted included. Lean on the backoff in step 1. ## Related diff --git a/llm-gateway/docs/errors/M202.mdx b/llm-gateway/docs/errors/M202.mdx index e5a7e00..6f240cc 100644 --- a/llm-gateway/docs/errors/M202.mdx +++ b/llm-gateway/docs/errors/M202.mdx @@ -1,7 +1,7 @@ --- title: "M202: Per-IP rate limit exceeded" sidebarTitle: "M202" -description: "Manifest error M202 fires when one IP sends more than 500 requests per minute across all harnesses. Returns HTTP 429. Fix: back off and stagger workers." +description: "M202 is returned when one IP exceeds 500 requests per minute across all harnesses." icon: "timer" keywords: ["M202", "Manifest M202", "per-IP rate limit", "shared IP", "429 Too Many Requests", "Manifest limits", "abuse protection"] diff --git a/llm-gateway/docs/errors/M203.mdx b/llm-gateway/docs/errors/M203.mdx index 08aced8..5b6a647 100644 --- a/llm-gateway/docs/errors/M203.mdx +++ b/llm-gateway/docs/errors/M203.mdx @@ -1,7 +1,7 @@ --- title: "M203: Concurrency limit exceeded" sidebarTitle: "M203" -description: "Manifest error M203 fires when a workspace has more than 10 in-flight requests. Returns HTTP 429. Common with parallel streaming. Fix: reduce parallelism." +description: "M203 is returned when a workspace has more than 10 requests in flight." icon: "timer" keywords: ["M203", "Manifest M203", "concurrency limit", "in-flight requests", "429 Too Many Requests", "parallel streams", "Manifest limits"] @@ -17,7 +17,7 @@ See https://manifest.build/llm-gateway/docs/errors/M203 ## Why it happened -Manifest caps in-flight requests at 10 per workspace, shared by all its harnesses. Streaming completions hold a slot until the stream ends, so an agent that fires off parallel streams can hit this fast. +The gateway caps in-flight requests at 10 per workspace, shared by all its harnesses. Streaming completions hold a slot until the stream ends, so an agent that fires off parallel streams can hit this fast. ## How to fix it diff --git a/llm-gateway/docs/errors/M204.mdx b/llm-gateway/docs/errors/M204.mdx index 78e2b1e..82a784c 100644 --- a/llm-gateway/docs/errors/M204.mdx +++ b/llm-gateway/docs/errors/M204.mdx @@ -1,7 +1,7 @@ --- title: "M204: Monthly request limit reached" sidebarTitle: "M204" -description: "Manifest error M204 fires when a Manifest Cloud Free workspace uses all monthly requests. Returns HTTP 402 with PLAN_LIMIT_REQUESTS. Fix: upgrade to Pro or wait for the monthly reset." +description: "M204 is returned when a free LLM Gateway Cloud workspace reaches its monthly request allowance." icon: "badge-alert" keywords: ["M204", "Manifest M204", "PLAN_LIMIT_REQUESTS", "monthly request limit", "free plan request limit", "Manifest billing", "402 Payment Required", "upgrade to Pro"] @@ -19,7 +19,7 @@ SDK and tool callers receive an HTTP 402 JSON error with `error.code` set to `PL ## Why it happened -Your Manifest Cloud workspace reached the Free plan monthly request quota. This is a Manifest plan restriction, not a provider error and not a limit you configured on a harness. +Your LLM Gateway Cloud workspace reached the Free plan monthly request quota. This is a Cloud plan restriction, not a provider error and not a limit you configured on a harness. ## How to fix it diff --git a/llm-gateway/docs/errors/M300.mdx b/llm-gateway/docs/errors/M300.mdx index b5705a3..914cb1c 100644 --- a/llm-gateway/docs/errors/M300.mdx +++ b/llm-gateway/docs/errors/M300.mdx @@ -1,7 +1,7 @@ --- title: "M300: Missing messages array" sidebarTitle: "M300" -description: "Manifest error M300 fires when the /v1/chat/completions body has no messages array (missing, not an array, or empty). Returns HTTP 400." +description: "M300 is returned when /v1/chat/completions receives no usable messages array." icon: "message-square-warning" keywords: ["M300", "Manifest M300", "messages array required", "empty messages", "400 Bad Request", "OpenAI compatible API", "chat completions validation", "Manifest validation"] @@ -17,7 +17,7 @@ See https://manifest.build/llm-gateway/docs/errors/M300 ## Why it happened -The OpenAI-compatible chat completions API requires a `messages` array with at least one entry. Manifest got a body where `messages` was missing, not an array, or empty. +The OpenAI-compatible chat completions API requires a `messages` array with at least one entry. The gateway received a body where `messages` was missing, not an array, or empty. ## How to fix it @@ -36,6 +36,6 @@ The OpenAI SDK handles this for you. If you're hand-rolling the request, eyeball ## Related -- [M303: Local provider unavailable on Manifest Cloud](/llm-gateway/docs/errors/M303) +- [M303: Local provider unavailable on LLM Gateway Cloud](/llm-gateway/docs/errors/M303) - [API reference](/llm-gateway/docs/reference/api) - [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M302.mdx b/llm-gateway/docs/errors/M302.mdx index 9e3ac97..c64d54b 100644 --- a/llm-gateway/docs/errors/M302.mdx +++ b/llm-gateway/docs/errors/M302.mdx @@ -1,7 +1,7 @@ --- title: "M302: Model not available" sidebarTitle: "M302" -description: "Manifest error M302 fires when an explicit model ID is not available for the authenticated harness. Fix: use GET /v1/models or make the provider available for this harness." +description: "M302 is returned when the requested model is unavailable to the authenticated harness." icon: "message-square-warning" keywords: ["M302", "Manifest M302", "model not available", "model not found", "GET /v1/models", "direct model routing", "provider not connected", "provider not enabled", "agent-visible models", "harness-visible models", "Manifest validation"] @@ -25,13 +25,13 @@ This can happen when: 1. The model ID is misspelled or retired. 2. The provider is not connected yet, or it is connected but not enabled for this harness. -3. You used a provider-native name when Manifest expects the exact ID returned by `GET /v1/models`. -4. Manifest refuses the request instead of falling back to automatic routing. +3. You used a provider-native name when the gateway expects the exact ID returned by `GET /v1/models`. +4. The gateway refuses the request instead of falling back to automatic routing. ## How to fix it -1. Call `GET /v1/models` with the same Manifest harness key. -2. Use `auto` to let Manifest route, or copy one of the listed model IDs exactly. +1. Call `GET /v1/models` with the same harness key. +2. Use `auto` to let the gateway route, or copy one of the listed model IDs exactly. 3. If the model should be available, open the dashboard link in the error and enable or connect that provider for this harness. 4. Retry the request. @@ -40,5 +40,5 @@ This can happen when: - [Routing: Route a specific model](/llm-gateway/docs/llm-gateway#route-a-specific-model) - [API reference: Listing models](/llm-gateway/docs/reference/api#listing-models) - [M300: Missing messages array](/llm-gateway/docs/errors/M300) -- [M303: Local provider unavailable on Manifest Cloud](/llm-gateway/docs/errors/M303) +- [M303: Local provider unavailable on LLM Gateway Cloud](/llm-gateway/docs/errors/M303) - [All error codes](/llm-gateway/docs/errors) diff --git a/llm-gateway/docs/errors/M303.mdx b/llm-gateway/docs/errors/M303.mdx index b1cf4df..6ed95d0 100644 --- a/llm-gateway/docs/errors/M303.mdx +++ b/llm-gateway/docs/errors/M303.mdx @@ -1,7 +1,7 @@ --- -title: "M303: Local provider unavailable on Manifest Cloud" +title: "M303: Local provider unavailable on Cloud" sidebarTitle: "M303" -description: "Manifest error M303 fires when a Manifest Cloud request routes to Ollama, LM Studio, or llama.cpp. Returns HTTP 400. Fix: self-host, or expose the runtime as a custom provider." +description: "M303 is returned when LLM Gateway Cloud is asked to use Ollama, LM Studio, or llama.cpp." icon: "server-off" keywords: ["M303", "Manifest M303", "local provider cloud", "Ollama Manifest Cloud", "LM Studio", "llama.cpp", "400 Bad Request", "custom provider", "tunnel"] @@ -17,9 +17,9 @@ See https://manifest.build/llm-gateway/docs/errors/M303 ## Why it happened -The built-in [local provider](/llm-gateway/docs/providers/local-models) tiles (Ollama, LM Studio, llama.cpp) route to `localhost`. On a self-hosted install that's your machine, which is the whole point. On Manifest Cloud, `localhost` is our server, which has no model running on it and no route back to your laptop. +The built-in [local provider](/llm-gateway/docs/providers/local-models) tiles (Ollama, LM Studio, llama.cpp) route to `localhost`. On a self-hosted install that's your machine, which is the whole point. On LLM Gateway Cloud, `localhost` is our server, which has no model running on it and no route back to your laptop. -Rather than dial an address that can't be yours, Manifest stops the request here. +Rather than dial an address that can't be yours, the gateway stops the request here. ## How to fix it diff --git a/llm-gateway/docs/errors/M500.mdx b/llm-gateway/docs/errors/M500.mdx index 9a42763..defa66a 100644 --- a/llm-gateway/docs/errors/M500.mdx +++ b/llm-gateway/docs/errors/M500.mdx @@ -1,7 +1,7 @@ --- title: "M500: Internal server error" sidebarTitle: "M500" -description: "Manifest error M500 is a generic 500 fallback for unhandled errors in the proxy. Usually transient. Fix: retry, then check backend logs on self-hosted." +description: "M500 is returned when the gateway encounters an unhandled internal error." icon: "server-crash" keywords: ["M500", "Manifest M500", "internal server error", "500 Internal Server Error", "5xx", "proxy crash", "Manifest server error", "self-hosted debugging"] @@ -17,7 +17,7 @@ See https://manifest.build/llm-gateway/docs/errors/M500 ## Why it happened -Manifest hit an unexpected error while handling your request. Usually that's a database hiccup, a downstream provider returning something unparseable, or a real bug. The actual stack trace is logged server-side, but masked in the response so we don't leak internals. +The gateway hit an unexpected error while handling your request. Usually that's a database hiccup, a downstream provider returning something unparseable, or a real bug. The actual stack trace is logged server-side, but masked in the response so we don't leak internals. ## How to fix it diff --git a/llm-gateway/docs/integrations/n8n.mdx b/llm-gateway/docs/integrations/n8n.mdx index 21148f0..b8f28dc 100644 --- a/llm-gateway/docs/integrations/n8n.mdx +++ b/llm-gateway/docs/integrations/n8n.mdx @@ -1,6 +1,6 @@ --- title: "n8n" -description: "Use Manifest as the chat model behind n8n AI Agents, or call it directly from a workflow, with the n8n-nodes-manifest community node." +description: "Use the Manifest LLM Gateway as the model endpoint for an n8n AI Agent or call it directly from a workflow." icon: "workflow" keywords: ["n8n", "n8n community node", "n8n-nodes-manifest", "n8n AI Agent", "Manifest Chat Model", "n8n self-hosted", "n8n LLM router", "n8n OpenAI alternative"] @@ -12,14 +12,14 @@ canonical: "https://manifest.build/llm-gateway/docs/integrations/n8n" -Manifest ships as a verified n8n community node, [`n8n-nodes-manifest`](https://www.npmjs.com/package/n8n-nodes-manifest). It installs from n8n's own **Community Nodes** screen on n8n Cloud and on self-hosted n8n alike, and gives you two nodes: +The gateway ships as a verified n8n community node, [`n8n-nodes-manifest`](https://www.npmjs.com/package/n8n-nodes-manifest). It installs from n8n's own **Community Nodes** screen on n8n Cloud and on self-hosted n8n alike, and gives you two nodes: - A language-model sub-node. Plug it into the **AI Agent** or **Basic LLM Chain** node in place of the OpenAI Chat Model, and Manifest routes every request the agent makes. + A language-model sub-node. Plug it into the **AI Agent** or **Basic LLM Chain** node in place of the OpenAI Chat Model, and the gateway routes every request the agent makes. - An action node for calling Manifest directly from a workflow: list models, create a chat completion, or create a Responses API call. + An action node for calling the gateway directly from a workflow: list models, create a chat completion, or create a Responses API call. @@ -71,24 +71,24 @@ Select **Test** on the credential. It calls `GET /v1/models`, so a passing test ### Reaching a self-hosted instance from n8n -The credential test fails with a connection error when n8n cannot reach Manifest. Where the two run decides the address: +The credential test fails with a connection error when n8n cannot reach the gateway. Where the two run decides the address: -- **n8n and Manifest in Docker on the same host.** `localhost` inside the n8n container is n8n itself. Put both containers on a shared Docker network and use the Manifest service name as the host, or use `http://host.docker.internal:2099` on Docker Desktop. -- **n8n on another machine.** The bundled compose file binds Manifest to `127.0.0.1` only. Set `HOST_BIND_ADDRESS=0.0.0.0` in Manifest's `.env`, as described in [Exposing on the LAN](/llm-gateway/docs/self-hosted#exposing-on-the-lan), so the port answers on the network. +- **n8n and the gateway in Docker on the same host.** `localhost` inside the n8n container is n8n itself. Put both containers on a shared Docker network and use the gateway service name as the host, or use `http://host.docker.internal:2099` on Docker Desktop. +- **n8n on another machine.** The bundled compose file binds the gateway to `127.0.0.1` only. Set `HOST_BIND_ADDRESS=0.0.0.0` in the gateway's `.env`, as described in [Exposing on the LAN](/llm-gateway/docs/self-hosted#exposing-on-the-lan), so the port answers on the network. - **Plain `http://` is fine.** The calls are server to server. The dashboard's browser security rules do not apply to them. -The keyless loopback shortcut that a development build of Manifest accepts does not exist in the production image. n8n always needs a real `mnfst_` key. +The keyless loopback shortcut that a development build of the gateway accepts does not exist in the production image. n8n always needs a real `mnfst_` key. -## Use Manifest behind an AI Agent +## Use the gateway behind an AI Agent On an **AI Agent** or **Basic LLM Chain** node, select the **Model** connector and choose **Manifest Chat Model**. - Select the Manifest credential. The model list is loaded from your instance. Leave it on `auto` to let your [routing rules](/llm-gateway/docs/llm-gateway#routing) pick the model, or choose a model ID to pin one. + Select the **Manifest API** credential. The model list is loaded from your instance. Leave it on `auto` to let your [routing rules](/llm-gateway/docs/llm-gateway#routing) pick the model, or choose a model ID to pin one. Streaming and tool calling work as they do with the OpenAI Chat Model, so agent tools run unchanged. Each call appears on the harness's **Requests** page with the model that served it and its cost. @@ -97,7 +97,7 @@ The keyless loopback shortcut that a development build of Manifest accepts does An option on the sub-node switches it from Chat Completions to the [Responses API](/llm-gateway/docs/reference/api#endpoints). Leave it off unless a model you route to needs Responses. -## Call Manifest from a workflow +## Call the gateway from a workflow The **Manifest** action node has three operations: @@ -111,7 +111,7 @@ For a first test, choose **Create Chat Completion**, set **Model** to `auto`, an ```json [ - { "role": "user", "content": "Reply with one sentence confirming that Manifest is connected." } + { "role": "user", "content": "Reply with one sentence confirming that the gateway is connected." } ] ``` diff --git a/llm-gateway/docs/introduction.mdx b/llm-gateway/docs/introduction.mdx index 216e522..a05b54e 100644 --- a/llm-gateway/docs/introduction.mdx +++ b/llm-gateway/docs/introduction.mdx @@ -1,7 +1,7 @@ --- -title: "Manifest documentation" +title: "Manifest LLM Gateway documentation" sidebarTitle: "Introduction" -description: "Open-source LLM router for agents. Connect your subscriptions, API keys, and local models to any harness through one endpoint, with fallbacks and spend limits." +description: "Connect models and subscriptions behind one OpenAI- and Anthropic-compatible endpoint, with routing, fallbacks, limits, and Autofix." icon: "house" keywords: ["LLM router", "AI subscriptions", "model routing", "open source", "LLM gateway", "quickstart", "first request", "create a harness", "gateway base URL"] @@ -13,9 +13,9 @@ canonical: "https://manifest.build/llm-gateway/docs/introduction" -Manifest is an open-source LLM router for agents and AI apps. Connect the providers you already use — subscriptions like ChatGPT or Claude, pay-per-token API keys, local models, and custom endpoints — and route every request through one OpenAI- and Anthropic-compatible URL. +The Manifest LLM Gateway connects the models and subscriptions you already use behind one OpenAI- and Anthropic-compatible endpoint. -Send `auto` as the model and Manifest routes each request by the rules you set, with fallbacks when one fails. Pin a real model ID instead, and the request goes straight to that model, with no routing and no fallback. Either way, you cap what each harness spends and see the cost of every request. +Send `auto` to apply your routing rules and fallbacks, or name a model to send the request there directly. Either way, you can cap what each harness spends and see the cost of every request. ## Key features @@ -39,10 +39,10 @@ Send `auto` as the model and Manifest routes each request by the rules you set, ## Cloud or self-hosted? -Manifest is open source and runs two ways. Both speak the same API, so pointing a client at one or the other is a URL change. +The gateway is open source and runs two ways. Both speak the same API, so pointing a client at one or the other is a URL change. - + Runs on our servers. Sign up, connect a provider, and you're routing. Recommended if you're starting out. @@ -52,7 +52,7 @@ Manifest is open source and runs two ways. Both speak the same API, so pointing -On the cloud there is nothing to install and nothing to configure on a server. You sign up, connect your providers, point your harness at the gateway URL, and everything else — routing, limits, [Autofix](/llm-gateway/docs/autofix), alerts — is a setting in the dashboard. The [environment variables](/llm-gateway/docs/reference/environment-variables) throughout these docs apply only to a Manifest instance you run yourself. +On Cloud there is nothing to install or configure on a server. You sign up, connect your providers, and point your harness at the gateway URL. Routing, limits, [Autofix](/llm-gateway/docs/autofix), and alerts are settings in the dashboard. The [environment variables](/llm-gateway/docs/reference/environment-variables) throughout these docs apply only to a self-hosted gateway. Cloud is also subject to [plan limits](/llm-gateway/docs/errors/M204): the free plan includes 10,000 routed requests per calendar month, shown by the usage meter in the dashboard sidebar and reset at 00:00 UTC on the 1st. And it can't reach [local models](/llm-gateway/docs/providers/local-models) running on your machine. A self-hosted instance has neither restriction. @@ -63,7 +63,7 @@ Cloud is also subject to [plan limits](/llm-gateway/docs/errors/M204): the free Sign up at [app.manifest.build](https://app.manifest.build) and a guided setup walks you through it. Step one creates a harness. Then connect a provider, and give your default tier a model to route to. The last step shows the gateway address and the harness key, which starts with `mnfst_`. A harness you create later reuses the providers your workspace already has. - The Manifest guided setup on step one, Create your first harness, with the five setup steps listed in the sidebar + The LLM Gateway guided setup on step one, Create your first harness, with the five setup steps listed in the sidebar ```bash @@ -73,12 +73,12 @@ curl -X POST https://app.manifest.build/v1/chat/completions \ -d '{"model": "auto", "messages": [{"role": "user", "content": "Hello"}]}' ``` -`auto` asks Manifest to route the request. The request appears on the Requests page with the model that served it and its cost. +`auto` asks the gateway to route the request. The request appears on the **Requests** page with the model that served it and its cost.
-Install Manifest first: [Self-hosted](/llm-gateway/docs/self-hosted) covers Docker and the managed platforms. Your instance starts empty. [First request](/llm-gateway/docs/self-hosted#first-request) takes you from there to a routed request, against `http://localhost:2099/v1`. +Install the gateway first. [Self-hosted](/llm-gateway/docs/self-hosted) covers Docker and the managed platforms. Your instance starts empty. [First request](/llm-gateway/docs/self-hosted#first-request) takes you from there to a routed request against `http://localhost:2099/v1`.
diff --git a/llm-gateway/docs/llm-gateway.mdx b/llm-gateway/docs/llm-gateway.mdx index 6fa8c30..0d2b215 100644 --- a/llm-gateway/docs/llm-gateway.mdx +++ b/llm-gateway/docs/llm-gateway.mdx @@ -1,13 +1,13 @@ --- title: "LLM Gateway" -description: "Every request passes through the gateway: limits check it, routing picks the model, and fallback catches it when that model fails." +description: "Route requests across models, enforce limits, and use fallbacks when a provider fails." icon: "split" keywords: ["LLM gateway", "model routing", "LLM router", "fallback chain", "hard limit", "custom tier", "default model", "fallback exhausted", "429 too many requests"] canonical: "https://manifest.build/llm-gateway/docs/llm-gateway" --- -Point your client at Manifest instead of a provider, and every request goes through the gateway. Three things happen there: limits decide whether the request runs at all, routing picks the model, and fallback steps in when that model fails. +Point your client at the gateway instead of a provider, and every request goes through the gateway. Three things happen there: limits decide whether the request runs at all, routing picks the model, and fallback steps in when that model fails. ## How a request flows @@ -22,7 +22,7 @@ Point your client at Manifest instead of a provider, and every request goes thro A custom tier matches on a request header, or the request goes to your default tier. Send a real model ID instead of `auto` to skip routing. A matching custom tier header still wins. - Manifest calls the resolved provider with your credentials and streams the response back. + The gateway calls the resolved provider with your credentials and streams the response back. If the provider returns an error, [Autofix](/llm-gateway/docs/autofix) may repair and resend the request once. Anything still failing moves down the fallback chain. @@ -33,7 +33,7 @@ Routing runs in-process. There's no extra network call and no added latency. ## Routing -Instead of hard-coding one model into every client, you point your client at Manifest and let it pick. Send `auto` as the model, and Manifest resolves the real model based on the rules you set on the dashboard **Routing** page. +Instead of hard-coding one model into every client, you point your client at the gateway and let it pick. Send `auto` as the model, and the gateway resolves the real model based on the rules you set on the dashboard **Routing** page. @@ -50,7 +50,7 @@ Every harness has a default tier: one model plus up to five fallbacks. You set i ### Custom tiers -Custom tiers route by request header. You create a tier on the dashboard, give it a header key and value, and pin it to a model with its own fallbacks. When an incoming request carries that header, Manifest sends it to that tier's model instead of the default. The header wins even when the body names an explicit model ID. +Custom tiers route by request header. You create a tier on the dashboard, give it a header key and value, and pin it to a model with its own fallbacks. When an incoming request carries that header, the gateway sends it to that tier's model instead of the default. The header wins even when the body names an explicit model ID. The header key is yours to choose (lowercase letters, numbers, and hyphens). A few names are reserved and rejected, including `authorization`, `cookie`, and `x-api-key`. Send the header from your client like any other: @@ -73,7 +73,7 @@ Create as many tiers as you need, each with its own model and parameters. This i ### Route a specific model -To skip routing for a single request, send a real model ID instead of `auto`. Manifest forwards it straight to that model's provider, with no tier lookup and no fallbacks. A matching [custom tier](#custom-tiers) header is the exception. The request then runs on the tier's model and the tier's fallbacks. Call [`GET /v1/models`](/llm-gateway/docs/reference/api#listing-models) to list the model IDs your harness can reach. +To skip routing for a single request, send a real model ID instead of `auto`. The gateway forwards it straight to that model's provider, with no tier lookup and no fallbacks. A matching [custom tier](#custom-tiers) header is the exception. The request then runs on the tier's model and the tier's fallbacks. Call [`GET /v1/models`](/llm-gateway/docs/reference/api#listing-models) to list the model IDs your harness can reach. ```bash curl -X POST https://app.manifest.build/v1/chat/completions \ @@ -87,11 +87,11 @@ curl -X POST https://app.manifest.build/v1/chat/completions \ The response comes back with `X-Manifest-Tier: direct`, so you can tell a direct call from a routed one. A direct model ID works on all three endpoints: `/v1/chat/completions`, `/v1/responses`, and the Anthropic `/v1/messages`. -The model list is scoped to the harness key. A model may be missing because its provider is not connected yet, or because the provider exists in your workspace but is not enabled for this harness. If you send a model ID that has no connected provider route, Manifest returns [M302: Model not available](/llm-gateway/docs/errors/M302). Send `auto` to use routing. +The model list is scoped to the harness key. A model may be missing because its provider is not connected yet, or because the provider exists in your workspace but is not enabled for this harness. If you send a model ID that has no connected provider route, the gateway returns [M302: Model not available](/llm-gateway/docs/errors/M302). Send `auto` to use routing. ## Fallback -When a model fails (provider outage, rate limit, bad request), Manifest retries with a backup model from the same tier. Your agent gets a response instead of an error. +When a model fails (provider outage, rate limit, bad request), the gateway retries with a backup model from the same tier. Your agent gets a response instead of an error. ### What triggers a fallback @@ -132,9 +132,9 @@ Fallback models are configured **per tier**. Each tier holds up to **5 fallback ### Hung providers and the per-attempt timeout -A provider that opens a connection but never returns eventually triggers a fallback via Manifest's per-attempt timeout (default 180 seconds), which surfaces as a synthetic `504 Gateway Timeout` and moves to the next model in the chain. +A provider that opens a connection but never returns eventually triggers a fallback via the gateway's per-attempt timeout (default 180 seconds), which surfaces as a synthetic `504 Gateway Timeout` and moves to the next model in the chain. -If your upstream client has its own timeout that fires at the same time, the client may disconnect first and Manifest will give up before reaching a healthy fallback. On self-hosted installs, lower [`PROVIDER_TIMEOUT_MS`](/llm-gateway/docs/reference/environment-variables) strictly below your client's timeout so the chain has room to run inside the client's window. +If your upstream client has its own timeout that fires at the same time, the client may disconnect first and the gateway will give up before reaching a healthy fallback. On self-hosted installs, lower [`PROVIDER_TIMEOUT_MS`](/llm-gateway/docs/reference/environment-variables) strictly below your client's timeout so the chain has room to run inside the client's window. ### Response headers @@ -151,7 +151,7 @@ A limit rule can block requests as well as email you. When a rule's action inclu The check runs before any provider is contacted, so a blocked request costs nothing. The block clears on its own at the start of the next period, or immediately if you raise the threshold. - On Manifest Cloud, running out of the Free plan's monthly requests is a + On LLM Gateway Cloud, running out of the Free plan's monthly requests is a different thing entirely: that returns [M204](/llm-gateway/docs/errors/M204) with HTTP `402` and isn't something you configured. diff --git a/llm-gateway/docs/observability.mdx b/llm-gateway/docs/observability.mdx index f2ceac7..36e40dc 100644 --- a/llm-gateway/docs/observability.mdx +++ b/llm-gateway/docs/observability.mdx @@ -1,6 +1,6 @@ --- title: "Observability" -description: "See what every harness spent, which models served it, and what failed. Set email alerts when spending crosses a threshold." +description: "Track cost, latency, models, providers, and failures across requests and provider attempts." icon: "activity" keywords: ["LLM observability", "AI cost tracking", "token usage", "spend alerts", "request log", "LLM analytics", "cost per agent", "cost per harness", "provider errors"] @@ -16,14 +16,14 @@ Every request through the gateway is recorded: which model served it, what it co ## Requests and provider attempts -Manifest tracks two things, and the difference matters once fallback is involved: +The gateway tracks two things, and the difference matters once fallback is involved: | | What it is | When you get more than one | |---|---|---| -| **Request** | One call from your agent to Manifest | Never. One call in, one Request. | -| **Provider attempt** | One call from Manifest to a provider | Every fallback step and every [Autofix](/llm-gateway/docs/autofix) retry adds another | +| **Request** | One call from your agent to the gateway | Never. One call in, one Request. | +| **Provider attempt** | One call from the gateway to a provider | Every fallback step and every [Autofix](/llm-gateway/docs/autofix) retry adds another | -A request that succeeds on the first try is one Request with one attempt. A request that fails twice before succeeding is still one Request, with three attempts. A request Manifest blocked itself, like a [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) or a malformed body, is one Request with one attempt that never reached a provider. That attempt is listed under the model you asked for, or **No provider** when the request named none, and carries the error that stopped it. +A request that succeeds on the first try is one Request with one attempt. A request that fails twice before succeeding is still one Request, with three attempts. A request blocked by the gateway, like a [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) or a malformed body, is one Request with one attempt that never reached a provider. That attempt is listed under the model you asked for, or **No provider** when the request named none, and carries the error that stopped it. This is why the request count and the provider call count don't match, and why cost is attributed to attempts: a failed attempt can still burn tokens. @@ -52,13 +52,13 @@ On the Free plan, the Overview covers up to 7 days of history; the 30-, 90-, and ### What's on each request -Opening a request in the log shows the whole story: the tier that routed it, the model and provider that served it, input and output tokens, computed cost, latency, the auth type used ([API key, subscription, or local](/llm-gateway/docs/reference/glossary#auth-type)), and every fallback hop with its own status. Failures carry an error code, so a Manifest rejection like [M100](/llm-gateway/docs/errors/M100) reads differently from a provider's own 500. +Opening a request in the log shows the whole story: the tier that routed it, the model and provider that served it, input and output tokens, computed cost, latency, the auth type used ([API key, subscription, or local](/llm-gateway/docs/reference/glossary#auth-type)), and every fallback hop with its own status. Failures carry an error code, so a gateway rejection like [M100](/llm-gateway/docs/errors/M100) reads differently from a provider's own 500. That's all metadata. To read the messages themselves, turn on [request logs](/llm-gateway/docs/request-logs) for the harness. Open a request, pick an attempt, and its **Messages** tab shows the body sent to the provider and the body that came back. Local models record `cost = 0` with real token counts and latency, since - nothing was billed. Custom providers do the same, because Manifest can't + nothing was billed. Custom providers do the same, because the gateway can't infer pricing for a model it doesn't know. diff --git a/llm-gateway/docs/providers/api-key-providers.mdx b/llm-gateway/docs/providers/api-key-providers.mdx index 165f081..ec942ee 100644 --- a/llm-gateway/docs/providers/api-key-providers.mdx +++ b/llm-gateway/docs/providers/api-key-providers.mdx @@ -1,14 +1,14 @@ --- title: "API key providers" sidebarTitle: "API keys" -description: "Bring your own API key for OpenAI, Anthropic, Google, xAI, DeepSeek, Mistral, Groq, Cerebras, AWS Bedrock, NVIDIA NIM, OpenRouter, Hugging Face, and more." +description: "Connect provider API keys for OpenAI, Anthropic, Google, xAI, DeepSeek, Mistral, Groq, Cerebras, AWS Bedrock, NVIDIA NIM, OpenRouter, Hugging Face, and more." icon: "key" keywords: ["OpenAI", "Anthropic", "Google Gemini", "xAI", "DeepSeek", "Mistral", "Qwen", "Moonshot", "MiniMax", "Z.ai", "OpenRouter", "Groq", "Cerebras", "Fireworks", "AWS Bedrock", "NVIDIA NIM", "Xiaomi MiMo", "Hugging Face", "Gemini Free", "API key"] canonical: "https://manifest.build/llm-gateway/docs/providers/api-key-providers" --- -Most providers work the standard way: sign up, generate an API key, paste it into Manifest. Routed requests go out with that key as the credential. +Most providers work the standard way: sign up, generate an API key, paste it into the gateway. Routed requests go out with that key as the credential. ## Supported providers @@ -41,7 +41,7 @@ New providers are added often, so **Providers → Usage-based** in the dashboard ### Gemini Free -Gemini Free is a managed tile. Manifest provisions the credential through its own gateway, so there's no key to fetch from Google first: add the tile and you have a free model to route to. It behaves like any other provider once connected. +Gemini Free is a managed tile. The gateway provisions the credential automatically, so there's no key to fetch from Google first: add the tile and you have a free model to route to. It behaves like any other provider once connected. BytePlus, NousResearch, Command Code, ClinePass, Kiro, GitHub Copilot, Ollama @@ -61,7 +61,7 @@ Gemini Free is a managed tile. Manifest provisions the credential through its ow **Connect** opens a panel where you paste your API key. - Manifest validates the key by fetching the provider's model list. That list + The gateway validates the key by fetching the provider's model list. That list is what routing sees until you refresh it. If the key is rejected, double-check the prefix below. @@ -105,4 +105,4 @@ Providers not listed here (Cerebras, NVIDIA NIM, AWS Bedrock, Kilo, OpenCode Zen ## Regional providers -Some providers serve more than one region. Alibaba Cloud, AWS Bedrock, and MiniMax let you pick a region when you connect them, and Manifest routes to the matching endpoint. +Some providers serve more than one region. Alibaba Cloud, AWS Bedrock, and MiniMax let you pick a region when you connect them, and the gateway routes to the matching endpoint. diff --git a/llm-gateway/docs/providers/custom-providers.mdx b/llm-gateway/docs/providers/custom-providers.mdx index 14e2499..a514a68 100644 --- a/llm-gateway/docs/providers/custom-providers.mdx +++ b/llm-gateway/docs/providers/custom-providers.mdx @@ -1,14 +1,14 @@ --- title: "Custom providers" sidebarTitle: "Custom" -description: "Plug in any OpenAI- or Anthropic-compatible endpoint, including vLLM, TGI, LocalAI, or your own internal service." +description: "Connect any OpenAI- or Anthropic-compatible endpoint, including vLLM, TGI, LocalAI, or an internal service." icon: "plug" keywords: ["vLLM", "TGI", "text-generation-inference", "LocalAI", "Xinference", "OpenLLM", "OpenAI-compatible", "Anthropic-compatible", "custom provider"] canonical: "https://manifest.build/llm-gateway/docs/providers/custom-providers" --- -If your endpoint speaks OpenAI or Anthropic, Manifest can route to it. Useful for self-hosted inference servers (vLLM, TGI, LocalAI), internal endpoints behind your VPN, or providers that aren't on the built-in list yet. +If your endpoint speaks OpenAI or Anthropic, the gateway can route to it. Useful for self-hosted inference servers (vLLM, TGI, LocalAI), internal endpoints behind your VPN, or providers that aren't on the built-in list yet. ## Compatible servers @@ -29,7 +29,7 @@ Common options that ship with one of these formats: [vLLM](https://github.com/vl **Add custom provider**. - Paste the base URL of your endpoint, e.g. `https://my-vllm.internal:8000/v1`. Manifest normalizes the trailing `/v1` automatically. + Paste the base URL of your endpoint, e.g. `https://my-vllm.internal:8000/v1`. The gateway normalizes the trailing `/v1` automatically. Choose **OpenAI** (`/v1/chat/completions`) or **Anthropic** (`/v1/messages`), whichever your server speaks. @@ -38,13 +38,13 @@ Common options that ship with one of these formats: [vLLM](https://github.com/vl If the endpoint requires authentication, paste an API key. It's sent as `Authorization: Bearer ` for OpenAI-format endpoints, or `x-api-key: ` for Anthropic-format endpoints. - Manifest calls `GET /v1/models` against your base URL and lists every model the endpoint reports. Pin one to your default or a custom tier and you're routed. + The gateway calls `GET /v1/models` against your base URL and lists every model the endpoint reports. Pin one to your default or a custom tier and you're routed. ## Model discovery -Manifest discovers models by hitting `GET /v1/models` and reading the `data[].id` field. If your server doesn't expose `/v1/models`, you can register models manually from the same panel. +The gateway discovers models by hitting `GET /v1/models` and reading the `data[].id` field. If your server doesn't expose `/v1/models`, you can register models manually from the same panel. Older builds of `llama.cpp` (pre-b3800) don't expose `/v1/models`. Either @@ -53,14 +53,14 @@ Manifest discovers models by hitting `GET /v1/models` and reading the ## Security -User-supplied URLs are revalidated on every request to defend against SSRF. On Manifest Cloud, Manifest blocks resolution to private IP ranges (`10.x`, `192.168.x`, `127.x`, link-local): a custom provider must use a public `https://` URL. Self-hosted deployments accept private addresses and plain `http://`, so they can reach a server inside your own network. +User-supplied URLs are revalidated on every request to defend against SSRF. On LLM Gateway Cloud, the gateway blocks resolution to private IP ranges (`10.x`, `192.168.x`, `127.x`, link-local): a custom provider must use a public `https://` URL. Self-hosted deployments accept private addresses and plain `http://`, so they can reach a server inside your own network. - Custom providers run with your Manifest instance's network access. If you - expose Manifest publicly, anyone with a valid harness key can route requests + Custom providers run with your gateway instance's network access. If you + expose the gateway publicly, anyone with a valid harness key can route requests through any custom provider you've added. Gate access accordingly. ## Cost tracking -Manifest can't infer pricing for unknown models. Custom-provider requests show up in the dashboard with `cost = 0` and `model = `. Token counts and latency are still recorded, so [hard limits](/llm-gateway/docs/llm-gateway#hard-limits) on token volume still work. +The gateway can't infer pricing for unknown models. Custom-provider requests show up in the dashboard with `cost = 0` and `model = `. Token counts and latency are still recorded, so [hard limits](/llm-gateway/docs/llm-gateway#hard-limits) on token volume still work. diff --git a/llm-gateway/docs/providers/local-models.mdx b/llm-gateway/docs/providers/local-models.mdx index da4714c..4090e37 100644 --- a/llm-gateway/docs/providers/local-models.mdx +++ b/llm-gateway/docs/providers/local-models.mdx @@ -1,16 +1,16 @@ --- title: "Local models" -description: "Run any GGUF model on your own hardware with Ollama, LM Studio, or llama.cpp. No API costs, no data leaving your machine." +description: "Connect a self-hosted LLM Gateway to Ollama, LM Studio, or llama.cpp on your own hardware." icon: "server" keywords: ["Ollama", "LM Studio", "llama.cpp", "GGUF", "local LLM", "self-hosted inference", "host.docker.internal"] canonical: "https://manifest.build/llm-gateway/docs/providers/local-models" --- -Local model providers run entirely on your own hardware. Manifest detects the running server, fetches the model list, and routes requests to `http://localhost:` like any other provider. No API key, no network egress, no per-token cost. +Local model providers run entirely on your own hardware. The gateway detects the running server, fetches the model list, and routes requests to `http://localhost:` like any other provider. No API key, no network egress, no per-token cost. - The built-in Ollama, LM Studio, and llama.cpp connections are self-hosted only. To reach a local model from [Manifest Cloud](https://app.manifest.build), expose the runtime on a public URL or tunnel and add it as a [custom provider](/llm-gateway/docs/providers/custom-providers). + The built-in Ollama, LM Studio, and llama.cpp connections are self-hosted only. To reach a local model from [LLM Gateway Cloud](https://app.manifest.build), expose the runtime on a public URL or tunnel and add it as a [custom provider](/llm-gateway/docs/providers/custom-providers). ## Supported runtimes @@ -46,7 +46,7 @@ All three speak OpenAI-compatible `/v1/chat/completions` and accept any GGUF mod -## Connect to Manifest +## Connect to the gateway @@ -54,16 +54,16 @@ All three speak OpenAI-compatible `/v1/chat/completions` and accept any GGUF mod your runtime (Ollama, LM Studio, or llama.cpp). - Manifest probes `http://localhost:/v1/models`. If the probe succeeds, every loaded model appears for routing. + The gateway probes `http://localhost:/v1/models`. If the probe succeeds, every loaded model appears for routing. Open your default or a custom tier and pick a local model as the primary. You can mix local and cloud models in the same fallback chain. -## Running Manifest in Docker +## Running the gateway in Docker -If you self-host Manifest in Docker, the container can't reach a local server bound to `127.0.0.1` on the host. Two of the three runtimes default to loopback and need an explicit override: +If you self-host the gateway in Docker, the container can't reach a local server bound to `127.0.0.1` on the host. Two of the three runtimes default to loopback and need an explicit override: @@ -90,8 +90,8 @@ If you self-host Manifest in Docker, the container can't reach a local server bo - Inside the Manifest container, the host is reachable as - `host.docker.internal`. Manifest sets this automatically when probing local + Inside the gateway container, the host is reachable as + `host.docker.internal`. The gateway sets this automatically when probing local providers. diff --git a/llm-gateway/docs/providers/subscription-based-providers.mdx b/llm-gateway/docs/providers/subscription-based-providers.mdx index 4992a58..d737d79 100644 --- a/llm-gateway/docs/providers/subscription-based-providers.mdx +++ b/llm-gateway/docs/providers/subscription-based-providers.mdx @@ -1,14 +1,14 @@ --- title: "Subscription-based providers" sidebarTitle: "Subscription-based" -description: "Reuse a paid plan you already have: ChatGPT, Claude, Gemini, Grok, GitHub Copilot, GLM Coding Plan, Kimi, Mistral Vibe, and more." +description: "Use supported subscriptions such as ChatGPT, Claude, Gemini, Grok, GitHub Copilot, GLM Coding Plan, Kimi, and Mistral Vibe." icon: "credit-card" keywords: ["ChatGPT Plus", "Claude Max", "GitHub Copilot", "GLM Coding Plan", "Mistral Vibe", "Gemini", "Grok", "Kimi", "OAuth", "device code"] canonical: "https://manifest.build/llm-gateway/docs/providers/subscription-based-providers" --- -If you already pay for ChatGPT Plus, Claude Max, GitHub Copilot, or one of the other plans listed below, Manifest can route through the subscription instead of an API key. Auth is OAuth, a device code, or a pasted subscription token, depending on the provider. +If you already pay for ChatGPT Plus, Claude Max, GitHub Copilot, or one of the other plans listed below, the gateway can route through the subscription instead of an API key. Auth is OAuth, a device code, or a pasted subscription token, depending on the provider. ## Supported subscriptions @@ -48,7 +48,7 @@ If you already pay for ChatGPT Plus, Claude Max, GitHub Copilot, or one of the o **Connect** on the provider. - Manifest opens the provider's OAuth page, shows you a device code to enter, + The gateway opens the provider's OAuth page, shows you a device code to enter, or asks you to paste a token, depending on the provider. Once it's saved, you're routed. @@ -66,19 +66,19 @@ A model your plan just gained doesn't show up on its own. Click **Refresh models Generate a token with the Claude CLI (`claude setup-token`) and paste it into - Manifest. Requests carry the `anthropic-beta: oauth-2025-04-20` header so + the gateway. Requests carry the `anthropic-beta: oauth-2025-04-20` header so Anthropic recognizes the subscription token. Available models match what your plan grants in claude.ai. - GitHub's device-code flow. Manifest gives you a short user code, you visit + GitHub's device-code flow. The gateway gives you a short user code, you visit [github.com/login/device](https://github.com/login/device), paste it, and approve. No API key is ever issued. Requests then go to - `api.githubcopilot.com/chat/completions` using a short-lived token Manifest + `api.githubcopilot.com/chat/completions` using a short-lived token the gateway refreshes for you. - MiniMax's device-code flow. Manifest gives you a short user code to approve, + MiniMax's device-code flow. The gateway gives you a short user code to approve, then requests use the Anthropic protocol via `api.minimax.io/anthropic` (international) or `api.minimaxi.com/anthropic` (China). @@ -95,7 +95,7 @@ A model your plan just gained doesn't show up on its own. Click **Refresh models - Subscription auth is rate-limited by the provider's plan, not by Manifest. If + Subscription auth is rate-limited by the provider's plan, not by the gateway. If you hit a plan ceiling, requests fall through to your next [fallback](/llm-gateway/docs/llm-gateway#fallback) model, usually a different provider on an API key. @@ -104,4 +104,4 @@ A model your plan just gained doesn't show up on its own. Click **Refresh models A common setup: subscription as the primary (predictable monthly cost), API-key provider as the fallback for when you hit the plan limit or want a model the subscription doesn't include. -Pin your subscription model to your default or a custom tier and add API-key models to the [fallback list](/llm-gateway/docs/llm-gateway#configuration). Manifest handles the switch. +Pin your subscription model to your default or a custom tier and add API-key models to the [fallback list](/llm-gateway/docs/llm-gateway#configuration). The gateway handles the switch. diff --git a/llm-gateway/docs/reference/api.mdx b/llm-gateway/docs/reference/api.mdx index 2fb2a71..65124ff 100644 --- a/llm-gateway/docs/reference/api.mdx +++ b/llm-gateway/docs/reference/api.mdx @@ -1,13 +1,13 @@ --- title: "API" -description: "The Manifest proxy speaks both OpenAI and Anthropic. Endpoints, auth, streaming, and error responses." +description: "OpenAI- and Anthropic-compatible endpoints exposed by the Manifest LLM Gateway." icon: "code" keywords: ["chat completions", "responses API", "Anthropic messages", "auto", "v1/models", "streaming", "SSE", "fallback exhausted"] canonical: "https://manifest.build/llm-gateway/docs/reference/api" --- -Manifest exposes both OpenAI and Anthropic-format endpoints on one proxy. Point your client at the Manifest URL, send `auto` as the model, and routing picks the real model behind the scenes. +The gateway exposes both OpenAI and Anthropic-format endpoints on one proxy. Point your client at the gateway URL, send `auto` as the model, and routing picks the real model behind the scenes. ## Base URL @@ -18,7 +18,7 @@ Manifest exposes both OpenAI and Anthropic-format endpoints on one proxy. Point ## Authentication -Every request requires a Manifest harness key: +Every request requires a harness key: ```http Authorization: Bearer mnfst_YOUR_KEY_HERE @@ -35,9 +35,9 @@ Generate a key from the dashboard's **Harnesses** page. Keys always start with ` | `POST` | `/v1/messages` | Anthropic | Anthropic SDK, Claude Code, anything that speaks the Messages API | | `GET` | `/v1/models` | OpenAI | Listing the models your harness can route to | -The proxy translates between formats internally, so you can send an OpenAI-shaped request and Manifest will reshape it before forwarding to an Anthropic-only model. The reverse works too. +The proxy translates between formats internally, so you can send an OpenAI-shaped request and the gateway will reshape it before forwarding to an Anthropic-only model. The reverse works too. -Translation carries what the request itself contains: messages, tools, and tool results. Manifest does not resolve `previous_response_id`, so send the full conversation in `input` on every `/v1/responses` request. +Translation carries what the request itself contains: messages, tools, and tool results. The gateway does not resolve `previous_response_id`, so send the full conversation in `input` on every `/v1/responses` request. ## Chat completions @@ -53,7 +53,7 @@ curl -X POST http://localhost:2099/v1/chat/completions \ }' ``` -Manifest adapts the body to the provider it picked. It replaces `model` with the real model ID. It converts the body to the format that provider expects. It renames parameters the provider spells differently, like `max_tokens` and `max_completion_tokens`. And it merges the model parameters saved on your harness into every attempt. Some fields only exist at OpenAI: `stream_options`, `reasoning_effort`, `modalities`, `audio`, and `prediction`. Those reach OpenAI and OpenRouter. Every other provider gets the request without them, and nothing warns you. +The gateway adapts the body to the provider it picked. It replaces `model` with the real model ID. It converts the body to the format that provider expects. It renames parameters the provider spells differently, like `max_tokens` and `max_completion_tokens`. And it merges the model parameters saved on your harness into every attempt. Some fields only exist at OpenAI: `stream_options`, `reasoning_effort`, `modalities`, `audio`, and `prediction`. Those reach OpenAI and OpenRouter. Every other provider gets the request without them, and nothing warns you. ## Anthropic messages @@ -80,7 +80,7 @@ curl http://localhost:2099/v1/models \ -H "Authorization: Bearer mnfst_YOUR_KEY_HERE" ``` -Send `auto` to let Manifest route, or send any listed model ID to skip routing and go straight to that provider. If you send a model ID that no connected provider can serve, Manifest returns [M302: Model not available](/llm-gateway/docs/errors/M302). See [Routing → Route a specific model](/llm-gateway/docs/llm-gateway#route-a-specific-model). +Send `auto` to let the gateway route, or send any listed model ID to skip routing and go straight to that provider. If you send a model ID that no connected provider can serve, the gateway returns [M302: Model not available](/llm-gateway/docs/errors/M302). See [Routing → Route a specific model](/llm-gateway/docs/llm-gateway#route-a-specific-model). ### Inspect model capabilities @@ -123,7 +123,7 @@ curl "http://localhost:2099/v1/models?capabilities=true" \ | `features` | Known feature support: `stream` and `tools` | | `supported_endpoints` | API endpoint formats the model supports. Present only when the provider publishes them in its own model list | -Capability fields are optional. A missing field means that support is unknown, not that the model does not support it. Manifest omits the entire `capabilities` object when it has no known metadata for a model. +Capability fields are optional. A missing field means that support is unknown, not that the model does not support it. The gateway omits the entire `capabilities` object when it has no known metadata for a model. The synthetic `auto` model never includes capabilities because it can resolve to a different model for each request. Concrete model IDs remain directly routable exactly as listed, including IDs with the `-subscription` suffix. @@ -175,7 +175,7 @@ curl "http://localhost:2099/v1/models?cost=true" \ | `input` | USD per million input tokens | | `output` | USD per million output tokens | -A zero value means the model has no per-token charge, as with some free or subscription-backed routes. If one price is unknown, Manifest omits only that field. If both prices are unknown, Manifest omits the entire `cost` object. +A zero value means the model has no per-token charge, as with some free or subscription-backed routes. If one price is unknown, the gateway omits only that field. If both prices are unknown, the gateway omits the entire `cost` object. The synthetic `auto` model never includes cost because its concrete model is selected for each request. To inspect both metadata types in one response, combine the query parameters: `?capabilities=true&cost=true`. @@ -204,8 +204,8 @@ Errors come in two shapes, depending on the caller. A tool or SDK call gets a re | Status | Meaning | |---|---| | `401` | Invalid or missing `Authorization` header | -| `402` | Manifest Free plan quota reached (`error.code = PLAN_LIMIT_REQUESTS`, [M204](/llm-gateway/docs/errors/M204)), or provider billing/quota error from the upstream | -| `429` | Manifest rate limit tripped ([M201](/llm-gateway/docs/errors/M201), [M202](/llm-gateway/docs/errors/M202), [M203](/llm-gateway/docs/errors/M203)), or a rate limit the provider itself returned | +| `402` | LLM Gateway Cloud Free plan quota reached (`error.code = PLAN_LIMIT_REQUESTS`, [M204](/llm-gateway/docs/errors/M204)), or provider billing/quota error from the upstream | +| `429` | gateway rate limit tripped ([M201](/llm-gateway/docs/errors/M201), [M202](/llm-gateway/docs/errors/M202), [M203](/llm-gateway/docs/errors/M203)), or a rate limit the provider itself returned | | `5xx` | Upstream provider error (triggers [fallback](/llm-gateway/docs/llm-gateway#fallback)) | A [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) block ([M200](/llm-gateway/docs/errors/M200)) is not in this table: it comes back as an HTTP `200` chat completion whose assistant message carries the block text. When the fallback chain is exhausted, the response keeps the primary model's real error status and carries `X-Manifest-Fallback-Exhausted: true`; the body keeps the provider's own error code, or `fallback_exhausted` when there is none. diff --git a/llm-gateway/docs/reference/environment-variables.mdx b/llm-gateway/docs/reference/environment-variables.mdx index 102a0f7..e220378 100644 --- a/llm-gateway/docs/reference/environment-variables.mdx +++ b/llm-gateway/docs/reference/environment-variables.mdx @@ -1,16 +1,16 @@ --- title: "Environment variables" -description: "Every environment variable Manifest reads at startup, grouped by what it controls: database, auth, rate limiting, email, OAuth, telemetry." +description: "Environment variables for a self-hosted Manifest LLM Gateway." icon: "settings" keywords: ["DATABASE_URL", "BETTER_AUTH_SECRET", "BETTER_AUTH_URL", "PORT", "BIND_ADDRESS", "THROTTLE_LIMIT", "EMAIL_PROVIDER", "Resend", "Mailgun", "SendGrid", "MANIFEST_TELEMETRY_DISABLED"] canonical: "https://manifest.build/llm-gateway/docs/reference/environment-variables" --- -Manifest reads its configuration from environment variables. In the bundled Docker setup these come from `~/manifest/.env`. For `docker run`, pass them with `-e`. For non-Docker installs, export them in the shell before launching the backend. +The gateway reads its configuration from environment variables. In the bundled Docker setup these come from `~/manifest/.env`. For `docker run`, pass them with `-e`. For non-Docker installs, export them in the shell before launching the backend. - This page is for [self-hosted](/llm-gateway/docs/self-hosted) instances only. On [Manifest + This page is for [self-hosted](/llm-gateway/docs/self-hosted) instances only. On [LLM Gateway Cloud](https://app.manifest.build) there is no server to configure and no `.env` to edit — everything you can change lives in the dashboard. @@ -24,7 +24,7 @@ Manifest reads its configuration from environment variables. In the bundled Dock | `BETTER_AUTH_URL` | No | `http://localhost:2099` | Public URL the dashboard is reachable on. Must match the browser URL | | `PORT` | No | `2099` | Dashboard port. Under the bundled compose file this sets both the published host port and the internal listener, and `BETTER_AUTH_URL` follows it | | `NODE_ENV` | No | `production` | Node environment. Telemetry is disabled when this isn't `production`. Fixed to `production` by the bundled compose file — the image is a production artifact | -| `MANIFEST_MODE` | No | — | Marks the install as self-hosted when Manifest can't tell on its own. Accepted value: `selfhosted` (`local` is a legacy alias with the same effect). Unset, Manifest detects Docker, Podman, and Kubernetes automatically; some platforms hide the container, which is why the [Render](/llm-gateway/docs/deploy/render), [Koyeb](/llm-gateway/docs/deploy/koyeb), [Easypanel](/llm-gateway/docs/deploy/easypanel), and [AWS](/llm-gateway/docs/deploy/aws) guides set it | +| `MANIFEST_MODE` | No | — | Marks the install as self-hosted when the gateway can't tell on its own. Accepted value: `selfhosted` (`local` is a legacy alias with the same effect). Unset, the gateway detects Docker, Podman, and Kubernetes automatically; some platforms hide the container, which is why the [Render](/llm-gateway/docs/deploy/render), [Koyeb](/llm-gateway/docs/deploy/koyeb), [Easypanel](/llm-gateway/docs/deploy/easypanel), and [AWS](/llm-gateway/docs/deploy/aws) guides set it | ## Network & security @@ -47,12 +47,12 @@ Manifest reads its configuration from environment variables. In the bundled Dock | `DB_POOL_MAX` | `10` | Max PostgreSQL connections in the main pool | | `AUTH_DB_POOL_MAX` | `5` | Separate pool used by Better Auth. Counted on top of `DB_POOL_MAX` when sizing your server's `max_connections` | | `RUN_MIGRATIONS_ON_BOOT` | `true` | Run pending migrations at startup. Set `false` on multi-replica deploys so only one instance migrates | -| `DB_TUNE_SESSION` | `true` | Apply Manifest's planner defaults at boot. Set `false` on managed Postgres where your role can't `ALTER ROLE` itself | +| `DB_TUNE_SESSION` | `true` | Apply the gateway's planner defaults at boot. Set `false` on managed Postgres where your role can't `ALTER ROLE` itself | | `SEED_DATA` | `false` | Seed demo data on first boot. Development only — the seeder refuses to run under `NODE_ENV=production`, so it has no effect on a Docker self-host. Use the first-run setup wizard to create your admin account | ## LLM proxy -How Manifest talks to upstream providers. +How the gateway talks to upstream providers. | Variable | Default | Description | |---|---|---| @@ -91,7 +91,7 @@ Storage and retention for [request logs](/llm-gateway/docs/request-logs). The bu | `AUTOFIX_GLOBAL_ENABLED` | `true` | Set `false` to turn Autofix off for the whole deployment. No call reaches the healing service, including the boot health check | | `AUTOFIX_TIMEOUT_MS` | `10000` | Timeout per healing call, in milliseconds | | `AUTOFIX_REPAIRABLE_STATUSES` | `400,404,422` | Which provider statuses are eligible for a repair | -| `AUTOFIX_HEALING_API_KEY` | unset | Sent as `x-api-key` to the healing service. Self-hosted installs don't need it; with no key set, the install announces its anonymous install id instead. Only relevant if Manifest issues you one | +| `AUTOFIX_HEALING_API_KEY` | unset | Sent as `x-api-key` to the healing service. Self-hosted installs don't need it; with no key set, the install announces its anonymous install id instead. Only relevant when provided by the hosted healing service | | `AUTOFIX_REPORT_ALL_4XX` | `false` | Opt-in. Also report request-side 4xx errors (except `401`/`402`/`403`/`429`) from harnesses with Autofix on, as diagnostic evidence with the secret-scrubbed request body. Bodies over 256 KB are dropped, not truncated. No fix comes back from it | ## Email @@ -127,11 +127,14 @@ Each provider activates automatically when both `*_CLIENT_ID` and `*_CLIENT_SECR | Variable | Provider | |---|---| -| `GOOGLE_CLIENT_ID` / `GOOGLE_CLIENT_SECRET` | Google | -| `GITHUB_CLIENT_ID` / `GITHUB_CLIENT_SECRET` | GitHub | -| `DISCORD_CLIENT_ID` / `DISCORD_CLIENT_SECRET` | Discord | +| `GOOGLE_CLIENT_ID` | Google OAuth client ID | +| `GOOGLE_CLIENT_SECRET` | Google OAuth client secret | +| `GITHUB_CLIENT_ID` | GitHub OAuth client ID | +| `GITHUB_CLIENT_SECRET` | GitHub OAuth client secret | +| `DISCORD_CLIENT_ID` | Discord OAuth client ID | +| `DISCORD_CLIENT_SECRET` | Discord OAuth client secret | -Subscription OAuth (ChatGPT, Claude, MiniMax) uses Manifest-side client IDs by default. Override only if you've registered your own app: +Subscription OAuth (ChatGPT, Claude, MiniMax) uses gateway-provided client IDs by default. Override only if you've registered your own app: | Variable | Description | |---|---| @@ -170,7 +173,7 @@ Sentry is opt-in and stays off unless you give it a DSN. ## Postgres bundled volume -When you run the bundled compose file, Manifest brings up its own Postgres container. To use a stronger password than the default, set **both** of these. They must agree, and special characters in the password must be percent-encoded in `DATABASE_URL`: +When you run the bundled compose file, the gateway brings up its own Postgres container. To use a stronger password than the default, set **both** of these. They must agree, and special characters in the password must be percent-encoded in `DATABASE_URL`: | Variable | Description | |---|---| diff --git a/llm-gateway/docs/reference/glossary.mdx b/llm-gateway/docs/reference/glossary.mdx index 70c673a..d2872e3 100644 --- a/llm-gateway/docs/reference/glossary.mdx +++ b/llm-gateway/docs/reference/glossary.mdx @@ -1,6 +1,6 @@ --- title: "Glossary" -description: "Definitions for harness, request, attempt, Autofix, tier, default and custom routing, direct routing, fallback chain, auto, and provider auth types." +description: "Definitions for the concepts used by the Manifest LLM Gateway." icon: "book" keywords: ["agent", "harness", "request", "attempt", "autofix", "tier", "custom tier", "default routing", "direct routing", "fallback chain", "auto", "provider", "auth type"] @@ -9,11 +9,11 @@ canonical: "https://manifest.build/llm-gateway/docs/reference/glossary" ## Harness -A configured client connection that sends requests through Manifest. Each harness has its own API key (`mnfst_...`), its own routing rules, and its own usage page. A harness typically corresponds to one tool or workflow (your IDE plugin, a Slack bot, a scheduled job), not one user. +A configured client connection that sends requests through the gateway. Each harness has its own API key (`mnfst_...`), its own routing rules, and its own usage page. A harness typically corresponds to one tool or workflow (your IDE plugin, a Slack bot, a scheduled job), not one user. ## Request -One call your client makes through Manifest. The dashboard's **Requests** page lists every request with its model, cost, and status. A single request can contain several [attempts](#attempt) when fallback or Autofix stepped in. +One call your client makes through the gateway. The dashboard's **Requests** page lists every request with its model, cost, and status. A single request can contain several [attempts](#attempt) when fallback or Autofix stepped in. ## Attempt @@ -21,11 +21,11 @@ One provider call inside a request. The initial call, each [fallback](#fallback) ## Autofix -The repair step for requests that fail because of the request itself: a parameter the provider rejects, a tool schema in the wrong shape, a model name that moved. Manifest corrects the failing request and sends it once more, before fallback. See [Autofix](/llm-gateway/docs/autofix). +The repair step for requests that fail because of the request itself: a parameter the provider rejects, a tool schema in the wrong shape, a model name that moved. The gateway corrects the failing request and sends it once more, before fallback. See [Autofix](/llm-gateway/docs/autofix). ## Auth type -The credential category Manifest uses to talk to a provider: +The credential category the gateway uses to talk to a provider: - **`api_key`** — classic per-token API key (most providers) - **`subscription`** — OAuth or a subscription token tied to a paid plan (ChatGPT Plus, Claude Max, GLM Coding Plan, and others) @@ -35,7 +35,7 @@ Auth type is recorded on every request and shows up in the dashboard's distribut ## Auto -The model ID you send to opt into routing. When Manifest sees `auto`, it applies your routing rules and picks the real model. Send a real model ID instead to skip routing and target one provider directly (see [Direct routing](#direct-routing)). +The model ID you send to opt into routing. When the gateway sees `auto`, it applies your routing rules and picks the real model. Send a real model ID instead to skip routing and target one provider directly (see [Direct routing](#direct-routing)). ## Custom tier @@ -43,19 +43,19 @@ A routing rule matched on a request header. You choose the header key and value ## Direct routing -Sending a real model ID (one that `GET /v1/models` lists) instead of `auto`. Manifest forwards the request straight to that provider and skips tiers and fallbacks. The response carries `X-Manifest-Tier: direct`. One exception: if the request also carries a matching custom-tier header, the header wins. +Sending a real model ID (one that `GET /v1/models` lists) instead of `auto`. The gateway forwards the request straight to that provider and skips tiers and fallbacks. The response carries `X-Manifest-Tier: direct`. One exception: if the request also carries a matching custom-tier header, the header wins. ## Fallback -The retry mechanism that kicks in when the primary model fails. Manifest tries the next model in the tier's fallback list, then the next, until one succeeds or the list is exhausted. See [Fallback](/llm-gateway/docs/llm-gateway#fallback) for triggers and config. +The retry mechanism that kicks in when the primary model fails. The gateway tries the next model in the tier's fallback list, then the next, until one succeeds or the list is exhausted. See [Fallback](/llm-gateway/docs/llm-gateway#fallback) for triggers and config. ## Fallback chain -The ordered list of models tried for a single tier, primary first. Up to 5 models. When all of them fail, Manifest returns the primary model's real error status, plus the header `X-Manifest-Fallback-Exhausted: true`. +The ordered list of models tried for a single tier, primary first. Up to 5 models. When all of them fail, the gateway returns the primary model's real error status, plus the header `X-Manifest-Fallback-Exhausted: true`. ## Provider -An upstream LLM service Manifest can route to. There are four kinds: +An upstream LLM service the gateway can route to. There are four kinds: - [API key](/llm-gateway/docs/providers/api-key-providers) — pay-per-token (OpenAI, Anthropic, Google, and more) - [Subscription](/llm-gateway/docs/providers/subscription-based-providers) — a plan you already pay for (ChatGPT Plus, Claude Max, and more) @@ -64,7 +64,7 @@ An upstream LLM service Manifest can route to. There are four kinds: ## Routing -The step that decides which model handles a request. Manifest checks your custom tiers first, then sends anything that doesn't match to your default model. It runs in-process with no extra network call. See [Routing](/llm-gateway/docs/llm-gateway). +The step that decides which model handles a request. The gateway checks your custom tiers first, then sends anything that doesn't match to your default model. It runs in-process with no extra network call. See [Routing](/llm-gateway/docs/llm-gateway). ## Tier diff --git a/llm-gateway/docs/reference/headers.mdx b/llm-gateway/docs/reference/headers.mdx index 563f8db..af83ca4 100644 --- a/llm-gateway/docs/reference/headers.mdx +++ b/llm-gateway/docs/reference/headers.mdx @@ -1,13 +1,13 @@ --- title: "Headers" -description: "The request headers Manifest reads and the X-Manifest-* response headers it returns, with what each one does." +description: "Request headers accepted by the Manifest LLM Gateway and the X-Manifest-* headers it returns." icon: "list" keywords: ["X-Manifest-Tier", "X-Manifest-Model", "X-Manifest-Provider", "X-Manifest-Reason", "X-Manifest-Fallback-From", "x-session-key", "custom routing header"] canonical: "https://manifest.build/llm-gateway/docs/reference/headers" --- -Manifest reads a few request headers, and returns a set of `X-Manifest-*` response headers so clients can see what happened without parsing the response body. +The gateway reads a few request headers, and returns a set of `X-Manifest-*` response headers so clients can see what happened without parsing the response body. ## Request headers @@ -15,9 +15,9 @@ Manifest reads a few request headers, and returns a set of `X-Manifest-*` respon | ----------------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | | `Authorization` | `Bearer mnfst_` | **Required.** Authenticates the harness. | | `Content-Type` | `application/json` | **Required.** | -| `anthropic-version` | `2023-06-01` | Accepted and ignored. Anthropic SDKs send it on their own. Manifest picks the version it sends upstream. | +| `anthropic-version` | `2023-06-01` | Accepted and ignored. Anthropic SDKs send it on their own. The gateway picks the version it sends upstream. | | Your custom tier header | your value | Routes the request to a [custom tier](/llm-gateway/docs/llm-gateway#custom-tiers). You set the key and value when you create the tier, so the exact header name is up to you. | -| `x-session-key` | any string | Groups requests into a session. Manifest keeps a session on the same upstream where the provider supports it, for sticky routing and prompt caching. | +| `x-session-key` | any string | Groups requests into a session. The gateway keeps a session on the same upstream where the provider supports it, for sticky routing and prompt caching. | The custom tier header is the only one that changes routing: send the header you configured on a tier and the request goes to that tier's model. The header wins even when the body names an explicit model ID. See [Routing → Custom](/llm-gateway/docs/llm-gateway#custom-tiers). @@ -36,7 +36,7 @@ Your custom tier's name appears on the request in the dashboard, never in a head ## Response headers (fallback only) -When the primary model fails and Manifest succeeds on a fallback, two extra headers are added: +When the primary model fails and the gateway succeeds on a fallback, two extra headers are added: | Header | Description | Example | | --------------------------- | --------------------------------------------------- | ------- | diff --git a/llm-gateway/docs/reference/telemetry.mdx b/llm-gateway/docs/reference/telemetry.mdx index 6d9882e..e8d441e 100644 --- a/llm-gateway/docs/reference/telemetry.mdx +++ b/llm-gateway/docs/reference/telemetry.mdx @@ -1,25 +1,25 @@ --- title: "Data and telemetry" sidebarTitle: "Data & telemetry" -description: "What Manifest stores about your requests, what's optional, and the anonymous daily report each self-hosted install sends. Includes the full field list and how to opt out." +description: "Data saved for the dashboard and anonymous telemetry reported by self-hosted LLM Gateway instances." icon: "shield" keywords: ["data privacy", "prompt storage", "message recording", "anonymous telemetry", "MANIFEST_TELEMETRY_DISABLED", "TELEMETRY_ENDPOINT", "install_id", "opt out", "self-hosted telemetry"] canonical: "https://manifest.build/llm-gateway/docs/reference/telemetry" --- -Two separate things happen to data as requests flow through Manifest: what your own instance records for the dashboard, and what a self-hosted install reports back to the project. This page covers both. +Two separate things happen to data as requests flow through the gateway: what your own instance records for the dashboard, and what a self-hosted install reports back to the project. This page covers both. -## What Manifest stores +## What the gateway stores -Manifest always keeps metadata about each request: model, provider, tier, token counts, cost, latency, and an error message. Error text is scrubbed for secrets before it lands, and the request headers Manifest stores drop credentials and IP addresses. The usage and cost views in the dashboard are built from that metadata alone. See [Observability](/llm-gateway/docs/observability) for what the record looks like in practice. +The gateway always keeps metadata about each request: model, provider, tier, token counts, cost, latency, and an error message. Error text is scrubbed for secrets before it lands, and the request headers the gateway stores drop credentials and IP addresses. The usage and cost views in the dashboard are built from that metadata alone. See [Observability](/llm-gateway/docs/observability) for what the record looks like in practice. -Message bodies are separate, and optional. With [request logs](/llm-gateway/docs/request-logs) on for a harness, Manifest also stores the request and response bodies of each provider attempt so you can read them back in the dashboard. New harnesses have it enabled; to make sure a harness records its logs, check its **Settings → Logs**. +Message bodies are separate, and optional. With [request logs](/llm-gateway/docs/request-logs) on for a harness, the gateway also stores the request and response bodies of each provider attempt so you can read them back in the dashboard. New harnesses have it enabled; to make sure a harness records its logs, check its **Settings → Logs**. -A stored body is exactly what went over the wire: every message, system instruction, tool definition and inline image, at full length. Manifest doesn't redact or truncate it, and there is no size limit. Stored bodies are held outside the database, in the object storage your instance is configured for, and deleted on a retention schedule. With logs off for a harness, that harness's prompts and completions stay between you and the provider. +A stored body is exactly what went over the wire: every message, system instruction, tool definition and inline image, at full length. The gateway doesn't redact or truncate it, and there is no size limit. Stored bodies are held outside the database, in the object storage your instance is configured for, and deleted on a retention schedule. With logs off for a harness, that harness's prompts and completions stay between you and the provider. - If a harness handles data you'd rather Manifest never hold onto, switch + If a harness handles data you'd rather the gateway never hold onto, switch **Enable logs** off in its **Settings → Logs** section. Metadata collection isn't affected either way. @@ -29,7 +29,7 @@ A stored body is exactly what went over the wire: every message, system instruct Once a day, each self-hosted install sends us a small anonymous report. That's how we know whether anyone's actually using the thing, and which providers are popular enough to deserve more work. It's aggregates, never content: no prompts, no messages, no keys, nothing tied to a user. Fifteen fields total. - This section applies to self-hosted installs only. Manifest Cloud has no + This section applies to self-hosted installs only. LLM Gateway Cloud has no separate telemetry report — your usage is already in your own dashboard. @@ -40,20 +40,20 @@ Once a day, each self-hosted install sends us a small anonymous report. That's h | `schema_version` | `1` | So the shape can grow without breaking old clients. Stays `1` for additive changes; bumps on breaking ones | | `install_id` | random UUIDv4 | Count distinct installs. Generated once on first boot, persisted, never rotated | | `manifest_version` | `5.47.0` | Version adoption across the fleet | -| `messages_total` | `1284` | Daily activity per install. One count per provider attempt, so a request that fell back twice counts three. Requests Manifest blocked count too | +| `messages_total` | `1284` | Daily activity per install. One count per provider attempt, so a request that fell back twice counts three. Requests the gateway blocked count too | | `messages_by_provider` | `{"anthropic": 700, "openai": 500}` | Provider mix. Anything we don't recognize collapses to `"custom"`, so self-hosted provider names and URLs stay local. Activity with no provider, like a blocked request, lands in `"unknown"` | | `messages_by_tier` | `{"default": 900, "simple": 300, ...}` | Routing tier usage. A tier outside the known set collapses to `"other"`, and activity with no tier lands in `"unknown"` | | `messages_by_auth_type` | `{"api_key": 1200, "subscription": 84}` | API key, paid subscription or local usage. Anything else collapses to `"other"`, and activity with no auth type lands in `"unknown"` | | `tokens_input_total` | `1_450_000` | Volume-weighted signal | | `tokens_output_total` | `890_000` | Same | -| `cost_usd_total` † | `47.83` | Sum of `cost_usd` Manifest computed at routing time, rounded to cents. Lets us see real dollar throughput instead of guessing from token counts. `0` for Ollama-only / free-API installs | +| `cost_usd_total` † | `47.83` | Sum of `cost_usd` the gateway computed at routing time, rounded to cents. Lets us see real dollar throughput instead of guessing from token counts. `0` for Ollama-only / free-API installs | | `cost_usd_by_provider` † | `{"anthropic": 30.50, "openai": 17.33}` | Per-provider split of `cost_usd_total`, rounded to cents. Same `"custom"` collapse rule as `messages_by_provider` — admin-configured BYOK pricing is never keyed by the raw provider name | | `agents_total` | `4` | Configuration scale | | `agents_by_platform` | `{"openclaw": 3, "hermes": 1}` | Which agent clients people use. A harness whose platform is `other` is keyed by its category instead: `personal:other`, `app:other` or `coding:other` | | `platform` | `linux` / `darwin` / `windows` | OS distribution | | `arch` | `x64` / `arm64` | Architecture distribution | -† *Optional. Installs running older Manifest versions omit these fields; receivers should feature-detect on presence rather than on `schema_version`. Cost values are derived from the same `input_tokens` / `output_tokens` we already ship, multiplied by Manifest's per-model pricing table — no new data leaves the box, just a rolled-up dollar figure for what's already disclosed.* +† *Optional. Installs running older gateway versions omit these fields; receivers should feature-detect on presence rather than on `schema_version`. Cost values are derived from the same `input_tokens` / `output_tokens` we already ship, multiplied by the gateway's per-model pricing table — no new data leaves the box, just a rolled-up dollar figure for what's already disclosed.* ### Never sent diff --git a/llm-gateway/docs/request-logs.mdx b/llm-gateway/docs/request-logs.mdx index 66c9441..75d78c8 100644 --- a/llm-gateway/docs/request-logs.mdx +++ b/llm-gateway/docs/request-logs.mdx @@ -1,6 +1,6 @@ --- title: "Request logs" -description: "Store the full request and response body of every provider attempt, and read them back as a conversation from the Requests page. One toggle per harness." +description: "Inspect the request received from your client and the body and response for each provider attempt." icon: "logs" keywords: ["request logs", "message recording", "request body", "response body", "LLM conversation log", "prompt logging", "full body log", "tool calls", "log retention", "S3 log storage"] @@ -12,7 +12,7 @@ canonical: "https://manifest.build/llm-gateway/docs/request-logs" -The [Requests page](/llm-gateway/docs/observability) tells you what a request cost and which model served it. Request logs tell you what was actually said: the body Manifest sent to the provider, and the body that came back. The dashboard calls this feature **Logs**. +The [Requests page](/llm-gateway/docs/observability) tells you what a request cost and which model served it. Request logs tell you what was actually said: the body the gateway sent to the provider, and the body that came back. The dashboard calls this feature **Logs**. It's one toggle per harness. @@ -22,11 +22,11 @@ Recording happens per **provider attempt**, not per request. A request that fall | | What's stored | |---|---| -| **Request** | The body Manifest forwarded to the provider, after routing rewrote the model and translated the protocol. Not the body your client sent | +| **Request** | The body the gateway forwarded to the provider, after routing rewrote the model and translated the protocol. Not the body your client sent | | **Response** | The parsed JSON body, or for a streamed attempt, the raw SSE stream as it arrived | | **Wire format** | Which protocol the exchange used, so the drawer can render it correctly | -A request Manifest blocked itself, on a [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) or a malformed body, still lists one attempt, labelled with the model you asked for or **No provider**. It carries the failure status and no **Messages**, **Tools**, or **Raw** tab: Manifest sent nothing to a provider, so there is nothing to read back. +A request blocked by the gateway, on a [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) or a malformed body, still lists one attempt, labelled with the model you asked for or **No provider**. It carries the failure status and no **Messages**, **Tools**, or **Raw** tab: the gateway sent nothing to a provider, so there is nothing to read back. ## Reading it back @@ -53,9 +53,9 @@ New harnesses have it on. To make sure a harness records its logs, check **Setti - Recording stores your prompts and completions. Everything else Manifest keeps + Recording stores your prompts and completions. Everything else the gateway keeps is metadata ([Data and telemetry](/llm-gateway/docs/reference/telemetry) covers the - distinction). If a harness handles data you'd rather Manifest never hold onto, + distinction). If a harness handles data you'd rather the gateway never hold onto, leave recording off for that harness. @@ -79,7 +79,7 @@ Self-hosted installs can override this with `REQUEST_RECORDING_RETENTION_DAYS`. Logs were reintroduced in **v6.18.0**. On an older self-hosted install, - [upgrade Manifest](/llm-gateway/docs/self-hosted#upgrading) first. + [upgrade the gateway](/llm-gateway/docs/self-hosted#upgrading) first. @@ -89,14 +89,14 @@ Recordings don't live in Postgres. They're gzipped and written to object storage The bundled compose file mounts a named volume `manifest_request_recordings` - at `/data/request-recordings` and points Manifest at it. Nothing to + at `/data/request-recordings` and points the gateway at it. Nothing to configure when your compose file includes that volume; an install from before v6.18.0 needs [one upgrade step](/llm-gateway/docs/self-hosted#upgrading) first. The volume survives `docker compose down` the same way the Postgres volume does. - Set a bucket and region, and Manifest writes there instead. Required for + Set a bucket and region, and the gateway writes there instead. Required for multi-replica deploys, since replicas can't share a local disk, and for any platform without a persistent volume. @@ -116,7 +116,7 @@ Recordings don't live in Postgres. They're gzipped and written to object storage `REQUEST_RECORDING_STORAGE` defaults to `auto`, which picks the backend by what you've configured: any S3 setting present means S3, otherwise the mounted filesystem path. Set it to `s3`, `filesystem`, or `disabled` to decide explicitly. - Under `auto`, setting *some* S3 variables commits Manifest to S3. A bucket + Under `auto`, setting *some* S3 variables commits the gateway to S3. A bucket without a region, or one access-key half without the other, leaves recording with no working backend instead of falling back to local disk. Configure it fully or not at all. @@ -132,7 +132,7 @@ A storage problem never stops traffic, and the dashboard never mentions it. Flip docker compose logs manifest | grep "Request recording storage" ``` -A line reading `Request recording storage unavailable: ` means Manifest is storing nothing, and the reason names what to fix. No such line means the backend resolved. +A line reading `Request recording storage unavailable: ` means the gateway is storing nothing, and the reason names what to fix. No such line means the backend resolved. ### Platforms without a persistent disk @@ -142,5 +142,5 @@ The one-click templates set this up for you. Render mounts a persistent disk. Ra - [Observability](/llm-gateway/docs/observability) — the request log these recordings hang off - [Autofix](/llm-gateway/docs/autofix) — comparing a failed request against its patched retry -- [Data and telemetry](/llm-gateway/docs/reference/telemetry) — what Manifest stores by default +- [Data and telemetry](/llm-gateway/docs/reference/telemetry) — what the gateway stores by default - [Environment variables](/llm-gateway/docs/reference/environment-variables#request-recordings) diff --git a/llm-gateway/docs/self-hosted.mdx b/llm-gateway/docs/self-hosted.mdx index 5033921..30a2cfc 100644 --- a/llm-gateway/docs/self-hosted.mdx +++ b/llm-gateway/docs/self-hosted.mdx @@ -1,18 +1,18 @@ --- title: "Self-hosting with Docker" sidebarTitle: "Docker" -description: "Run Manifest on your own machine with Docker. Covers the quick installer, docker-compose, bringing your own PostgreSQL, signed images, upgrades, and backups." +description: "Run the Manifest LLM Gateway on your own machine with Docker, including installation, upgrades, and backups." icon: "docker" keywords: ["self-hosted LLM router", "Docker", "docker-compose", "PostgreSQL", "BETTER_AUTH_SECRET", "docker run", "upgrade", "backup"] canonical: "https://manifest.build/llm-gateway/docs/self-hosted" --- -Run the full Manifest stack on your own machine. No Node.js required, just Docker. +Run the full gateway stack on your own machine. No Node.js required, just Docker. -To run Manifest somewhere other than your own machine, see the [other self-hosting paths](/llm-gateway/docs/deploy). +To run the gateway somewhere other than your own machine, see the [other self-hosting paths](/llm-gateway/docs/deploy). -All three paths end in the same place: a running stack at [http://localhost:2099](http://localhost:2099). On first access, Manifest takes you to a setup screen where you create the admin account. No demo credentials are pre-seeded. +All three paths end in the same place: a running stack at [http://localhost:2099](http://localhost:2099). On first access, the gateway takes you to a setup screen where you create the admin account. No demo credentials are pre-seeded. The bundled compose file binds port 2099 to `127.0.0.1` only, so the dashboard is reachable on the host machine but not over the LAN. See [Exposing on the LAN](#exposing-on-the-lan) to change this. @@ -136,7 +136,7 @@ Signing up leaves you with an empty instance. Three steps to a routed request. - **Subscriptions** — reuse a plan you already pay for (ChatGPT, Claude, GLM Coding Plan, …) - **Local** — Ollama, LM Studio, or llama.cpp running on the host - Manifest discovers the available models as soon as the connection is saved. + The gateway discovers the available models as soon as the connection is saved. Every harness has its own key, shown when you create it and again under the harness's **Settings**. It starts with `mnfst_`. @@ -151,18 +151,18 @@ Signing up leaves you with an empty instance. Three steps to a routed request. -d '{"model": "auto", "messages": [{"role": "user", "content": "Hello"}]}' ``` - `"model": "auto"` asks Manifest to route the request. Any other name is treated as an explicit choice: if the model isn't available to your harness, the request fails with [M302](/llm-gateway/docs/errors/M302). Send `auto` to use routing. + `"model": "auto"` asks the gateway to route the request. Any other name is treated as an explicit choice: if the model isn't available to your harness, the request fails with [M302](/llm-gateway/docs/errors/M302). Send `auto` to use routing. The request shows up in the dashboard straight away, with the model that served it and what it cost. - Errors raised by Manifest itself carry an `M###` code, a plain-English cause, and a link to the matching page under [manifest.build/docs/errors](/llm-gateway/docs/errors). The three you are most likely to see on a fresh install: + Errors raised by The gateway itself carry an `M###` code, a plain-English cause, and a link to the matching page under [manifest.build/docs/errors](/llm-gateway/docs/errors). The three you are most likely to see on a fresh install: | Code | Means | | --- | --- | - | `M003` | The token isn't a Manifest key — it doesn't start with `mnfst_` | + | `M003` | The token isn't a harness key — it doesn't start with `mnfst_` | | `M005` | Well-formed key, but this instance doesn't know it. Copying the literal `mnfst_YOUR_KEY_HERE` above gets you this — replace it with the real key | | `M101` | The key is fine; no provider is connected yet | | `M100` | Routing picked a provider that has no API key on file | @@ -246,7 +246,7 @@ Images are built for both `linux/amd64` and `linux/arm64`. ## Upgrading -Manifest ships a new image on every release. To upgrade an existing compose install: +The gateway ships a new image on every release. To upgrade an existing compose install: ```bash docker compose pull @@ -334,7 +334,7 @@ docker compose down -v # destroys all data The first two are required and the installer generates both, so a default install boots without you setting anything. - The installer also generates `MANIFEST_ENCRYPTION_KEY`. If you are installing by hand, set it too. Left unset, Manifest falls back to `BETTER_AUTH_SECRET` for at-rest encryption and warns on every boot — meaning one leaked session-signing secret also decrypts every stored provider key and OAuth token. Set it **before first boot**: introducing it later means re-encrypting what is already in the database. + The installer also generates `MANIFEST_ENCRYPTION_KEY`. If you are installing by hand, set it too. Left unset, the gateway falls back to `BETTER_AUTH_SECRET` for at-rest encryption and warns on every boot — meaning one leaked session-signing secret also decrypts every stored provider key and OAuth token. Set it **before first boot**: introducing it later means re-encrypting what is already in the database. Everything else is optional: provider timeouts, email delivery for alerts and password resets, OAuth logins, connection-pool sizing, recording storage and retention, Autofix, and Sentry. See [Environment variables](/llm-gateway/docs/reference/environment-variables) for the full list with defaults. @@ -354,7 +354,7 @@ docker compose down -v # Stop and delete all data ## Data and privacy -Manifest always keeps metadata about each request — model, provider, tier, token counts, cost, latency. Message bodies are separate and optional: with [request logs](/llm-gateway/docs/request-logs) on for a harness, prompts and completions are stored too, in the recordings volume rather than the database. New harnesses have it enabled. See [Data and telemetry](/llm-gateway/docs/reference/telemetry). +The gateway always keeps metadata about each request — model, provider, tier, token counts, cost, latency. Message bodies are separate and optional: with [request logs](/llm-gateway/docs/request-logs) on for a harness, prompts and completions are stored too, in the recordings volume rather than the database. New harnesses have it enabled. See [Data and telemetry](/llm-gateway/docs/reference/telemetry). ## Telemetry diff --git a/observability.mdx b/observability.mdx index f2ceac7..36e40dc 100644 --- a/observability.mdx +++ b/observability.mdx @@ -1,6 +1,6 @@ --- title: "Observability" -description: "See what every harness spent, which models served it, and what failed. Set email alerts when spending crosses a threshold." +description: "Track cost, latency, models, providers, and failures across requests and provider attempts." icon: "activity" keywords: ["LLM observability", "AI cost tracking", "token usage", "spend alerts", "request log", "LLM analytics", "cost per agent", "cost per harness", "provider errors"] @@ -16,14 +16,14 @@ Every request through the gateway is recorded: which model served it, what it co ## Requests and provider attempts -Manifest tracks two things, and the difference matters once fallback is involved: +The gateway tracks two things, and the difference matters once fallback is involved: | | What it is | When you get more than one | |---|---|---| -| **Request** | One call from your agent to Manifest | Never. One call in, one Request. | -| **Provider attempt** | One call from Manifest to a provider | Every fallback step and every [Autofix](/llm-gateway/docs/autofix) retry adds another | +| **Request** | One call from your agent to the gateway | Never. One call in, one Request. | +| **Provider attempt** | One call from the gateway to a provider | Every fallback step and every [Autofix](/llm-gateway/docs/autofix) retry adds another | -A request that succeeds on the first try is one Request with one attempt. A request that fails twice before succeeding is still one Request, with three attempts. A request Manifest blocked itself, like a [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) or a malformed body, is one Request with one attempt that never reached a provider. That attempt is listed under the model you asked for, or **No provider** when the request named none, and carries the error that stopped it. +A request that succeeds on the first try is one Request with one attempt. A request that fails twice before succeeding is still one Request, with three attempts. A request blocked by the gateway, like a [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) or a malformed body, is one Request with one attempt that never reached a provider. That attempt is listed under the model you asked for, or **No provider** when the request named none, and carries the error that stopped it. This is why the request count and the provider call count don't match, and why cost is attributed to attempts: a failed attempt can still burn tokens. @@ -52,13 +52,13 @@ On the Free plan, the Overview covers up to 7 days of history; the 30-, 90-, and ### What's on each request -Opening a request in the log shows the whole story: the tier that routed it, the model and provider that served it, input and output tokens, computed cost, latency, the auth type used ([API key, subscription, or local](/llm-gateway/docs/reference/glossary#auth-type)), and every fallback hop with its own status. Failures carry an error code, so a Manifest rejection like [M100](/llm-gateway/docs/errors/M100) reads differently from a provider's own 500. +Opening a request in the log shows the whole story: the tier that routed it, the model and provider that served it, input and output tokens, computed cost, latency, the auth type used ([API key, subscription, or local](/llm-gateway/docs/reference/glossary#auth-type)), and every fallback hop with its own status. Failures carry an error code, so a gateway rejection like [M100](/llm-gateway/docs/errors/M100) reads differently from a provider's own 500. That's all metadata. To read the messages themselves, turn on [request logs](/llm-gateway/docs/request-logs) for the harness. Open a request, pick an attempt, and its **Messages** tab shows the body sent to the provider and the body that came back. Local models record `cost = 0` with real token counts and latency, since - nothing was billed. Custom providers do the same, because Manifest can't + nothing was billed. Custom providers do the same, because the gateway can't infer pricing for a model it doesn't know. diff --git a/providers/api-key-providers.mdx b/providers/api-key-providers.mdx index 165f081..ec942ee 100644 --- a/providers/api-key-providers.mdx +++ b/providers/api-key-providers.mdx @@ -1,14 +1,14 @@ --- title: "API key providers" sidebarTitle: "API keys" -description: "Bring your own API key for OpenAI, Anthropic, Google, xAI, DeepSeek, Mistral, Groq, Cerebras, AWS Bedrock, NVIDIA NIM, OpenRouter, Hugging Face, and more." +description: "Connect provider API keys for OpenAI, Anthropic, Google, xAI, DeepSeek, Mistral, Groq, Cerebras, AWS Bedrock, NVIDIA NIM, OpenRouter, Hugging Face, and more." icon: "key" keywords: ["OpenAI", "Anthropic", "Google Gemini", "xAI", "DeepSeek", "Mistral", "Qwen", "Moonshot", "MiniMax", "Z.ai", "OpenRouter", "Groq", "Cerebras", "Fireworks", "AWS Bedrock", "NVIDIA NIM", "Xiaomi MiMo", "Hugging Face", "Gemini Free", "API key"] canonical: "https://manifest.build/llm-gateway/docs/providers/api-key-providers" --- -Most providers work the standard way: sign up, generate an API key, paste it into Manifest. Routed requests go out with that key as the credential. +Most providers work the standard way: sign up, generate an API key, paste it into the gateway. Routed requests go out with that key as the credential. ## Supported providers @@ -41,7 +41,7 @@ New providers are added often, so **Providers → Usage-based** in the dashboard ### Gemini Free -Gemini Free is a managed tile. Manifest provisions the credential through its own gateway, so there's no key to fetch from Google first: add the tile and you have a free model to route to. It behaves like any other provider once connected. +Gemini Free is a managed tile. The gateway provisions the credential automatically, so there's no key to fetch from Google first: add the tile and you have a free model to route to. It behaves like any other provider once connected. BytePlus, NousResearch, Command Code, ClinePass, Kiro, GitHub Copilot, Ollama @@ -61,7 +61,7 @@ Gemini Free is a managed tile. Manifest provisions the credential through its ow **Connect** opens a panel where you paste your API key. - Manifest validates the key by fetching the provider's model list. That list + The gateway validates the key by fetching the provider's model list. That list is what routing sees until you refresh it. If the key is rejected, double-check the prefix below. @@ -105,4 +105,4 @@ Providers not listed here (Cerebras, NVIDIA NIM, AWS Bedrock, Kilo, OpenCode Zen ## Regional providers -Some providers serve more than one region. Alibaba Cloud, AWS Bedrock, and MiniMax let you pick a region when you connect them, and Manifest routes to the matching endpoint. +Some providers serve more than one region. Alibaba Cloud, AWS Bedrock, and MiniMax let you pick a region when you connect them, and the gateway routes to the matching endpoint. diff --git a/providers/custom-providers.mdx b/providers/custom-providers.mdx index 14e2499..a514a68 100644 --- a/providers/custom-providers.mdx +++ b/providers/custom-providers.mdx @@ -1,14 +1,14 @@ --- title: "Custom providers" sidebarTitle: "Custom" -description: "Plug in any OpenAI- or Anthropic-compatible endpoint, including vLLM, TGI, LocalAI, or your own internal service." +description: "Connect any OpenAI- or Anthropic-compatible endpoint, including vLLM, TGI, LocalAI, or an internal service." icon: "plug" keywords: ["vLLM", "TGI", "text-generation-inference", "LocalAI", "Xinference", "OpenLLM", "OpenAI-compatible", "Anthropic-compatible", "custom provider"] canonical: "https://manifest.build/llm-gateway/docs/providers/custom-providers" --- -If your endpoint speaks OpenAI or Anthropic, Manifest can route to it. Useful for self-hosted inference servers (vLLM, TGI, LocalAI), internal endpoints behind your VPN, or providers that aren't on the built-in list yet. +If your endpoint speaks OpenAI or Anthropic, the gateway can route to it. Useful for self-hosted inference servers (vLLM, TGI, LocalAI), internal endpoints behind your VPN, or providers that aren't on the built-in list yet. ## Compatible servers @@ -29,7 +29,7 @@ Common options that ship with one of these formats: [vLLM](https://github.com/vl **Add custom provider**. - Paste the base URL of your endpoint, e.g. `https://my-vllm.internal:8000/v1`. Manifest normalizes the trailing `/v1` automatically. + Paste the base URL of your endpoint, e.g. `https://my-vllm.internal:8000/v1`. The gateway normalizes the trailing `/v1` automatically. Choose **OpenAI** (`/v1/chat/completions`) or **Anthropic** (`/v1/messages`), whichever your server speaks. @@ -38,13 +38,13 @@ Common options that ship with one of these formats: [vLLM](https://github.com/vl If the endpoint requires authentication, paste an API key. It's sent as `Authorization: Bearer ` for OpenAI-format endpoints, or `x-api-key: ` for Anthropic-format endpoints. - Manifest calls `GET /v1/models` against your base URL and lists every model the endpoint reports. Pin one to your default or a custom tier and you're routed. + The gateway calls `GET /v1/models` against your base URL and lists every model the endpoint reports. Pin one to your default or a custom tier and you're routed. ## Model discovery -Manifest discovers models by hitting `GET /v1/models` and reading the `data[].id` field. If your server doesn't expose `/v1/models`, you can register models manually from the same panel. +The gateway discovers models by hitting `GET /v1/models` and reading the `data[].id` field. If your server doesn't expose `/v1/models`, you can register models manually from the same panel. Older builds of `llama.cpp` (pre-b3800) don't expose `/v1/models`. Either @@ -53,14 +53,14 @@ Manifest discovers models by hitting `GET /v1/models` and reading the ## Security -User-supplied URLs are revalidated on every request to defend against SSRF. On Manifest Cloud, Manifest blocks resolution to private IP ranges (`10.x`, `192.168.x`, `127.x`, link-local): a custom provider must use a public `https://` URL. Self-hosted deployments accept private addresses and plain `http://`, so they can reach a server inside your own network. +User-supplied URLs are revalidated on every request to defend against SSRF. On LLM Gateway Cloud, the gateway blocks resolution to private IP ranges (`10.x`, `192.168.x`, `127.x`, link-local): a custom provider must use a public `https://` URL. Self-hosted deployments accept private addresses and plain `http://`, so they can reach a server inside your own network. - Custom providers run with your Manifest instance's network access. If you - expose Manifest publicly, anyone with a valid harness key can route requests + Custom providers run with your gateway instance's network access. If you + expose the gateway publicly, anyone with a valid harness key can route requests through any custom provider you've added. Gate access accordingly. ## Cost tracking -Manifest can't infer pricing for unknown models. Custom-provider requests show up in the dashboard with `cost = 0` and `model = `. Token counts and latency are still recorded, so [hard limits](/llm-gateway/docs/llm-gateway#hard-limits) on token volume still work. +The gateway can't infer pricing for unknown models. Custom-provider requests show up in the dashboard with `cost = 0` and `model = `. Token counts and latency are still recorded, so [hard limits](/llm-gateway/docs/llm-gateway#hard-limits) on token volume still work. diff --git a/providers/local-models.mdx b/providers/local-models.mdx index da4714c..4090e37 100644 --- a/providers/local-models.mdx +++ b/providers/local-models.mdx @@ -1,16 +1,16 @@ --- title: "Local models" -description: "Run any GGUF model on your own hardware with Ollama, LM Studio, or llama.cpp. No API costs, no data leaving your machine." +description: "Connect a self-hosted LLM Gateway to Ollama, LM Studio, or llama.cpp on your own hardware." icon: "server" keywords: ["Ollama", "LM Studio", "llama.cpp", "GGUF", "local LLM", "self-hosted inference", "host.docker.internal"] canonical: "https://manifest.build/llm-gateway/docs/providers/local-models" --- -Local model providers run entirely on your own hardware. Manifest detects the running server, fetches the model list, and routes requests to `http://localhost:` like any other provider. No API key, no network egress, no per-token cost. +Local model providers run entirely on your own hardware. The gateway detects the running server, fetches the model list, and routes requests to `http://localhost:` like any other provider. No API key, no network egress, no per-token cost. - The built-in Ollama, LM Studio, and llama.cpp connections are self-hosted only. To reach a local model from [Manifest Cloud](https://app.manifest.build), expose the runtime on a public URL or tunnel and add it as a [custom provider](/llm-gateway/docs/providers/custom-providers). + The built-in Ollama, LM Studio, and llama.cpp connections are self-hosted only. To reach a local model from [LLM Gateway Cloud](https://app.manifest.build), expose the runtime on a public URL or tunnel and add it as a [custom provider](/llm-gateway/docs/providers/custom-providers). ## Supported runtimes @@ -46,7 +46,7 @@ All three speak OpenAI-compatible `/v1/chat/completions` and accept any GGUF mod -## Connect to Manifest +## Connect to the gateway @@ -54,16 +54,16 @@ All three speak OpenAI-compatible `/v1/chat/completions` and accept any GGUF mod your runtime (Ollama, LM Studio, or llama.cpp). - Manifest probes `http://localhost:/v1/models`. If the probe succeeds, every loaded model appears for routing. + The gateway probes `http://localhost:/v1/models`. If the probe succeeds, every loaded model appears for routing. Open your default or a custom tier and pick a local model as the primary. You can mix local and cloud models in the same fallback chain. -## Running Manifest in Docker +## Running the gateway in Docker -If you self-host Manifest in Docker, the container can't reach a local server bound to `127.0.0.1` on the host. Two of the three runtimes default to loopback and need an explicit override: +If you self-host the gateway in Docker, the container can't reach a local server bound to `127.0.0.1` on the host. Two of the three runtimes default to loopback and need an explicit override: @@ -90,8 +90,8 @@ If you self-host Manifest in Docker, the container can't reach a local server bo - Inside the Manifest container, the host is reachable as - `host.docker.internal`. Manifest sets this automatically when probing local + Inside the gateway container, the host is reachable as + `host.docker.internal`. The gateway sets this automatically when probing local providers. diff --git a/providers/subscription-based-providers.mdx b/providers/subscription-based-providers.mdx index 4992a58..d737d79 100644 --- a/providers/subscription-based-providers.mdx +++ b/providers/subscription-based-providers.mdx @@ -1,14 +1,14 @@ --- title: "Subscription-based providers" sidebarTitle: "Subscription-based" -description: "Reuse a paid plan you already have: ChatGPT, Claude, Gemini, Grok, GitHub Copilot, GLM Coding Plan, Kimi, Mistral Vibe, and more." +description: "Use supported subscriptions such as ChatGPT, Claude, Gemini, Grok, GitHub Copilot, GLM Coding Plan, Kimi, and Mistral Vibe." icon: "credit-card" keywords: ["ChatGPT Plus", "Claude Max", "GitHub Copilot", "GLM Coding Plan", "Mistral Vibe", "Gemini", "Grok", "Kimi", "OAuth", "device code"] canonical: "https://manifest.build/llm-gateway/docs/providers/subscription-based-providers" --- -If you already pay for ChatGPT Plus, Claude Max, GitHub Copilot, or one of the other plans listed below, Manifest can route through the subscription instead of an API key. Auth is OAuth, a device code, or a pasted subscription token, depending on the provider. +If you already pay for ChatGPT Plus, Claude Max, GitHub Copilot, or one of the other plans listed below, the gateway can route through the subscription instead of an API key. Auth is OAuth, a device code, or a pasted subscription token, depending on the provider. ## Supported subscriptions @@ -48,7 +48,7 @@ If you already pay for ChatGPT Plus, Claude Max, GitHub Copilot, or one of the o **Connect** on the provider. - Manifest opens the provider's OAuth page, shows you a device code to enter, + The gateway opens the provider's OAuth page, shows you a device code to enter, or asks you to paste a token, depending on the provider. Once it's saved, you're routed. @@ -66,19 +66,19 @@ A model your plan just gained doesn't show up on its own. Click **Refresh models Generate a token with the Claude CLI (`claude setup-token`) and paste it into - Manifest. Requests carry the `anthropic-beta: oauth-2025-04-20` header so + the gateway. Requests carry the `anthropic-beta: oauth-2025-04-20` header so Anthropic recognizes the subscription token. Available models match what your plan grants in claude.ai. - GitHub's device-code flow. Manifest gives you a short user code, you visit + GitHub's device-code flow. The gateway gives you a short user code, you visit [github.com/login/device](https://github.com/login/device), paste it, and approve. No API key is ever issued. Requests then go to - `api.githubcopilot.com/chat/completions` using a short-lived token Manifest + `api.githubcopilot.com/chat/completions` using a short-lived token the gateway refreshes for you. - MiniMax's device-code flow. Manifest gives you a short user code to approve, + MiniMax's device-code flow. The gateway gives you a short user code to approve, then requests use the Anthropic protocol via `api.minimax.io/anthropic` (international) or `api.minimaxi.com/anthropic` (China). @@ -95,7 +95,7 @@ A model your plan just gained doesn't show up on its own. Click **Refresh models - Subscription auth is rate-limited by the provider's plan, not by Manifest. If + Subscription auth is rate-limited by the provider's plan, not by the gateway. If you hit a plan ceiling, requests fall through to your next [fallback](/llm-gateway/docs/llm-gateway#fallback) model, usually a different provider on an API key. @@ -104,4 +104,4 @@ A model your plan just gained doesn't show up on its own. Click **Refresh models A common setup: subscription as the primary (predictable monthly cost), API-key provider as the fallback for when you hit the plan limit or want a model the subscription doesn't include. -Pin your subscription model to your default or a custom tier and add API-key models to the [fallback list](/llm-gateway/docs/llm-gateway#configuration). Manifest handles the switch. +Pin your subscription model to your default or a custom tier and add API-key models to the [fallback list](/llm-gateway/docs/llm-gateway#configuration). The gateway handles the switch. diff --git a/reference/api.mdx b/reference/api.mdx index 2fb2a71..65124ff 100644 --- a/reference/api.mdx +++ b/reference/api.mdx @@ -1,13 +1,13 @@ --- title: "API" -description: "The Manifest proxy speaks both OpenAI and Anthropic. Endpoints, auth, streaming, and error responses." +description: "OpenAI- and Anthropic-compatible endpoints exposed by the Manifest LLM Gateway." icon: "code" keywords: ["chat completions", "responses API", "Anthropic messages", "auto", "v1/models", "streaming", "SSE", "fallback exhausted"] canonical: "https://manifest.build/llm-gateway/docs/reference/api" --- -Manifest exposes both OpenAI and Anthropic-format endpoints on one proxy. Point your client at the Manifest URL, send `auto` as the model, and routing picks the real model behind the scenes. +The gateway exposes both OpenAI and Anthropic-format endpoints on one proxy. Point your client at the gateway URL, send `auto` as the model, and routing picks the real model behind the scenes. ## Base URL @@ -18,7 +18,7 @@ Manifest exposes both OpenAI and Anthropic-format endpoints on one proxy. Point ## Authentication -Every request requires a Manifest harness key: +Every request requires a harness key: ```http Authorization: Bearer mnfst_YOUR_KEY_HERE @@ -35,9 +35,9 @@ Generate a key from the dashboard's **Harnesses** page. Keys always start with ` | `POST` | `/v1/messages` | Anthropic | Anthropic SDK, Claude Code, anything that speaks the Messages API | | `GET` | `/v1/models` | OpenAI | Listing the models your harness can route to | -The proxy translates between formats internally, so you can send an OpenAI-shaped request and Manifest will reshape it before forwarding to an Anthropic-only model. The reverse works too. +The proxy translates between formats internally, so you can send an OpenAI-shaped request and the gateway will reshape it before forwarding to an Anthropic-only model. The reverse works too. -Translation carries what the request itself contains: messages, tools, and tool results. Manifest does not resolve `previous_response_id`, so send the full conversation in `input` on every `/v1/responses` request. +Translation carries what the request itself contains: messages, tools, and tool results. The gateway does not resolve `previous_response_id`, so send the full conversation in `input` on every `/v1/responses` request. ## Chat completions @@ -53,7 +53,7 @@ curl -X POST http://localhost:2099/v1/chat/completions \ }' ``` -Manifest adapts the body to the provider it picked. It replaces `model` with the real model ID. It converts the body to the format that provider expects. It renames parameters the provider spells differently, like `max_tokens` and `max_completion_tokens`. And it merges the model parameters saved on your harness into every attempt. Some fields only exist at OpenAI: `stream_options`, `reasoning_effort`, `modalities`, `audio`, and `prediction`. Those reach OpenAI and OpenRouter. Every other provider gets the request without them, and nothing warns you. +The gateway adapts the body to the provider it picked. It replaces `model` with the real model ID. It converts the body to the format that provider expects. It renames parameters the provider spells differently, like `max_tokens` and `max_completion_tokens`. And it merges the model parameters saved on your harness into every attempt. Some fields only exist at OpenAI: `stream_options`, `reasoning_effort`, `modalities`, `audio`, and `prediction`. Those reach OpenAI and OpenRouter. Every other provider gets the request without them, and nothing warns you. ## Anthropic messages @@ -80,7 +80,7 @@ curl http://localhost:2099/v1/models \ -H "Authorization: Bearer mnfst_YOUR_KEY_HERE" ``` -Send `auto` to let Manifest route, or send any listed model ID to skip routing and go straight to that provider. If you send a model ID that no connected provider can serve, Manifest returns [M302: Model not available](/llm-gateway/docs/errors/M302). See [Routing → Route a specific model](/llm-gateway/docs/llm-gateway#route-a-specific-model). +Send `auto` to let the gateway route, or send any listed model ID to skip routing and go straight to that provider. If you send a model ID that no connected provider can serve, the gateway returns [M302: Model not available](/llm-gateway/docs/errors/M302). See [Routing → Route a specific model](/llm-gateway/docs/llm-gateway#route-a-specific-model). ### Inspect model capabilities @@ -123,7 +123,7 @@ curl "http://localhost:2099/v1/models?capabilities=true" \ | `features` | Known feature support: `stream` and `tools` | | `supported_endpoints` | API endpoint formats the model supports. Present only when the provider publishes them in its own model list | -Capability fields are optional. A missing field means that support is unknown, not that the model does not support it. Manifest omits the entire `capabilities` object when it has no known metadata for a model. +Capability fields are optional. A missing field means that support is unknown, not that the model does not support it. The gateway omits the entire `capabilities` object when it has no known metadata for a model. The synthetic `auto` model never includes capabilities because it can resolve to a different model for each request. Concrete model IDs remain directly routable exactly as listed, including IDs with the `-subscription` suffix. @@ -175,7 +175,7 @@ curl "http://localhost:2099/v1/models?cost=true" \ | `input` | USD per million input tokens | | `output` | USD per million output tokens | -A zero value means the model has no per-token charge, as with some free or subscription-backed routes. If one price is unknown, Manifest omits only that field. If both prices are unknown, Manifest omits the entire `cost` object. +A zero value means the model has no per-token charge, as with some free or subscription-backed routes. If one price is unknown, the gateway omits only that field. If both prices are unknown, the gateway omits the entire `cost` object. The synthetic `auto` model never includes cost because its concrete model is selected for each request. To inspect both metadata types in one response, combine the query parameters: `?capabilities=true&cost=true`. @@ -204,8 +204,8 @@ Errors come in two shapes, depending on the caller. A tool or SDK call gets a re | Status | Meaning | |---|---| | `401` | Invalid or missing `Authorization` header | -| `402` | Manifest Free plan quota reached (`error.code = PLAN_LIMIT_REQUESTS`, [M204](/llm-gateway/docs/errors/M204)), or provider billing/quota error from the upstream | -| `429` | Manifest rate limit tripped ([M201](/llm-gateway/docs/errors/M201), [M202](/llm-gateway/docs/errors/M202), [M203](/llm-gateway/docs/errors/M203)), or a rate limit the provider itself returned | +| `402` | LLM Gateway Cloud Free plan quota reached (`error.code = PLAN_LIMIT_REQUESTS`, [M204](/llm-gateway/docs/errors/M204)), or provider billing/quota error from the upstream | +| `429` | gateway rate limit tripped ([M201](/llm-gateway/docs/errors/M201), [M202](/llm-gateway/docs/errors/M202), [M203](/llm-gateway/docs/errors/M203)), or a rate limit the provider itself returned | | `5xx` | Upstream provider error (triggers [fallback](/llm-gateway/docs/llm-gateway#fallback)) | A [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) block ([M200](/llm-gateway/docs/errors/M200)) is not in this table: it comes back as an HTTP `200` chat completion whose assistant message carries the block text. When the fallback chain is exhausted, the response keeps the primary model's real error status and carries `X-Manifest-Fallback-Exhausted: true`; the body keeps the provider's own error code, or `fallback_exhausted` when there is none. diff --git a/reference/environment-variables.mdx b/reference/environment-variables.mdx index 102a0f7..e220378 100644 --- a/reference/environment-variables.mdx +++ b/reference/environment-variables.mdx @@ -1,16 +1,16 @@ --- title: "Environment variables" -description: "Every environment variable Manifest reads at startup, grouped by what it controls: database, auth, rate limiting, email, OAuth, telemetry." +description: "Environment variables for a self-hosted Manifest LLM Gateway." icon: "settings" keywords: ["DATABASE_URL", "BETTER_AUTH_SECRET", "BETTER_AUTH_URL", "PORT", "BIND_ADDRESS", "THROTTLE_LIMIT", "EMAIL_PROVIDER", "Resend", "Mailgun", "SendGrid", "MANIFEST_TELEMETRY_DISABLED"] canonical: "https://manifest.build/llm-gateway/docs/reference/environment-variables" --- -Manifest reads its configuration from environment variables. In the bundled Docker setup these come from `~/manifest/.env`. For `docker run`, pass them with `-e`. For non-Docker installs, export them in the shell before launching the backend. +The gateway reads its configuration from environment variables. In the bundled Docker setup these come from `~/manifest/.env`. For `docker run`, pass them with `-e`. For non-Docker installs, export them in the shell before launching the backend. - This page is for [self-hosted](/llm-gateway/docs/self-hosted) instances only. On [Manifest + This page is for [self-hosted](/llm-gateway/docs/self-hosted) instances only. On [LLM Gateway Cloud](https://app.manifest.build) there is no server to configure and no `.env` to edit — everything you can change lives in the dashboard. @@ -24,7 +24,7 @@ Manifest reads its configuration from environment variables. In the bundled Dock | `BETTER_AUTH_URL` | No | `http://localhost:2099` | Public URL the dashboard is reachable on. Must match the browser URL | | `PORT` | No | `2099` | Dashboard port. Under the bundled compose file this sets both the published host port and the internal listener, and `BETTER_AUTH_URL` follows it | | `NODE_ENV` | No | `production` | Node environment. Telemetry is disabled when this isn't `production`. Fixed to `production` by the bundled compose file — the image is a production artifact | -| `MANIFEST_MODE` | No | — | Marks the install as self-hosted when Manifest can't tell on its own. Accepted value: `selfhosted` (`local` is a legacy alias with the same effect). Unset, Manifest detects Docker, Podman, and Kubernetes automatically; some platforms hide the container, which is why the [Render](/llm-gateway/docs/deploy/render), [Koyeb](/llm-gateway/docs/deploy/koyeb), [Easypanel](/llm-gateway/docs/deploy/easypanel), and [AWS](/llm-gateway/docs/deploy/aws) guides set it | +| `MANIFEST_MODE` | No | — | Marks the install as self-hosted when the gateway can't tell on its own. Accepted value: `selfhosted` (`local` is a legacy alias with the same effect). Unset, the gateway detects Docker, Podman, and Kubernetes automatically; some platforms hide the container, which is why the [Render](/llm-gateway/docs/deploy/render), [Koyeb](/llm-gateway/docs/deploy/koyeb), [Easypanel](/llm-gateway/docs/deploy/easypanel), and [AWS](/llm-gateway/docs/deploy/aws) guides set it | ## Network & security @@ -47,12 +47,12 @@ Manifest reads its configuration from environment variables. In the bundled Dock | `DB_POOL_MAX` | `10` | Max PostgreSQL connections in the main pool | | `AUTH_DB_POOL_MAX` | `5` | Separate pool used by Better Auth. Counted on top of `DB_POOL_MAX` when sizing your server's `max_connections` | | `RUN_MIGRATIONS_ON_BOOT` | `true` | Run pending migrations at startup. Set `false` on multi-replica deploys so only one instance migrates | -| `DB_TUNE_SESSION` | `true` | Apply Manifest's planner defaults at boot. Set `false` on managed Postgres where your role can't `ALTER ROLE` itself | +| `DB_TUNE_SESSION` | `true` | Apply the gateway's planner defaults at boot. Set `false` on managed Postgres where your role can't `ALTER ROLE` itself | | `SEED_DATA` | `false` | Seed demo data on first boot. Development only — the seeder refuses to run under `NODE_ENV=production`, so it has no effect on a Docker self-host. Use the first-run setup wizard to create your admin account | ## LLM proxy -How Manifest talks to upstream providers. +How the gateway talks to upstream providers. | Variable | Default | Description | |---|---|---| @@ -91,7 +91,7 @@ Storage and retention for [request logs](/llm-gateway/docs/request-logs). The bu | `AUTOFIX_GLOBAL_ENABLED` | `true` | Set `false` to turn Autofix off for the whole deployment. No call reaches the healing service, including the boot health check | | `AUTOFIX_TIMEOUT_MS` | `10000` | Timeout per healing call, in milliseconds | | `AUTOFIX_REPAIRABLE_STATUSES` | `400,404,422` | Which provider statuses are eligible for a repair | -| `AUTOFIX_HEALING_API_KEY` | unset | Sent as `x-api-key` to the healing service. Self-hosted installs don't need it; with no key set, the install announces its anonymous install id instead. Only relevant if Manifest issues you one | +| `AUTOFIX_HEALING_API_KEY` | unset | Sent as `x-api-key` to the healing service. Self-hosted installs don't need it; with no key set, the install announces its anonymous install id instead. Only relevant when provided by the hosted healing service | | `AUTOFIX_REPORT_ALL_4XX` | `false` | Opt-in. Also report request-side 4xx errors (except `401`/`402`/`403`/`429`) from harnesses with Autofix on, as diagnostic evidence with the secret-scrubbed request body. Bodies over 256 KB are dropped, not truncated. No fix comes back from it | ## Email @@ -127,11 +127,14 @@ Each provider activates automatically when both `*_CLIENT_ID` and `*_CLIENT_SECR | Variable | Provider | |---|---| -| `GOOGLE_CLIENT_ID` / `GOOGLE_CLIENT_SECRET` | Google | -| `GITHUB_CLIENT_ID` / `GITHUB_CLIENT_SECRET` | GitHub | -| `DISCORD_CLIENT_ID` / `DISCORD_CLIENT_SECRET` | Discord | +| `GOOGLE_CLIENT_ID` | Google OAuth client ID | +| `GOOGLE_CLIENT_SECRET` | Google OAuth client secret | +| `GITHUB_CLIENT_ID` | GitHub OAuth client ID | +| `GITHUB_CLIENT_SECRET` | GitHub OAuth client secret | +| `DISCORD_CLIENT_ID` | Discord OAuth client ID | +| `DISCORD_CLIENT_SECRET` | Discord OAuth client secret | -Subscription OAuth (ChatGPT, Claude, MiniMax) uses Manifest-side client IDs by default. Override only if you've registered your own app: +Subscription OAuth (ChatGPT, Claude, MiniMax) uses gateway-provided client IDs by default. Override only if you've registered your own app: | Variable | Description | |---|---| @@ -170,7 +173,7 @@ Sentry is opt-in and stays off unless you give it a DSN. ## Postgres bundled volume -When you run the bundled compose file, Manifest brings up its own Postgres container. To use a stronger password than the default, set **both** of these. They must agree, and special characters in the password must be percent-encoded in `DATABASE_URL`: +When you run the bundled compose file, the gateway brings up its own Postgres container. To use a stronger password than the default, set **both** of these. They must agree, and special characters in the password must be percent-encoded in `DATABASE_URL`: | Variable | Description | |---|---| diff --git a/reference/glossary.mdx b/reference/glossary.mdx index 70c673a..d2872e3 100644 --- a/reference/glossary.mdx +++ b/reference/glossary.mdx @@ -1,6 +1,6 @@ --- title: "Glossary" -description: "Definitions for harness, request, attempt, Autofix, tier, default and custom routing, direct routing, fallback chain, auto, and provider auth types." +description: "Definitions for the concepts used by the Manifest LLM Gateway." icon: "book" keywords: ["agent", "harness", "request", "attempt", "autofix", "tier", "custom tier", "default routing", "direct routing", "fallback chain", "auto", "provider", "auth type"] @@ -9,11 +9,11 @@ canonical: "https://manifest.build/llm-gateway/docs/reference/glossary" ## Harness -A configured client connection that sends requests through Manifest. Each harness has its own API key (`mnfst_...`), its own routing rules, and its own usage page. A harness typically corresponds to one tool or workflow (your IDE plugin, a Slack bot, a scheduled job), not one user. +A configured client connection that sends requests through the gateway. Each harness has its own API key (`mnfst_...`), its own routing rules, and its own usage page. A harness typically corresponds to one tool or workflow (your IDE plugin, a Slack bot, a scheduled job), not one user. ## Request -One call your client makes through Manifest. The dashboard's **Requests** page lists every request with its model, cost, and status. A single request can contain several [attempts](#attempt) when fallback or Autofix stepped in. +One call your client makes through the gateway. The dashboard's **Requests** page lists every request with its model, cost, and status. A single request can contain several [attempts](#attempt) when fallback or Autofix stepped in. ## Attempt @@ -21,11 +21,11 @@ One provider call inside a request. The initial call, each [fallback](#fallback) ## Autofix -The repair step for requests that fail because of the request itself: a parameter the provider rejects, a tool schema in the wrong shape, a model name that moved. Manifest corrects the failing request and sends it once more, before fallback. See [Autofix](/llm-gateway/docs/autofix). +The repair step for requests that fail because of the request itself: a parameter the provider rejects, a tool schema in the wrong shape, a model name that moved. The gateway corrects the failing request and sends it once more, before fallback. See [Autofix](/llm-gateway/docs/autofix). ## Auth type -The credential category Manifest uses to talk to a provider: +The credential category the gateway uses to talk to a provider: - **`api_key`** — classic per-token API key (most providers) - **`subscription`** — OAuth or a subscription token tied to a paid plan (ChatGPT Plus, Claude Max, GLM Coding Plan, and others) @@ -35,7 +35,7 @@ Auth type is recorded on every request and shows up in the dashboard's distribut ## Auto -The model ID you send to opt into routing. When Manifest sees `auto`, it applies your routing rules and picks the real model. Send a real model ID instead to skip routing and target one provider directly (see [Direct routing](#direct-routing)). +The model ID you send to opt into routing. When the gateway sees `auto`, it applies your routing rules and picks the real model. Send a real model ID instead to skip routing and target one provider directly (see [Direct routing](#direct-routing)). ## Custom tier @@ -43,19 +43,19 @@ A routing rule matched on a request header. You choose the header key and value ## Direct routing -Sending a real model ID (one that `GET /v1/models` lists) instead of `auto`. Manifest forwards the request straight to that provider and skips tiers and fallbacks. The response carries `X-Manifest-Tier: direct`. One exception: if the request also carries a matching custom-tier header, the header wins. +Sending a real model ID (one that `GET /v1/models` lists) instead of `auto`. The gateway forwards the request straight to that provider and skips tiers and fallbacks. The response carries `X-Manifest-Tier: direct`. One exception: if the request also carries a matching custom-tier header, the header wins. ## Fallback -The retry mechanism that kicks in when the primary model fails. Manifest tries the next model in the tier's fallback list, then the next, until one succeeds or the list is exhausted. See [Fallback](/llm-gateway/docs/llm-gateway#fallback) for triggers and config. +The retry mechanism that kicks in when the primary model fails. The gateway tries the next model in the tier's fallback list, then the next, until one succeeds or the list is exhausted. See [Fallback](/llm-gateway/docs/llm-gateway#fallback) for triggers and config. ## Fallback chain -The ordered list of models tried for a single tier, primary first. Up to 5 models. When all of them fail, Manifest returns the primary model's real error status, plus the header `X-Manifest-Fallback-Exhausted: true`. +The ordered list of models tried for a single tier, primary first. Up to 5 models. When all of them fail, the gateway returns the primary model's real error status, plus the header `X-Manifest-Fallback-Exhausted: true`. ## Provider -An upstream LLM service Manifest can route to. There are four kinds: +An upstream LLM service the gateway can route to. There are four kinds: - [API key](/llm-gateway/docs/providers/api-key-providers) — pay-per-token (OpenAI, Anthropic, Google, and more) - [Subscription](/llm-gateway/docs/providers/subscription-based-providers) — a plan you already pay for (ChatGPT Plus, Claude Max, and more) @@ -64,7 +64,7 @@ An upstream LLM service Manifest can route to. There are four kinds: ## Routing -The step that decides which model handles a request. Manifest checks your custom tiers first, then sends anything that doesn't match to your default model. It runs in-process with no extra network call. See [Routing](/llm-gateway/docs/llm-gateway). +The step that decides which model handles a request. The gateway checks your custom tiers first, then sends anything that doesn't match to your default model. It runs in-process with no extra network call. See [Routing](/llm-gateway/docs/llm-gateway). ## Tier diff --git a/reference/headers.mdx b/reference/headers.mdx index 563f8db..af83ca4 100644 --- a/reference/headers.mdx +++ b/reference/headers.mdx @@ -1,13 +1,13 @@ --- title: "Headers" -description: "The request headers Manifest reads and the X-Manifest-* response headers it returns, with what each one does." +description: "Request headers accepted by the Manifest LLM Gateway and the X-Manifest-* headers it returns." icon: "list" keywords: ["X-Manifest-Tier", "X-Manifest-Model", "X-Manifest-Provider", "X-Manifest-Reason", "X-Manifest-Fallback-From", "x-session-key", "custom routing header"] canonical: "https://manifest.build/llm-gateway/docs/reference/headers" --- -Manifest reads a few request headers, and returns a set of `X-Manifest-*` response headers so clients can see what happened without parsing the response body. +The gateway reads a few request headers, and returns a set of `X-Manifest-*` response headers so clients can see what happened without parsing the response body. ## Request headers @@ -15,9 +15,9 @@ Manifest reads a few request headers, and returns a set of `X-Manifest-*` respon | ----------------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | | `Authorization` | `Bearer mnfst_` | **Required.** Authenticates the harness. | | `Content-Type` | `application/json` | **Required.** | -| `anthropic-version` | `2023-06-01` | Accepted and ignored. Anthropic SDKs send it on their own. Manifest picks the version it sends upstream. | +| `anthropic-version` | `2023-06-01` | Accepted and ignored. Anthropic SDKs send it on their own. The gateway picks the version it sends upstream. | | Your custom tier header | your value | Routes the request to a [custom tier](/llm-gateway/docs/llm-gateway#custom-tiers). You set the key and value when you create the tier, so the exact header name is up to you. | -| `x-session-key` | any string | Groups requests into a session. Manifest keeps a session on the same upstream where the provider supports it, for sticky routing and prompt caching. | +| `x-session-key` | any string | Groups requests into a session. The gateway keeps a session on the same upstream where the provider supports it, for sticky routing and prompt caching. | The custom tier header is the only one that changes routing: send the header you configured on a tier and the request goes to that tier's model. The header wins even when the body names an explicit model ID. See [Routing → Custom](/llm-gateway/docs/llm-gateway#custom-tiers). @@ -36,7 +36,7 @@ Your custom tier's name appears on the request in the dashboard, never in a head ## Response headers (fallback only) -When the primary model fails and Manifest succeeds on a fallback, two extra headers are added: +When the primary model fails and the gateway succeeds on a fallback, two extra headers are added: | Header | Description | Example | | --------------------------- | --------------------------------------------------- | ------- | diff --git a/reference/telemetry.mdx b/reference/telemetry.mdx index 6d9882e..e8d441e 100644 --- a/reference/telemetry.mdx +++ b/reference/telemetry.mdx @@ -1,25 +1,25 @@ --- title: "Data and telemetry" sidebarTitle: "Data & telemetry" -description: "What Manifest stores about your requests, what's optional, and the anonymous daily report each self-hosted install sends. Includes the full field list and how to opt out." +description: "Data saved for the dashboard and anonymous telemetry reported by self-hosted LLM Gateway instances." icon: "shield" keywords: ["data privacy", "prompt storage", "message recording", "anonymous telemetry", "MANIFEST_TELEMETRY_DISABLED", "TELEMETRY_ENDPOINT", "install_id", "opt out", "self-hosted telemetry"] canonical: "https://manifest.build/llm-gateway/docs/reference/telemetry" --- -Two separate things happen to data as requests flow through Manifest: what your own instance records for the dashboard, and what a self-hosted install reports back to the project. This page covers both. +Two separate things happen to data as requests flow through the gateway: what your own instance records for the dashboard, and what a self-hosted install reports back to the project. This page covers both. -## What Manifest stores +## What the gateway stores -Manifest always keeps metadata about each request: model, provider, tier, token counts, cost, latency, and an error message. Error text is scrubbed for secrets before it lands, and the request headers Manifest stores drop credentials and IP addresses. The usage and cost views in the dashboard are built from that metadata alone. See [Observability](/llm-gateway/docs/observability) for what the record looks like in practice. +The gateway always keeps metadata about each request: model, provider, tier, token counts, cost, latency, and an error message. Error text is scrubbed for secrets before it lands, and the request headers the gateway stores drop credentials and IP addresses. The usage and cost views in the dashboard are built from that metadata alone. See [Observability](/llm-gateway/docs/observability) for what the record looks like in practice. -Message bodies are separate, and optional. With [request logs](/llm-gateway/docs/request-logs) on for a harness, Manifest also stores the request and response bodies of each provider attempt so you can read them back in the dashboard. New harnesses have it enabled; to make sure a harness records its logs, check its **Settings → Logs**. +Message bodies are separate, and optional. With [request logs](/llm-gateway/docs/request-logs) on for a harness, the gateway also stores the request and response bodies of each provider attempt so you can read them back in the dashboard. New harnesses have it enabled; to make sure a harness records its logs, check its **Settings → Logs**. -A stored body is exactly what went over the wire: every message, system instruction, tool definition and inline image, at full length. Manifest doesn't redact or truncate it, and there is no size limit. Stored bodies are held outside the database, in the object storage your instance is configured for, and deleted on a retention schedule. With logs off for a harness, that harness's prompts and completions stay between you and the provider. +A stored body is exactly what went over the wire: every message, system instruction, tool definition and inline image, at full length. The gateway doesn't redact or truncate it, and there is no size limit. Stored bodies are held outside the database, in the object storage your instance is configured for, and deleted on a retention schedule. With logs off for a harness, that harness's prompts and completions stay between you and the provider. - If a harness handles data you'd rather Manifest never hold onto, switch + If a harness handles data you'd rather the gateway never hold onto, switch **Enable logs** off in its **Settings → Logs** section. Metadata collection isn't affected either way. @@ -29,7 +29,7 @@ A stored body is exactly what went over the wire: every message, system instruct Once a day, each self-hosted install sends us a small anonymous report. That's how we know whether anyone's actually using the thing, and which providers are popular enough to deserve more work. It's aggregates, never content: no prompts, no messages, no keys, nothing tied to a user. Fifteen fields total. - This section applies to self-hosted installs only. Manifest Cloud has no + This section applies to self-hosted installs only. LLM Gateway Cloud has no separate telemetry report — your usage is already in your own dashboard. @@ -40,20 +40,20 @@ Once a day, each self-hosted install sends us a small anonymous report. That's h | `schema_version` | `1` | So the shape can grow without breaking old clients. Stays `1` for additive changes; bumps on breaking ones | | `install_id` | random UUIDv4 | Count distinct installs. Generated once on first boot, persisted, never rotated | | `manifest_version` | `5.47.0` | Version adoption across the fleet | -| `messages_total` | `1284` | Daily activity per install. One count per provider attempt, so a request that fell back twice counts three. Requests Manifest blocked count too | +| `messages_total` | `1284` | Daily activity per install. One count per provider attempt, so a request that fell back twice counts three. Requests the gateway blocked count too | | `messages_by_provider` | `{"anthropic": 700, "openai": 500}` | Provider mix. Anything we don't recognize collapses to `"custom"`, so self-hosted provider names and URLs stay local. Activity with no provider, like a blocked request, lands in `"unknown"` | | `messages_by_tier` | `{"default": 900, "simple": 300, ...}` | Routing tier usage. A tier outside the known set collapses to `"other"`, and activity with no tier lands in `"unknown"` | | `messages_by_auth_type` | `{"api_key": 1200, "subscription": 84}` | API key, paid subscription or local usage. Anything else collapses to `"other"`, and activity with no auth type lands in `"unknown"` | | `tokens_input_total` | `1_450_000` | Volume-weighted signal | | `tokens_output_total` | `890_000` | Same | -| `cost_usd_total` † | `47.83` | Sum of `cost_usd` Manifest computed at routing time, rounded to cents. Lets us see real dollar throughput instead of guessing from token counts. `0` for Ollama-only / free-API installs | +| `cost_usd_total` † | `47.83` | Sum of `cost_usd` the gateway computed at routing time, rounded to cents. Lets us see real dollar throughput instead of guessing from token counts. `0` for Ollama-only / free-API installs | | `cost_usd_by_provider` † | `{"anthropic": 30.50, "openai": 17.33}` | Per-provider split of `cost_usd_total`, rounded to cents. Same `"custom"` collapse rule as `messages_by_provider` — admin-configured BYOK pricing is never keyed by the raw provider name | | `agents_total` | `4` | Configuration scale | | `agents_by_platform` | `{"openclaw": 3, "hermes": 1}` | Which agent clients people use. A harness whose platform is `other` is keyed by its category instead: `personal:other`, `app:other` or `coding:other` | | `platform` | `linux` / `darwin` / `windows` | OS distribution | | `arch` | `x64` / `arm64` | Architecture distribution | -† *Optional. Installs running older Manifest versions omit these fields; receivers should feature-detect on presence rather than on `schema_version`. Cost values are derived from the same `input_tokens` / `output_tokens` we already ship, multiplied by Manifest's per-model pricing table — no new data leaves the box, just a rolled-up dollar figure for what's already disclosed.* +† *Optional. Installs running older gateway versions omit these fields; receivers should feature-detect on presence rather than on `schema_version`. Cost values are derived from the same `input_tokens` / `output_tokens` we already ship, multiplied by the gateway's per-model pricing table — no new data leaves the box, just a rolled-up dollar figure for what's already disclosed.* ### Never sent diff --git a/request-logs.mdx b/request-logs.mdx index 66c9441..75d78c8 100644 --- a/request-logs.mdx +++ b/request-logs.mdx @@ -1,6 +1,6 @@ --- title: "Request logs" -description: "Store the full request and response body of every provider attempt, and read them back as a conversation from the Requests page. One toggle per harness." +description: "Inspect the request received from your client and the body and response for each provider attempt." icon: "logs" keywords: ["request logs", "message recording", "request body", "response body", "LLM conversation log", "prompt logging", "full body log", "tool calls", "log retention", "S3 log storage"] @@ -12,7 +12,7 @@ canonical: "https://manifest.build/llm-gateway/docs/request-logs" -The [Requests page](/llm-gateway/docs/observability) tells you what a request cost and which model served it. Request logs tell you what was actually said: the body Manifest sent to the provider, and the body that came back. The dashboard calls this feature **Logs**. +The [Requests page](/llm-gateway/docs/observability) tells you what a request cost and which model served it. Request logs tell you what was actually said: the body the gateway sent to the provider, and the body that came back. The dashboard calls this feature **Logs**. It's one toggle per harness. @@ -22,11 +22,11 @@ Recording happens per **provider attempt**, not per request. A request that fall | | What's stored | |---|---| -| **Request** | The body Manifest forwarded to the provider, after routing rewrote the model and translated the protocol. Not the body your client sent | +| **Request** | The body the gateway forwarded to the provider, after routing rewrote the model and translated the protocol. Not the body your client sent | | **Response** | The parsed JSON body, or for a streamed attempt, the raw SSE stream as it arrived | | **Wire format** | Which protocol the exchange used, so the drawer can render it correctly | -A request Manifest blocked itself, on a [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) or a malformed body, still lists one attempt, labelled with the model you asked for or **No provider**. It carries the failure status and no **Messages**, **Tools**, or **Raw** tab: Manifest sent nothing to a provider, so there is nothing to read back. +A request blocked by the gateway, on a [hard limit](/llm-gateway/docs/llm-gateway#hard-limits) or a malformed body, still lists one attempt, labelled with the model you asked for or **No provider**. It carries the failure status and no **Messages**, **Tools**, or **Raw** tab: the gateway sent nothing to a provider, so there is nothing to read back. ## Reading it back @@ -53,9 +53,9 @@ New harnesses have it on. To make sure a harness records its logs, check **Setti - Recording stores your prompts and completions. Everything else Manifest keeps + Recording stores your prompts and completions. Everything else the gateway keeps is metadata ([Data and telemetry](/llm-gateway/docs/reference/telemetry) covers the - distinction). If a harness handles data you'd rather Manifest never hold onto, + distinction). If a harness handles data you'd rather the gateway never hold onto, leave recording off for that harness. @@ -79,7 +79,7 @@ Self-hosted installs can override this with `REQUEST_RECORDING_RETENTION_DAYS`. Logs were reintroduced in **v6.18.0**. On an older self-hosted install, - [upgrade Manifest](/llm-gateway/docs/self-hosted#upgrading) first. + [upgrade the gateway](/llm-gateway/docs/self-hosted#upgrading) first. @@ -89,14 +89,14 @@ Recordings don't live in Postgres. They're gzipped and written to object storage The bundled compose file mounts a named volume `manifest_request_recordings` - at `/data/request-recordings` and points Manifest at it. Nothing to + at `/data/request-recordings` and points the gateway at it. Nothing to configure when your compose file includes that volume; an install from before v6.18.0 needs [one upgrade step](/llm-gateway/docs/self-hosted#upgrading) first. The volume survives `docker compose down` the same way the Postgres volume does. - Set a bucket and region, and Manifest writes there instead. Required for + Set a bucket and region, and the gateway writes there instead. Required for multi-replica deploys, since replicas can't share a local disk, and for any platform without a persistent volume. @@ -116,7 +116,7 @@ Recordings don't live in Postgres. They're gzipped and written to object storage `REQUEST_RECORDING_STORAGE` defaults to `auto`, which picks the backend by what you've configured: any S3 setting present means S3, otherwise the mounted filesystem path. Set it to `s3`, `filesystem`, or `disabled` to decide explicitly. - Under `auto`, setting *some* S3 variables commits Manifest to S3. A bucket + Under `auto`, setting *some* S3 variables commits the gateway to S3. A bucket without a region, or one access-key half without the other, leaves recording with no working backend instead of falling back to local disk. Configure it fully or not at all. @@ -132,7 +132,7 @@ A storage problem never stops traffic, and the dashboard never mentions it. Flip docker compose logs manifest | grep "Request recording storage" ``` -A line reading `Request recording storage unavailable: ` means Manifest is storing nothing, and the reason names what to fix. No such line means the backend resolved. +A line reading `Request recording storage unavailable: ` means the gateway is storing nothing, and the reason names what to fix. No such line means the backend resolved. ### Platforms without a persistent disk @@ -142,5 +142,5 @@ The one-click templates set this up for you. Render mounts a persistent disk. Ra - [Observability](/llm-gateway/docs/observability) — the request log these recordings hang off - [Autofix](/llm-gateway/docs/autofix) — comparing a failed request against its patched retry -- [Data and telemetry](/llm-gateway/docs/reference/telemetry) — what Manifest stores by default +- [Data and telemetry](/llm-gateway/docs/reference/telemetry) — what the gateway stores by default - [Environment variables](/llm-gateway/docs/reference/environment-variables#request-recordings) diff --git a/self-hosted.mdx b/self-hosted.mdx index 5033921..30a2cfc 100644 --- a/self-hosted.mdx +++ b/self-hosted.mdx @@ -1,18 +1,18 @@ --- title: "Self-hosting with Docker" sidebarTitle: "Docker" -description: "Run Manifest on your own machine with Docker. Covers the quick installer, docker-compose, bringing your own PostgreSQL, signed images, upgrades, and backups." +description: "Run the Manifest LLM Gateway on your own machine with Docker, including installation, upgrades, and backups." icon: "docker" keywords: ["self-hosted LLM router", "Docker", "docker-compose", "PostgreSQL", "BETTER_AUTH_SECRET", "docker run", "upgrade", "backup"] canonical: "https://manifest.build/llm-gateway/docs/self-hosted" --- -Run the full Manifest stack on your own machine. No Node.js required, just Docker. +Run the full gateway stack on your own machine. No Node.js required, just Docker. -To run Manifest somewhere other than your own machine, see the [other self-hosting paths](/llm-gateway/docs/deploy). +To run the gateway somewhere other than your own machine, see the [other self-hosting paths](/llm-gateway/docs/deploy). -All three paths end in the same place: a running stack at [http://localhost:2099](http://localhost:2099). On first access, Manifest takes you to a setup screen where you create the admin account. No demo credentials are pre-seeded. +All three paths end in the same place: a running stack at [http://localhost:2099](http://localhost:2099). On first access, the gateway takes you to a setup screen where you create the admin account. No demo credentials are pre-seeded. The bundled compose file binds port 2099 to `127.0.0.1` only, so the dashboard is reachable on the host machine but not over the LAN. See [Exposing on the LAN](#exposing-on-the-lan) to change this. @@ -136,7 +136,7 @@ Signing up leaves you with an empty instance. Three steps to a routed request. - **Subscriptions** — reuse a plan you already pay for (ChatGPT, Claude, GLM Coding Plan, …) - **Local** — Ollama, LM Studio, or llama.cpp running on the host - Manifest discovers the available models as soon as the connection is saved. + The gateway discovers the available models as soon as the connection is saved. Every harness has its own key, shown when you create it and again under the harness's **Settings**. It starts with `mnfst_`. @@ -151,18 +151,18 @@ Signing up leaves you with an empty instance. Three steps to a routed request. -d '{"model": "auto", "messages": [{"role": "user", "content": "Hello"}]}' ``` - `"model": "auto"` asks Manifest to route the request. Any other name is treated as an explicit choice: if the model isn't available to your harness, the request fails with [M302](/llm-gateway/docs/errors/M302). Send `auto` to use routing. + `"model": "auto"` asks the gateway to route the request. Any other name is treated as an explicit choice: if the model isn't available to your harness, the request fails with [M302](/llm-gateway/docs/errors/M302). Send `auto` to use routing. The request shows up in the dashboard straight away, with the model that served it and what it cost. - Errors raised by Manifest itself carry an `M###` code, a plain-English cause, and a link to the matching page under [manifest.build/docs/errors](/llm-gateway/docs/errors). The three you are most likely to see on a fresh install: + Errors raised by The gateway itself carry an `M###` code, a plain-English cause, and a link to the matching page under [manifest.build/docs/errors](/llm-gateway/docs/errors). The three you are most likely to see on a fresh install: | Code | Means | | --- | --- | - | `M003` | The token isn't a Manifest key — it doesn't start with `mnfst_` | + | `M003` | The token isn't a harness key — it doesn't start with `mnfst_` | | `M005` | Well-formed key, but this instance doesn't know it. Copying the literal `mnfst_YOUR_KEY_HERE` above gets you this — replace it with the real key | | `M101` | The key is fine; no provider is connected yet | | `M100` | Routing picked a provider that has no API key on file | @@ -246,7 +246,7 @@ Images are built for both `linux/amd64` and `linux/arm64`. ## Upgrading -Manifest ships a new image on every release. To upgrade an existing compose install: +The gateway ships a new image on every release. To upgrade an existing compose install: ```bash docker compose pull @@ -334,7 +334,7 @@ docker compose down -v # destroys all data The first two are required and the installer generates both, so a default install boots without you setting anything. - The installer also generates `MANIFEST_ENCRYPTION_KEY`. If you are installing by hand, set it too. Left unset, Manifest falls back to `BETTER_AUTH_SECRET` for at-rest encryption and warns on every boot — meaning one leaked session-signing secret also decrypts every stored provider key and OAuth token. Set it **before first boot**: introducing it later means re-encrypting what is already in the database. + The installer also generates `MANIFEST_ENCRYPTION_KEY`. If you are installing by hand, set it too. Left unset, the gateway falls back to `BETTER_AUTH_SECRET` for at-rest encryption and warns on every boot — meaning one leaked session-signing secret also decrypts every stored provider key and OAuth token. Set it **before first boot**: introducing it later means re-encrypting what is already in the database. Everything else is optional: provider timeouts, email delivery for alerts and password resets, OAuth logins, connection-pool sizing, recording storage and retention, Autofix, and Sentry. See [Environment variables](/llm-gateway/docs/reference/environment-variables) for the full list with defaults. @@ -354,7 +354,7 @@ docker compose down -v # Stop and delete all data ## Data and privacy -Manifest always keeps metadata about each request — model, provider, tier, token counts, cost, latency. Message bodies are separate and optional: with [request logs](/llm-gateway/docs/request-logs) on for a harness, prompts and completions are stored too, in the recordings volume rather than the database. New harnesses have it enabled. See [Data and telemetry](/llm-gateway/docs/reference/telemetry). +The gateway always keeps metadata about each request — model, provider, tier, token counts, cost, latency. Message bodies are separate and optional: with [request logs](/llm-gateway/docs/request-logs) on for a harness, prompts and completions are stored too, in the recordings volume rather than the database. New harnesses have it enabled. See [Data and telemetry](/llm-gateway/docs/reference/telemetry). ## Telemetry From 543b02efa87f3b962c33b8b6c3ce5f1e71190fbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Conejo?= Date: Fri, 11 Sep 2026 10:49:34 +0200 Subject: [PATCH 3/3] Add Self-Healing docs header link --- docs.json | 2 +- scripts/no-tab-jump.js | 31 +++++++++++++++++++++++++ style.css | 51 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/docs.json b/docs.json index 6a38707..d606c87 100644 --- a/docs.json +++ b/docs.json @@ -11,7 +11,7 @@ "css": "/style.css", "js": "/scripts/no-tab-jump.js", "banner": { - "content": "You are reading the **Manifest LLM Gateway documentation**. Looking for Manifest, the self-healing API? [Read the Manifest documentation](https://manifest.build/docs).", + "content": "You are reading the **Manifest LLM Gateway documentation**. Looking for the self-healing API documentation? [Read the docs](https://manifest.build/docs).", "dismissible": false }, "colors": { diff --git a/scripts/no-tab-jump.js b/scripts/no-tab-jump.js index 18a39b7..f5c8e5d 100644 --- a/scripts/no-tab-jump.js +++ b/scripts/no-tab-jump.js @@ -17,3 +17,34 @@ if (window.location.hash && document.querySelector('[role="tab"]')) { history.replaceState(null, '', window.location.pathname); window.scrollTo(0, scrollY); } + +// Keep the primary Manifest product visible from the Gateway documentation. +// Mintlify re-renders the navbar during client-side navigation, so create the +// link outside the theme-owned tree and restore it if that tree changes. +(function () { + function ensureSelfHealingLink() { + if (document.querySelector('.docs-self-healing')) return; + + var link = document.createElement('a'); + link.className = 'docs-self-healing'; + link.href = 'https://manifest.build/docs'; + link.setAttribute('aria-label', 'Open the Manifest Self-Healing documentation'); + link.innerHTML = + 'Manifest Self-Healing'; + document.body.appendChild(link); + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', ensureSelfHealingLink); + } else { + ensureSelfHealingLink(); + } + + new MutationObserver(ensureSelfHealingLink).observe(document.body, { + childList: true, + subtree: true, + }); +})(); diff --git a/style.css b/style.css index 247165e..96271dc 100644 --- a/style.css +++ b/style.css @@ -491,6 +491,46 @@ html.dark .docs-book-call { color: #0f172a !important; } +/* Secondary CTA to the main Manifest Self-Healing documentation */ +.docs-self-healing { + position: fixed; + top: calc(7.5px + var(--banner-height, 40px)); + right: 166px; + z-index: 60; + display: inline-flex; + align-items: center; + gap: 7px; + padding: 8px 13px; + border: 1px solid #d8d3ca; + border-radius: 10px; + background: #f8f6f1; + color: #475569 !important; + font-size: 14px; + font-weight: 600; + line-height: 1; + text-decoration: none !important; + box-shadow: 0 1px 2px rgba(15, 23, 42, 0.04); +} +.docs-self-healing svg { + width: 18px; + height: 18px; + flex: 0 0 auto; +} +.docs-self-healing:hover { + border-color: #b8b1a6; + color: #0f172a !important; +} +html.dark .docs-self-healing { + border-color: rgba(255, 255, 255, 0.16); + background: #252525; + color: #d1d5db !important; + box-shadow: none; +} +html.dark .docs-self-healing:hover { + border-color: rgba(255, 255, 255, 0.3); + color: #ffffff !important; +} + /* Below lg the banner scrolls with the page instead of staying fixed, so the fixed elements keep their original offsets */ @media (max-width: 1023px) { @@ -500,6 +540,17 @@ html.dark .docs-book-call { .docs-book-call { top: 7.5px; } + .docs-self-healing { + top: 7.5px; + } +} + +/* Keep the compact header usable on tablets and phones. The Self-Healing + destination remains available from the announcement banner. */ +@media (max-width: 767px) { + .docs-self-healing { + display: none; + } } /* Paragraph spans nested in deploy-mode wrappers miss the theme's prose