{"openapi":"3.1.0","info":{"title":"Langfuse evaluators, rules and annotation queues (your keys) API","version":"0.1.0","description":"Langfuse evaluators, rules and annotation queues (your keys) lets an AI assistant read your own Langfuse Evaluators data through the Langfuse Evaluators API with a key you supply (Langfuse public key (starts with pk-lf-)). Read-only: 10 tools (list_evaluators, get_evaluator, list_evaluator_versions, list_evaluation_rules, get_evaluation_rule, list_annotation_queues, get_annotation_queue,...\n\nEvery operation is read-only or compute-only. The same tools are available over MCP (streamable HTTP) at https://byok-langfuse-evaluators.connectorbay.com/mcp.","contact":{"name":"Unify Dynamics LLC","email":"agent@connectorbay.com","url":"https://connectorbay.com/byok-langfuse-evaluators/docs"},"termsOfService":"https://connectorbay.com/byok-langfuse-evaluators/terms"},"servers":[{"url":"https://byok-langfuse-evaluators.connectorbay.com"}],"externalDocs":{"url":"https://connectorbay.com/byok-langfuse-evaluators/docs"},"security":[{"upstreamKey":[]}],"components":{"securitySchemes":{"bearer":{"type":"http","scheme":"bearer","description":"Optional. A Pro API key raises rate limits; no credentials are required."},"upstreamKey":{"type":"apiKey","in":"header","name":"X-Upstream-Key","description":"Your own key for the service this connector wraps (bring your own key). Used for that one call only; never stored, logged or cached."}},"schemas":{"Error":{"type":"object","properties":{"ok":{"const":false},"error":{"type":"object"}}}}},"paths":{"/v1/tools/list_evaluators":{"post":{"operationId":"list_evaluators","summary":"List evaluators (LLM-as-judge and code) with name, type, status, current version and dates. The evaluator prompt and sou","description":"List evaluators (LLM-as-judge and code) with name, type, status, current version and dates. The evaluator prompt and source code are not returned. Use for \"Which evaluators are set up in my Langfuse project\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":20,"description":"How many items per page (1 to 100).","title":"Limit"},"cursor":{"anyOf":[{"maxLength":2000,"type":"string"},{"type":"null"}],"default":null,"description":"Cursor from the previous answer's meta.cursor for the next page.","title":"Cursor"}},"title":"list_evaluatorsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/get_evaluator":{"post":{"operationId":"get_evaluator","summary":"Get one evaluator by id: name, type, status, pause reason, current version and dates. The evaluator prompt and source co","description":"Get one evaluator by id: name, type, status, pause reason, current version and dates. The evaluator prompt and source code are not returned. Use for \"Show Langfuse evaluator ev_abc123\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"evaluator_id":{"description":"Evaluator id as returned by list_evaluators.","maxLength":100,"title":"Evaluator Id","type":"string"}},"required":["evaluator_id"],"title":"get_evaluatorArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_evaluator_versions":{"post":{"operationId":"list_evaluator_versions","summary":"List the versions of one evaluator, newest first, with version number and creation time. The evaluator prompt and source","description":"List the versions of one evaluator, newest first, with version number and creation time. The evaluator prompt and source code are not returned. Use for \"How many versions does Langfuse evaluator ev_abc123 have\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"evaluator_id":{"description":"Evaluator id as returned by list_evaluators.","maxLength":100,"title":"Evaluator Id","type":"string"},"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":20,"description":"How many items per page (1 to 100).","title":"Limit"},"cursor":{"anyOf":[{"maxLength":2000,"type":"string"},{"type":"null"}],"default":null,"description":"Cursor from the previous answer's meta.cursor for the next page.","title":"Cursor"}},"required":["evaluator_id"],"title":"list_evaluator_versionsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_evaluation_rules":{"post":{"operationId":"list_evaluation_rules","summary":"List evaluation rules (which live data gets scored) with enabled flag, sampling rate and evaluator assignments. Use for ","description":"List evaluation rules (which live data gets scored) with enabled flag, sampling rate and evaluator assignments. Use for \"Which Langfuse evaluation rules are enabled and at what sampling rate\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":20,"description":"How many items per page (1 to 100).","title":"Limit"},"cursor":{"anyOf":[{"maxLength":2000,"type":"string"},{"type":"null"}],"default":null,"description":"Cursor from the previous answer's meta.cursor for the next page.","title":"Cursor"}},"title":"list_evaluation_rulesArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/get_evaluation_rule":{"post":{"operationId":"get_evaluation_rule","summary":"Get one evaluation rule by id with enabled flag, sampling rate, filters and evaluator assignments. Use for \"Show Langfus","description":"Get one evaluation rule by id with enabled flag, sampling rate, filters and evaluator assignments. Use for \"Show Langfuse evaluation rule rule_1\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"evaluation_rule_id":{"description":"Evaluation rule id as returned by list_evaluation_rules.","maxLength":100,"title":"Evaluation Rule Id","type":"string"}},"required":["evaluation_rule_id"],"title":"get_evaluation_ruleArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_annotation_queues":{"post":{"operationId":"list_annotation_queues","summary":"List annotation queues (human review queues) with name, description and the score configs used. Use for \"What annotation","description":"List annotation queues (human review queues) with name, description and the score configs used. Use for \"What annotation queues exist in Langfuse\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":20,"description":"How many items per page (1 to 100).","title":"Limit"},"page":{"anyOf":[{"maximum":10000,"minimum":1,"type":"integer"},{"type":"null"}],"default":1,"description":"Page number, starting at 1.","title":"Page"}},"title":"list_annotation_queuesArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/get_annotation_queue":{"post":{"operationId":"get_annotation_queue","summary":"Get one annotation queue by id: name, description and score config ids. Use for \"Show Langfuse annotation queue aq_1\".","description":"Get one annotation queue by id: name, description and score config ids. Use for \"Show Langfuse annotation queue aq_1\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"queue_id":{"description":"Annotation queue id as returned by list_annotation_queues.","maxLength":100,"title":"Queue Id","type":"string"}},"required":["queue_id"],"title":"get_annotation_queueArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_annotation_queue_items":{"post":{"operationId":"list_annotation_queue_items","summary":"List the items of an annotation queue (traces, observations or sessions to review) with status and completion time. Use ","description":"List the items of an annotation queue (traces, observations or sessions to review) with status and completion time. Use for \"How many items are still pending in Langfuse annotation queue aq_1\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"queue_id":{"description":"Annotation queue id as returned by list_annotation_queues.","maxLength":100,"title":"Queue Id","type":"string"},"status":{"anyOf":[{"enum":["PENDING","COMPLETED"],"type":"string"},{"type":"null"}],"default":null,"description":"Only items with this status.","title":"Status"},"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":20,"description":"How many items per page (1 to 100).","title":"Limit"},"page":{"anyOf":[{"maximum":10000,"minimum":1,"type":"integer"},{"type":"null"}],"default":1,"description":"Page number, starting at 1.","title":"Page"}},"required":["queue_id"],"title":"list_annotation_queue_itemsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_dashboards":{"post":{"operationId":"list_dashboards","summary":"List custom dashboards (unstable API) with name, description and dates; the widget definitions are not returned. Use for","description":"List custom dashboards (unstable API) with name, description and dates; the widget definitions are not returned. Use for \"What dashboards have been built in Langfuse\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":20,"description":"How many items per page (1 to 100).","title":"Limit"},"page":{"anyOf":[{"maximum":10000,"minimum":1,"type":"integer"},{"type":"null"}],"default":1,"description":"Page number, starting at 1.","title":"Page"}},"title":"list_dashboardsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_llm_connections":{"post":{"operationId":"list_llm_connections","summary":"List the LLM provider connections configured for evaluators: provider, adapter and base URL (secrets are not returned). ","description":"List the LLM provider connections configured for evaluators: provider, adapter and base URL (secrets are not returned). Use for \"Which LLM providers are connected in my Langfuse project\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":20,"description":"How many items per page (1 to 100).","title":"Limit"},"page":{"anyOf":[{"maximum":10000,"minimum":1,"type":"integer"},{"type":"null"}],"default":1,"description":"Page number, starting at 1.","title":"Page"}},"title":"list_llm_connectionsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}}}}