EdgeAIG's picture
download
raw
43.8 kB
import * as Context from "../../Context.js";
import * as Effect from "../../Effect.js";
import * as FiberSet from "../../FiberSet.js";
import { constFalse, identity, pipe } from "../../Function.js";
import * as Option from "../../Option.js";
import * as Predicate from "../../Predicate.js";
import * as Queue from "../../Queue.js";
import { CurrentConcurrency } from "../../References.js";
import * as Schema from "../../Schema.js";
import * as SchemaAST from "../../SchemaAST.js";
import * as Sink from "../../Sink.js";
import * as Stream from "../../Stream.js";
import * as AiError from "./AiError.js";
import { defaultIdGenerator, IdGenerator } from "./IdGenerator.js";
import * as InternalCodecTransformer from "./internal/codec-transformer.js";
import * as Prompt from "./Prompt.js";
import * as Response from "./Response.js";
import * as ResponseIdTracker from "./ResponseIdTracker.js";
import { CurrentSpanTransformer } from "./Telemetry.js";
import * as Toolkit from "./Toolkit.js";
// =============================================================================
// Service Definition
// =============================================================================
/**
* Service tag for AI model services.
*
* **When to use**
*
* Use to access or provide text generation, streaming generation, structured
* output, and tool-calling capabilities through the Effect context.
*
* **Example** (Accessing the language model service)
*
* ```ts
* import { Effect } from "effect"
* import { LanguageModel } from "effect/unstable/ai"
*
* const program = Effect.gen(function*() {
* const model = yield* LanguageModel.LanguageModel
* const response = yield* model.generateText({
* prompt: "What is machine learning?"
* })
* return response.text
* })
* ```
*
* @category services
* @since 4.0.0
*/
export class LanguageModel extends /*#__PURE__*/Context.Service()("effect/unstable/ai/LanguageModel") {}
/**
* The default codec transformer that passes schemas through without
* provider-specific rewrites.
*
* **When to use**
*
* Use as the codec transformer for provider implementations when the provider
* accepts the JSON Schema generated from an `Effect` Schema codec without
* provider-specific rewrites.
*
* **Details**
*
* The transformer returns the original codec, resolves a top-level `$ref`, and
* copies schema definitions into `$defs`.
*
* @see {@link CodecTransformer} for the structured-output transformer contract
* @see {@link make} for where this transformer is used as the default
*
* @category services
* @since 4.0.0
*/
export const defaultCodecTransformer = InternalCodecTransformer.defaultCodecTransformer;
/**
* Response class for text generation operations, with accessors for extracting text, tool calls, usage information, and other response parts from generated content.
*
* **Example** (Inspecting a text response)
*
* ```ts
* import { Effect } from "effect"
* import { LanguageModel } from "effect/unstable/ai"
*
* const program = Effect.gen(function*() {
* const response = yield* LanguageModel.generateText({
* prompt: "Explain photosynthesis"
* })
*
* console.log(response.text) // Generated text content
* console.log(response.finishReason) // "stop", "length", etc.
* console.log(response.usage) // Usage information
*
* return response
* })
* ```
*
* @category models
* @since 4.0.0
*/
export class GenerateTextResponse {
content;
constructor(content) {
this.content = content;
}
/**
* Extracts and concatenates all text parts from the response.
*/
get text() {
const text = [];
for (const part of this.content) {
if (part.type === "text") {
text.push(part.text);
}
}
return text.join("");
}
/**
* Returns all reasoning parts from the response.
*/
get reasoning() {
return this.content.filter(part => part.type === "reasoning");
}
/**
* Extracts and concatenates all reasoning text, or undefined if none exists.
*/
get reasoningText() {
const text = [];
for (const part of this.content) {
if (part.type === "reasoning") {
text.push(part.text);
}
}
return text.length === 0 ? undefined : text.join("");
}
/**
* Returns all tool call parts from the response.
*/
get toolCalls() {
return this.content.filter(part => part.type === "tool-call");
}
/**
* Returns all tool result parts from the response.
*/
get toolResults() {
return this.content.filter(part => part.type === "tool-result");
}
/**
* The reason why text generation finished.
*/
get finishReason() {
const finishPart = this.content.find(part => part.type === "finish");
return Predicate.isUndefined(finishPart) ? "unknown" : finishPart.reason;
}
/**
* Token usage statistics for the generation request.
*/
get usage() {
const finishPart = this.content.find(part => part.type === "finish");
if (Predicate.isUndefined(finishPart)) {
return new Response.Usage({
inputTokens: {
uncached: undefined,
total: undefined,
cacheRead: undefined,
cacheWrite: undefined
},
outputTokens: {
total: undefined,
text: undefined,
reasoning: undefined
}
});
}
return finishPart.usage;
}
}
/**
* Response class for structured object generation operations.
*
* **Example** (Inspecting an object response)
*
* ```ts
* import { Effect, Schema } from "effect"
* import { LanguageModel } from "effect/unstable/ai"
*
* const UserSchema = Schema.Struct({
* name: Schema.String,
* email: Schema.String
* })
*
* const program = Effect.gen(function*() {
* const response = yield* LanguageModel.generateObject({
* prompt: "Create user: John Doe, john@example.com",
* schema: UserSchema
* })
*
* console.log(response.value) // { name: "John Doe", email: "john@example.com" }
* console.log(response.text) // Raw generated text
*
* return response.value
* })
* ```
*
* @category models
* @since 4.0.0
*/
export class GenerateObjectResponse extends GenerateTextResponse {
/**
* The parsed structured object that conforms to the provided schema.
*/
value;
constructor(value, content) {
super(content);
this.value = value;
}
}
/**
* Creates a LanguageModel service from provider-specific text generation and
* streaming implementations.
*
* **When to use**
*
* Use when you are implementing a provider adapter and need to expose the
* standard language-model service while keeping provider-specific request hooks
* behind it.
*
* **Details**
*
* The returned service implements `generateText`, `generateObject`, and
* `streamText`. It prepares `ProviderOptions` for each request, including the
* normalized prompt, tools, tool choice, response format, tracing span, and
* incremental response fields, before calling the supplied provider hook.
* Structured object generation uses the `generateText` hook and the configured
* `codecTransformer`, or `defaultCodecTransformer` when none is supplied.
*
* **Gotchas**
*
* Provider hooks must return encoded response parts that match the toolkit and
* response format prepared in `ProviderOptions`; invalid parts fail decoding as
* `AiError.InvalidOutputError`.
*
* @see {@link Service} for the returned service contract
* @see {@link ProviderOptions} for the normalized options passed to provider hooks
* @see {@link defaultCodecTransformer} for the default structured-output schema transformer
*
* @category constructors
* @since 4.0.0
*/
export const make = /*#__PURE__*/Effect.fnUntraced(function* (params) {
const codecTransformer = params.codecTransformer ?? defaultCodecTransformer;
const parentSpanTransformer = yield* Effect.serviceOption(CurrentSpanTransformer);
const getSpanTransformer = Effect.serviceOption(CurrentSpanTransformer).pipe(Effect.map(Option.orElse(() => parentSpanTransformer)));
const idGenerator = yield* Effect.serviceOption(IdGenerator).pipe(Effect.map(Option.getOrElse(() => defaultIdGenerator)));
const generateText = options => Effect.useSpan("LanguageModel.generateText", {
attributes: {
concurrency: options.concurrency,
toolChoice: options.toolChoice
}
}, Effect.fnUntraced(function* (span) {
const spanTransformer = yield* getSpanTransformer;
const providerOptions = {
prompt: Prompt.make(options.prompt),
tools: [],
toolChoice: "none",
responseFormat: {
type: "text"
},
span,
previousResponseId: undefined,
incrementalPrompt: undefined
};
const content = yield* generateContent(options, providerOptions);
applySpanTransformer(spanTransformer, content, providerOptions);
return new GenerateTextResponse(content);
}, Effect.catchTag("SchemaError", error => Effect.fail(AiError.make({
module: "LanguageModel",
method: "generateText",
reason: AiError.InvalidOutputError.fromSchemaError(error)
}))), (effect, span) => Effect.withParentSpan(effect, span, {
captureStackTrace: false
}), Effect.provideService(IdGenerator, idGenerator)));
const generateObject = options => {
const objectName = getObjectName(options.objectName, options.schema);
return Effect.useSpan("LanguageModel.generateObject", {
attributes: {
objectName,
concurrency: options.concurrency,
toolChoice: options.toolChoice
}
}, Effect.fnUntraced(function* (span) {
const spanTransformer = yield* getSpanTransformer;
const providerOptions = {
prompt: Prompt.make(options.prompt),
tools: [],
toolChoice: "none",
responseFormat: {
type: "json",
objectName,
schema: options.schema
},
span,
previousResponseId: undefined,
incrementalPrompt: undefined
};
const content = yield* generateContent(options, providerOptions);
applySpanTransformer(spanTransformer, content, providerOptions);
const {
codec
} = yield* Effect.try({
try: () => codecTransformer(options.schema),
catch: error => AiError.make({
module: "LanguageModel",
method: "generateObject",
reason: new AiError.UnsupportedSchemaError({
description: error instanceof Error ? error.message : String(error)
})
})
});
const value = yield* resolveStructuredOutput(content, codec);
return new GenerateObjectResponse(value, content);
}, Effect.catchTag("SchemaError", error => Effect.fail(AiError.make({
module: "LanguageModel",
method: "generateObject",
reason: AiError.InvalidOutputError.fromSchemaError(error)
}))), (effect, span) => Effect.withParentSpan(effect, span, {
captureStackTrace: false
}), Effect.provideService(IdGenerator, idGenerator)));
};
const streamText = Effect.fnUntraced(function* (options) {
const span = yield* Effect.makeSpanScoped("LanguageModel.streamText", {
attributes: {
concurrency: options.concurrency,
toolChoice: options.toolChoice
}
});
const providerOptions = {
prompt: Prompt.make(options.prompt),
tools: [],
toolChoice: "none",
responseFormat: {
type: "text"
},
span,
previousResponseId: undefined,
incrementalPrompt: undefined
};
// Resolve the content stream for the request
const stream = yield* streamContent(options, providerOptions);
// Return the stream immediately if there is no span transformer
const spanTransformer = yield* getSpanTransformer;
if (Option.isNone(spanTransformer)) {
return stream;
}
// Otherwise aggregate generated content and apply the span transformer
// when the stream is finished
const content = [];
return stream.pipe(Stream.mapArray(parts => {
content.push(...parts);
return parts;
}), Stream.ensuring(Effect.sync(() => {
spanTransformer.value({
...providerOptions,
response: content
});
})));
}, Stream.unwrap, Stream.mapError(error => Schema.isSchemaError(error) ? AiError.make({
module: "LanguageModel",
method: "streamText",
reason: AiError.InvalidOutputError.fromSchemaError(error)
}) : error), Stream.provideService(IdGenerator, idGenerator));
const generateContent = Effect.fnUntraced(function* (options, providerOptions) {
const tracker = Option.getOrUndefined(yield* Effect.serviceOption(ResponseIdTracker.ResponseIdTracker));
const toolChoice = options.toolChoice ?? "auto";
const generateWithNonIncrementalFallback = () => {
const requestOptions = {
...providerOptions
};
const fallbackPrompt = requestOptions.prompt;
const fallbackOptions = {
...requestOptions,
prompt: fallbackPrompt,
incrementalPrompt: undefined,
previousResponseId: undefined
};
return requestOptions.incrementalPrompt ? params.generateText(requestOptions).pipe(Effect.catchReason("AiError", "InvalidRequestError", _ => params.generateText(fallbackOptions))) : params.generateText(requestOptions);
};
// Check for pending approvals that need resolution
const {
approved,
denied
} = collectToolApprovals(providerOptions.prompt.content, {
excludeResolved: true
});
const hasPendingApprovals = approved.length > 0 || denied.length > 0;
// If there is no toolkit, the generated content can be returned immediately
if (Predicate.isUndefined(options.toolkit)) {
// But first check if we have pending approvals that require a toolkit
if (hasPendingApprovals) {
return yield* AiError.make({
module: "LanguageModel",
method: "generateText",
reason: new AiError.ToolkitRequiredError({
pendingApprovals: [...approved, ...denied].map(result => result.toolCall?.name).filter(Predicate.isNotUndefined)
})
});
}
if (tracker) {
const prepared = tracker.prepareUnsafe(providerOptions.prompt);
if (Option.isSome(prepared)) {
providerOptions.previousResponseId = prepared.value.previousResponseId;
providerOptions.incrementalPrompt = prepared.value.prompt;
}
}
const ResponseSchema = Schema.mutable(Schema.Array(Response.Part(Toolkit.empty)));
const rawContent = yield* generateWithNonIncrementalFallback();
const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent);
if (tracker) {
const responseMetadata = content.find(part => part.type === "response-metadata");
if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) {
tracker.markParts(providerOptions.prompt.content, responseMetadata.id);
}
}
return content;
}
// If there is a toolkit resolve and apply it to the provider options
const toolkit = yield* resolveToolkit(options.toolkit);
// If the resolved toolkit is empty, return the generated content immediately
if (Object.values(toolkit.tools).length === 0) {
// But first check if we have pending approvals that require a toolkit
if (hasPendingApprovals) {
return yield* AiError.make({
module: "LanguageModel",
method: "generateText",
reason: new AiError.ToolkitRequiredError({
pendingApprovals: [...approved, ...denied].map(result => result.toolCall?.name).filter(Predicate.isNotUndefined)
})
});
}
if (tracker) {
const prepared = tracker.prepareUnsafe(providerOptions.prompt);
if (Option.isSome(prepared)) {
providerOptions.previousResponseId = prepared.value.previousResponseId;
providerOptions.incrementalPrompt = prepared.value.prompt;
}
}
const ResponseSchema = Schema.mutable(Schema.Array(Response.Part(Toolkit.empty)));
const rawContent = yield* generateWithNonIncrementalFallback();
const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent);
if (tracker) {
const responseMetadata = content.find(part => part.type === "response-metadata");
if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) {
tracker.markParts(providerOptions.prompt.content, responseMetadata.id);
}
}
return content;
}
// Pre-resolve pending tool approvals before calling the LLM
if (hasPendingApprovals) {
for (const approval of approved) {
if (approval.toolCall && !toolkit.tools[approval.toolCall.name]) {
return yield* AiError.make({
module: "LanguageModel",
method: "generateText",
reason: new AiError.ToolNotFoundError({
toolName: approval.toolCall.name,
availableTools: Object.keys(toolkit.tools)
})
});
}
}
const approvedResults = yield* executeApprovedToolCalls(approved, toolkit, options.concurrency);
const deniedResults = createDenialResults(denied);
const preResolvedResults = [...approvedResults, ...deniedResults];
if (preResolvedResults.length > 0) {
providerOptions.prompt = Prompt.fromMessages([...providerOptions.prompt.content, Prompt.makeMessage("tool", {
content: preResolvedResults
})]);
}
}
// Strip all resolved approval artifacts (both current and from previous
// rounds) in a single pass before sending to the provider.
{
const {
approved: allResolved,
denied: allDenied
} = collectToolApprovals(providerOptions.prompt.content);
if (allResolved.length > 0 || allDenied.length > 0) {
providerOptions.prompt = stripResolvedApprovals(providerOptions.prompt, allResolved, allDenied);
}
}
const tools = typeof toolChoice === "object" && "oneOf" in toolChoice ? Object.values(toolkit.tools).filter(tool => toolChoice.oneOf.includes(tool.name)) : Object.values(toolkit.tools);
providerOptions.tools = tools;
providerOptions.toolChoice = toolChoice;
if (tracker) {
const prepared = tracker.prepareUnsafe(providerOptions.prompt);
if (Option.isSome(prepared)) {
providerOptions.previousResponseId = prepared.value.previousResponseId;
providerOptions.incrementalPrompt = prepared.value.prompt;
}
}
// Construct the response schema with the tools from the toolkit
const ResponseSchema = Schema.mutable(Schema.Array(Response.Part(toolkit)));
// If tool call resolution is disabled, return the response without
// resolving the tool calls that were generated
if (options.disableToolCallResolution === true) {
const rawContent = yield* generateWithNonIncrementalFallback();
const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent);
if (tracker) {
const responseMetadata = content.find(part => part.type === "response-metadata");
if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) {
tracker.markParts(providerOptions.prompt.content, responseMetadata.id);
}
}
return content;
}
const rawContent = yield* generateWithNonIncrementalFallback();
// Resolve the generated tool calls
const toolResults = yield* resolveToolCalls(rawContent, toolkit, providerOptions.prompt.content, options.concurrency).pipe(Stream.filter(result => result.type === "tool-approval-request" || result.preliminary === false), Stream.runCollect);
const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent);
if (tracker) {
const responseMetadata = content.find(part => part.type === "response-metadata");
if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) {
tracker.markParts(providerOptions.prompt.content, responseMetadata.id);
}
}
// Return the content merged with the tool call results
return [...content, ...toolResults];
});
const streamContent = Effect.fnUntraced(function* (options, providerOptions) {
const tracker = Option.getOrUndefined(yield* Effect.serviceOption(ResponseIdTracker.ResponseIdTracker));
const toolChoice = options.toolChoice ?? "auto";
const streamWithNonIncrementalFallback = () => {
const requestOptions = {
...providerOptions
};
const fallbackPrompt = requestOptions.prompt;
const fallbackOptions = {
...requestOptions,
prompt: fallbackPrompt,
incrementalPrompt: undefined,
previousResponseId: undefined
};
return requestOptions.incrementalPrompt ? params.streamText(requestOptions).pipe(Stream.catchReason("AiError", "InvalidRequestError", _ => params.streamText(fallbackOptions))) : params.streamText(requestOptions);
};
// Check for pending approvals that need resolution
const {
approved: pendingApproved,
denied: pendingDenied
} = collectToolApprovals(providerOptions.prompt.content, {
excludeResolved: true
});
const hasPendingApprovals = pendingApproved.length > 0 || pendingDenied.length > 0;
// If there is no toolkit, return immediately
if (Predicate.isUndefined(options.toolkit)) {
// But first check if we have pending approvals that require a toolkit
if (hasPendingApprovals) {
return yield* AiError.make({
module: "LanguageModel",
method: "streamText",
reason: new AiError.ToolkitRequiredError({
pendingApprovals: [...pendingApproved, ...pendingDenied].map(a => a.toolCall?.name).filter(Predicate.isNotUndefined)
})
});
}
if (tracker) {
const prepared = tracker.prepareUnsafe(providerOptions.prompt);
if (Option.isSome(prepared)) {
providerOptions.previousResponseId = prepared.value.previousResponseId;
providerOptions.incrementalPrompt = prepared.value.prompt;
}
}
const schema = Schema.NonEmptyArray(Response.StreamPart(Toolkit.empty));
const decodeParts = Schema.decodeEffect(schema);
return pipe(streamWithNonIncrementalFallback(), Stream.mapArrayEffect(parts => decodeParts(parts).pipe(tracker ? Effect.tap(decodedParts => {
for (const part of decodedParts) {
if (part.type === "response-metadata" && Predicate.isNotUndefined(part.id)) {
tracker.markParts(providerOptions.prompt.content, part.id);
}
}
return Effect.void;
}) : identity)));
}
// If there is a toolkit resolve and apply it to the provider options
const toolkit = yield* resolveToolkit(options.toolkit);
// If the toolkit is empty, return immediately
if (Object.values(toolkit.tools).length === 0) {
// But first check if we have pending approvals that require a toolkit
if (hasPendingApprovals) {
return yield* AiError.make({
module: "LanguageModel",
method: "streamText",
reason: new AiError.ToolkitRequiredError({
pendingApprovals: [...pendingApproved, ...pendingDenied].map(a => a.toolCall?.name).filter(Predicate.isNotUndefined)
})
});
}
if (tracker) {
const prepared = tracker.prepareUnsafe(providerOptions.prompt);
if (Option.isSome(prepared)) {
providerOptions.previousResponseId = prepared.value.previousResponseId;
providerOptions.incrementalPrompt = prepared.value.prompt;
}
}
const schema = Schema.NonEmptyArray(Response.StreamPart(Toolkit.empty));
const decodeParts = Schema.decodeEffect(schema);
return pipe(streamWithNonIncrementalFallback(), Stream.mapArrayEffect(parts => decodeParts(parts).pipe(tracker ? Effect.tap(decodedParts => {
for (const part of decodedParts) {
if (part.type === "response-metadata" && part.id) {
tracker.markParts(providerOptions.prompt.content, part.id);
}
}
return Effect.void;
}) : identity)));
}
// Pre-resolve pending tool approvals before calling the LLM
let preResolvedStreamParts = [];
if (hasPendingApprovals) {
for (const approval of pendingApproved) {
if (approval.toolCall && !toolkit.tools[approval.toolCall.name]) {
return yield* AiError.make({
module: "LanguageModel",
method: "streamText",
reason: new AiError.ToolNotFoundError({
toolName: approval.toolCall.name,
availableTools: Object.keys(toolkit.tools)
})
});
}
}
const approvedResults = yield* executeApprovedToolCalls(pendingApproved, toolkit, options.concurrency);
const deniedResults = createDenialResults(pendingDenied);
const preResolvedResults = [...approvedResults, ...deniedResults];
if (preResolvedResults.length > 0) {
providerOptions.prompt = Prompt.fromMessages([...providerOptions.prompt.content, Prompt.makeMessage("tool", {
content: preResolvedResults
})]);
}
// Emit pre-resolved tool-results as stream parts so Chat.streamText
// persists them to history. This lets collectToolApprovals find them
// on subsequent rounds and skip the now-resolved approvals.
// Note: r.result is already encoded (from executeApprovedToolCalls /
// createDenialResults), so it goes into both result and encodedResult.
for (const r of preResolvedResults) {
preResolvedStreamParts.push(Response.makePart("tool-result", {
id: r.id,
name: r.name,
providerExecuted: false,
preliminary: false,
result: r.result,
encodedResult: r.result,
isFailure: r.isFailure
}));
}
}
// Strip all resolved approval artifacts (both current and from previous
// rounds) in a single pass before sending to the provider.
const {
approved: allResolved,
denied: allDenied
} = collectToolApprovals(providerOptions.prompt.content);
if (allResolved.length > 0 || allDenied.length > 0) {
providerOptions.prompt = stripResolvedApprovals(providerOptions.prompt, allResolved, allDenied);
}
const tools = typeof toolChoice === "object" && "oneOf" in toolChoice ? Object.values(toolkit.tools).filter(tool => toolChoice.oneOf.includes(tool.name)) : Object.values(toolkit.tools);
providerOptions.tools = tools;
providerOptions.toolChoice = toolChoice;
if (tracker) {
const prepared = tracker.prepareUnsafe(providerOptions.prompt);
if (Option.isSome(prepared)) {
providerOptions.previousResponseId = prepared.value.previousResponseId;
providerOptions.incrementalPrompt = prepared.value.prompt;
}
}
// If tool call resolution is disabled, return the response without
// resolving the tool calls that were generated
if (options.disableToolCallResolution === true) {
const schema = Schema.NonEmptyArray(Response.StreamPart(toolkit));
const decodeParts = Schema.decodeEffect(schema);
return streamWithNonIncrementalFallback().pipe(Stream.mapArrayEffect(parts => decodeParts(parts).pipe(tracker ? Effect.tap(decodedParts => {
for (const part of decodedParts) {
if (part.type === "response-metadata" && Predicate.isNotUndefined(part.id)) {
tracker.markParts(providerOptions.prompt.content, part.id);
}
}
return Effect.void;
}) : identity)));
}
const ResponseSchema = Schema.NonEmptyArray(Response.StreamPart(toolkit));
const decodeParts = Schema.decodeEffect(ResponseSchema);
// Queue for decoded parts and tool results
const queue = yield* Queue.make();
const deferredFinishParts = [];
// Emit pre-resolved tool results so Chat.streamText persists them to
// history. This ensures collectToolApprovals({ excludeResolved }) can
// find the corresponding tool-results on future rounds.
if (preResolvedStreamParts.length > 0) {
yield* Queue.offerAll(queue, preResolvedStreamParts);
}
// FiberSet to track concurrent tool call handlers
const toolCallFibers = yield* FiberSet.make();
// Helper function to handle tool calls with approval logic
const handleToolCall = Effect.fnUntraced(function* (part) {
const tool = toolkit.tools[part.name];
if (!tool) return;
const needsApproval = yield* isApprovalNeeded(tool, part, providerOptions.prompt.content);
if (needsApproval) {
const idGen = yield* IdGenerator;
const approvalId = yield* idGen.generateId();
const approvalPart = Response.makePart("tool-approval-request", {
approvalId,
toolCallId: part.id
});
yield* Queue.offer(queue, approvalPart);
return;
}
yield* toolkit.handle(part.name, part.params).pipe(Stream.unwrap, Stream.runForEach(result => {
const toolResultPart = Response.makePart("tool-result", {
id: part.id,
name: part.name,
providerExecuted: false,
...result
});
return Queue.offer(queue, toolResultPart);
}));
});
yield* streamWithNonIncrementalFallback().pipe(Stream.runForEachArray(Effect.fnUntraced(function* (chunk) {
const parts = yield* decodeParts(chunk);
if (tracker) {
for (const part of parts) {
if (part.type === "response-metadata" && part.id) {
tracker.markParts(providerOptions.prompt.content, part.id);
}
}
}
// Defer finish parts until all tool handlers complete. This guarantees
// tool results are emitted before finish in streaming mode.
const immediateParts = [];
for (const part of parts) {
if (part.type === "finish") {
deferredFinishParts.push(part);
} else {
immediateParts.push(part);
}
}
if (immediateParts.length > 0) {
yield* Queue.offerAll(queue, immediateParts);
}
// Fork tool call handlers - use the raw chunk for encoded params
for (const part of chunk) {
if (part.type === "tool-call" && part.providerExecuted !== true) {
yield* FiberSet.run(toolCallFibers, handleToolCall(part));
}
}
})),
// Wait for all tool calls to either:
// - complete (FiberSet.awaitEmpty)
// - fail (FiberSet.join)
Effect.andThen(Effect.raceFirst(FiberSet.join(toolCallFibers), FiberSet.awaitEmpty(toolCallFibers))), Effect.andThen(Queue.offerAll(queue, deferredFinishParts)),
// And then end the queue
Effect.andThen(Queue.end(queue)), Effect.tapCause(cause => Queue.failCause(queue, cause)), Effect.forkScoped);
return Stream.fromQueue(queue);
});
return {
generateText: generateText,
generateObject,
streamText: streamText
};
});
// =============================================================================
// Accessors
// =============================================================================
/**
* Generates text using a language model.
*
* **Example** (Generating text with options)
*
* ```ts
* import { Effect } from "effect"
* import { LanguageModel } from "effect/unstable/ai"
*
* const program = Effect.gen(function*() {
* const response = yield* LanguageModel.generateText({
* prompt: "Write a haiku about programming",
* toolChoice: "none"
* })
*
* console.log(response.text)
* console.log(response.usage.inputTokens.total)
*
* return response
* })
* ```
*
* @category text generation
* @since 4.0.0
*/
export const generateText = options => Effect.flatMap(Effect.service(LanguageModel), model => model.generateText(options));
/**
* Generates a structured object from a schema using a language model.
*
* **Example** (Generating an object)
*
* ```ts
* import { Effect, Schema } from "effect"
* import { LanguageModel } from "effect/unstable/ai"
*
* const EventSchema = Schema.Struct({
* title: Schema.String,
* date: Schema.String,
* location: Schema.String
* })
*
* const program = Effect.gen(function*() {
* const response = yield* LanguageModel.generateObject({
* prompt:
* "Extract event info: Tech Conference on March 15th in San Francisco",
* schema: EventSchema,
* objectName: "event"
* })
*
* console.log(response.value)
* // { title: "Tech Conference", date: "March 15th", location: "San Francisco" }
*
* return response.value
* })
* ```
*
* @category object generation
* @since 4.0.0
*/
export const generateObject = options => Effect.flatMap(Effect.service(LanguageModel), model => model.generateObject(options));
/**
* Generates text using a language model with streaming output.
*
* **Details**
*
* Returns a stream of response parts that are emitted as soon as they are
* available from the model, enabling real-time text generation experiences.
*
* **Example** (Streaming text deltas)
*
* ```ts
* import { Console, Effect, Stream } from "effect"
* import { LanguageModel } from "effect/unstable/ai"
*
* const program = LanguageModel.streamText({
* prompt: "Write a story about a space explorer"
* }).pipe(Stream.runForEach((part) => {
* if (part.type === "text-delta") {
* return Console.log(part.delta)
* }
* return Effect.void
* }))
* ```
*
* @category text generation
* @since 4.0.0
*/
export const streamText = options => Stream.unwrap(Effect.map(Effect.service(LanguageModel), model => model.streamText(options)));
const collectToolApprovals = (messages, options) => {
const requests = new Map();
const responses = [];
const toolCallsById = new Map();
const toolResultIds = new Set();
// Collect all tool approval requests, responses, tool calls, and tool results
for (const message of messages) {
if (message.role === "assistant") {
for (const part of message.content) {
if (part.type === "tool-approval-request") {
requests.set(part.approvalId, {
approvalId: part.approvalId,
toolCallId: part.toolCallId
});
}
if (part.type === "tool-call") {
toolCallsById.set(part.id, part);
}
}
}
if (message.role === "tool") {
for (const part of message.content) {
if (part.type === "tool-approval-response") {
responses.push({
approvalId: part.approvalId,
approved: part.approved,
reason: part.reason
});
}
if (part.type === "tool-result") {
toolResultIds.add(part.id);
}
}
}
}
const approved = [];
const denied = [];
for (const response of responses) {
const request = requests.get(response.approvalId);
if (Predicate.isNotUndefined(request)) {
// Skip if already resolved
if (options?.excludeResolved && toolResultIds.has(request.toolCallId)) {
continue;
}
const result = {
...response,
toolCallId: request.toolCallId,
toolCall: toolCallsById.get(request.toolCallId)
};
if (response.approved) {
approved.push(result);
} else {
denied.push(result);
}
}
}
return {
approved,
denied
};
};
/**
* Strip resolved approval artifacts from the prompt before sending to the
* provider. After pre-resolving approvals (executing approved tools and
* creating denial results), the original `tool-approval-request` parts in
* assistant messages and `tool-approval-response` parts in tool messages are
* no longer needed. Leaving them in causes provider-specific errors (e.g.
* OpenAI rejects `mcp_approval_response` items that reference approval
* requests it never issued).
*/
const stripResolvedApprovals = (prompt, approved, denied) => {
const resolvedApprovalIds = new Set();
for (const a of approved) resolvedApprovalIds.add(a.approvalId);
for (const d of denied) resolvedApprovalIds.add(d.approvalId);
const cleanedMessages = [];
for (const message of prompt.content) {
if (message.role === "assistant") {
const filteredContent = message.content.filter(part => part.type !== "tool-approval-request" || !resolvedApprovalIds.has(part.approvalId));
if (filteredContent.length > 0) {
cleanedMessages.push(Prompt.makeMessage("assistant", {
content: filteredContent,
options: message.options
}));
}
} else if (message.role === "tool") {
const filteredContent = message.content.filter(part => part.type !== "tool-approval-response" || !resolvedApprovalIds.has(part.approvalId));
if (filteredContent.length > 0) {
cleanedMessages.push(Prompt.makeMessage("tool", {
content: filteredContent,
options: message.options
}));
}
} else {
cleanedMessages.push(message);
}
}
return Prompt.fromMessages(cleanedMessages);
};
const isApprovalNeeded = /*#__PURE__*/Effect.fnUntraced(function* (tool, toolCall, messages) {
if (Predicate.isUndefined(tool.needsApproval)) {
return false;
}
if (typeof tool.needsApproval === "function") {
const params = yield* Schema.decodeUnknownEffect(tool.parametersSchema)(toolCall.params);
const result = tool.needsApproval(params, {
toolCallId: toolCall.id,
messages
});
return Effect.isEffect(result) ? yield* result : result;
}
return tool.needsApproval;
}, /*#__PURE__*/Effect.orElseSucceed(constFalse));
const executeApprovedToolCalls = (approvals, toolkit, concurrency) => {
const executeTool = Effect.fnUntraced(function* (approval) {
const toolCall = approval.toolCall;
if (Predicate.isUndefined(toolCall)) {
return yield* Effect.die("Approval missing tool call reference");
}
const tool = toolkit.tools[toolCall.name];
if (Predicate.isUndefined(tool)) {
return yield* AiError.make({
module: "LanguageModel",
method: "generateText",
reason: new AiError.ToolNotFoundError({
toolName: toolCall.name,
availableTools: Object.keys(toolkit.tools)
})
});
}
const resultStream = yield* toolkit.handle(toolCall.name, toolCall.params);
const terminalResult = yield* resultStream.pipe(Stream.filter(result => result.preliminary === false), Stream.run(Sink.last()), Effect.flatMap(Option.match({
onNone: () => Effect.die("Tool handler did not produce a final result"),
onSome: Effect.succeed
})));
return Prompt.makePart("tool-result", {
id: approval.toolCallId,
name: toolCall.name,
isFailure: terminalResult.isFailure,
result: terminalResult.encodedResult
});
});
return Effect.gen(function* () {
const resolveConcurrency = concurrency === "inherit" ? yield* Effect.service(CurrentConcurrency) : concurrency ?? "unbounded";
return yield* Effect.forEach(approvals, executeTool, {
concurrency: resolveConcurrency
});
});
};
const createDenialResults = denials => {
const results = [];
for (const denial of denials) {
if (Predicate.isNotUndefined(denial.toolCall)) {
results.push(Prompt.makePart("tool-result", {
id: denial.toolCallId,
name: denial.toolCall.name,
isFailure: true,
result: {
type: "execution-denied",
reason: denial.reason
}
}));
}
}
return results;
};
const resolveToolCalls = (content, toolkit, messages, concurrency) => {
const toolCalls = [];
for (const part of content) {
if (part.type === "tool-call") {
if (part.providerExecuted === true) {
continue;
}
toolCalls.push(part);
}
}
const {
approved,
denied
} = collectToolApprovals(messages);
const approvedToolCallIds = new Set(approved.map(approval => approval.toolCallId));
const deniedByToolCallId = new Map(denied.map(denial => [denial.toolCallId, denial]));
const streams = toolCalls.map(toolCall => Effect.gen(function* () {
const tool = toolkit.tools[toolCall.name];
if (!tool) {
return Stream.empty;
}
if (deniedByToolCallId.has(toolCall.id)) {
const denial = deniedByToolCallId.get(toolCall.id);
return Stream.succeed(Response.makePart("tool-result", {
id: toolCall.id,
name: toolCall.name,
providerExecuted: false,
isFailure: true,
result: {
type: "execution-denied",
reason: denial.reason
},
encodedResult: {
type: "execution-denied",
reason: denial.reason
},
preliminary: false
}));
}
if (approvedToolCallIds.has(toolCall.id)) {
return toolkit.handle(toolCall.name, toolCall.params).pipe(Stream.unwrap, Stream.map(result => Response.makePart("tool-result", {
id: toolCall.id,
name: toolCall.name,
providerExecuted: false,
...result
})));
}
const needsApproval = yield* isApprovalNeeded(tool, toolCall, messages);
if (needsApproval) {
const generator = yield* IdGenerator;
const approvalId = yield* generator.generateId();
return Stream.succeed(Response.makePart("tool-approval-request", {
approvalId,
toolCallId: toolCall.id
}));
}
return toolkit.handle(toolCall.name, toolCall.params).pipe(Stream.unwrap, Stream.map(result => Response.makePart("tool-result", {
id: toolCall.id,
name: toolCall.name,
providerExecuted: false,
...result
})));
}).pipe(Stream.unwrap));
const resolveConcurrency = concurrency === "inherit" ? Effect.service(CurrentConcurrency) : Effect.succeed(concurrency ?? "unbounded");
return resolveConcurrency.pipe(Effect.map(concurrency => Stream.mergeAll(streams, {
concurrency
})), Stream.unwrap);
};
// =============================================================================
// Utilities
// =============================================================================
const resolveToolkit = toolkit => Effect.isEffect(toolkit) ? toolkit : Effect.succeed(toolkit);
/** @internal */
export const getObjectName = (objectName, schema) => {
if (Predicate.isNotUndefined(objectName)) {
return objectName;
}
if ("identifier" in schema && typeof schema.identifier === "string") {
return schema.identifier;
}
const identifier = SchemaAST.resolveIdentifier(schema.ast);
if (typeof identifier === "string") {
return identifier;
}
return "generateObject";
};
const resolveStructuredOutput = /*#__PURE__*/Effect.fnUntraced(function* (response, schema) {
const texts = [];
for (const part of response) {
if (part.type === "text") {
texts.push(part.text);
}
}
const text = texts.join("");
if (text.length === 0) {
return yield* AiError.make({
module: "LanguageModel",
method: "generateObject",
reason: new AiError.StructuredOutputError({
description: "No text content in response",
responseText: text
})
});
}
const decode = Schema.decodeEffect(Schema.fromJsonString(schema));
return yield* Effect.mapError(decode(text), error => AiError.make({
module: "LanguageModel",
method: "generateObject",
reason: AiError.StructuredOutputError.fromSchemaError(error, text)
}));
});
const applySpanTransformer = (transformer, response, options) => {
if (Option.isSome(transformer)) {
transformer.value({
...options,
response: response
});
}
};
//# sourceMappingURL=LanguageModel.js.map

Xet Storage Details

Size:
43.8 kB
·
Xet hash:
b372462e19fac8383a0c4abb46d87c6bd2efc5032c73731e0d41c624c83cf172

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.