import OpenAI from "openai";
const openai = new OpenAI({
baseURL: "https://api.inference.net/v1",
apiKey: "<YOUR_API_KEY>",
});
const completion = await openai.chat.completions.create({
model: "glm-5",
messages: [
{
role: "user",
content: "What is the meaning of life?"
}
],
stream: true,
});
for await (const chunk of completion) {
process.stdout.write(chunk.choices[0]?.delta.content as string);
}import OpenAI from "openai";
const openai = new OpenAI({
baseURL: "https://api.inference.net/v1",
apiKey: "<YOUR_API_KEY>",
});
const completion = await openai.chat.completions.create({
model: "glm-5",
messages: [
{
role: "user",
content: "What is the meaning of life?"
}
],
stream: true,
});
for await (const chunk of completion) {
process.stdout.write(chunk.choices[0]?.delta.content as string);
}Prompt caching
- Keep the prompt prefix stable and byte-identical across calls.
- Pass a stable session_id for routing stickiness and a stable prompt_cache_key for cache affinity.
- Check usage.prompt_tokens_details.cached_tokens in the response to confirm cache reads.
{
"session_id": "my-stable-session",
"prompt_cache_key": "v1-stable-prefix",
"messages": [{ "role": "user", "content": "..." }]
}{
"session_id": "my-stable-session",
"prompt_cache_key": "v1-stable-prefix",
"messages": [{ "role": "user", "content": "..." }]
}