### Install ai-tokenizer with Bun

Source: https://github.com/coder/ai-tokenizer/blob/main/README.md

Install the ai-tokenizer package using Bun. This is the recommended package manager for this library.

```sh
bun install ai-tokenizer
```

--------------------------------

### Access and Use Model Configurations

Source: https://context7.com/coder/ai-tokenizer/llms.txt

Access pre-configured settings for various AI models, including encoding type, context window, and pricing. Use these configurations to create a tokenizer and calculate costs.

```typescript
import { models, Tokenizer, type Model, type ModelName } from "ai-tokenizer";
import * as encoding from "ai-tokenizer/encoding";

// Access model configuration
const gpt5 = models["openai/gpt-5"];
console.log(gpt5.name);           // "GPT-5"
console.log(gpt5.encoding);       // "o200k_base"
console.log(gpt5.contextWindow);  // 128000
console.log(gpt5.maxTokens);      // 16384
console.log(gpt5.pricing);        // { input: 0.0000025, output: 0.00001, ... }

// Create tokenizer from model's encoding
const model = models["anthropic/claude-sonnet-4.5"];
const tokenizer = new Tokenizer(encoding[model.encoding]);

// Calculate costs
const inputText = "Write a comprehensive analysis of machine learning trends.";
const tokenCount = tokenizer.count(inputText);
const inputCostUSD = tokenCount * model.pricing.input;
console.log(`Input tokens: ${tokenCount}, Cost: $${inputCostUSD.toFixed(6)}`);
// Output: Input tokens: 9, Cost: $0.000027

// List available models
const modelNames = Object.keys(models) as ModelName[];
console.log(`Available models: ${modelNames.length}`);
// Output: Available models: 90+
```

--------------------------------

### Select and Use Different Encodings

Source: https://context7.com/coder/ai-tokenizer/llms.txt

Import and use various encoding formats provided by the library for different AI model families. Encodings can also be imported from the main encoding index.

```typescript
import Tokenizer from "ai-tokenizer";
import * as o200k_base from "ai-tokenizer/encoding/o200k_base";   // GPT-4o, GPT-5, o3
import * as cl100k_base from "ai-tokenizer/encoding/cl100k_base"; // GPT-4, GPT-3.5
import * as p50k_base from "ai-tokenizer/encoding/p50k_base";     // Codex models
import * as claude from "ai-tokenizer/encoding/claude";           // Claude models

// Use o200k_base for OpenAI GPT-4o and newer models
const gptTokenizer = new Tokenizer(o200k_base);
const gptTokens = gptTokenizer.encode("function hello() { return 42; }");
console.log("GPT tokens:", gptTokens.length);
// Output: GPT tokens: 10

// Use claude encoding for Anthropic Claude models
const claudeTokenizer = new Tokenizer(claude);
const claudeTokens = claudeTokenizer.encode("function hello() { return 42; }");
console.log("Claude tokens:", claudeTokens.length);
// Output: Claude tokens: 11

// Encodings can also be imported from the index
import * as encoding from "ai-tokenizer/encoding";
const tokenizer2 = new Tokenizer(encoding.cl100k_base);
```

--------------------------------

### Initialize and Use Tokenizer

Source: https://context7.com/coder/ai-tokenizer/llms.txt

Initialize the Tokenizer with a specific encoding and use its methods to encode text, decode tokens, and count tokens. Handles special tokens during encoding.

```typescript
import Tokenizer from "ai-tokenizer";
import * as o200k_base from "ai-tokenizer/encoding/o200k_base";

// Initialize tokenizer with an encoding
const tokenizer = new Tokenizer(o200k_base);

// Encode text to token IDs
const tokens = tokenizer.encode("Hello, world!");
console.log(tokens);
// Output: [9906, 11, 1917, 0]

// Decode tokens back to text
const text = tokenizer.decode(tokens);
console.log(text);
// Output: "Hello, world!"

// Count tokens in text
const count = tokenizer.count("The quick brown fox jumps over the lazy dog.");
console.log(count);
// Output: 10

// Handle special tokens
const tokensWithSpecial = tokenizer.encode(
  "Hello <|endoftext|> world",
  ["<|endoftext|>"],  // allowedSpecial
  []                   // disallowedSpecial
);
```

--------------------------------

### Estimate tokens with AI SDK

Source: https://github.com/coder/ai-tokenizer/blob/main/README.md

Estimate token counts for messages and tools using the AI SDK integration. Ensure all necessary imports are included. Note that no API calls are made, and no keys are required.

```ts
import Tokenizer, { models } from "ai-tokenizer";
import { count } from "ai-tokenizer/sdk";
import * as encoding from "ai-tokenizer/encoding";

// Find the respective model.
const model = models["openai/gpt-5"];
const tokenizer = new Tokenizer(encoding[model.encoding]);
const result = count({
  tokenizer,
  model,
  // Every message is counted separately
  messages,
  // Tool descriptions and input schemas are counted
  tools,
})

// {
//   total: 150,
//   messages: [
//     {
//       total: 120,
//       content: [
//         { type: "text", total: 100 },
//         { type: "tool-call", total: 20, input: 15 }
//       ]
//     }
//   ],
//   tools: {
//     total: 30,
//     definitions: {
//       myTool: {
//         name: 5,
//         description: 10,
//         inputSchema: 15
//       }
//     }
//   }
// }

// No API calls are performed; no keys.

const costUSD = result.total * model.pricing.input;
```

--------------------------------

### Count Tokens for AI SDK Messages and Tools

Source: https://context7.com/coder/ai-tokenizer/llms.txt

Estimates token usage for messages and tool definitions, providing a detailed breakdown of costs.

```typescript
import Tokenizer, { models } from "ai-tokenizer";
import { count } from "ai-tokenizer/sdk";
import * as encoding from "ai-tokenizer/encoding";
import type { ModelMessage } from "ai";
import { z } from "zod";

// Setup tokenizer and model
const model = models["openai/gpt-5"];
const tokenizer = new Tokenizer(encoding[model.encoding]);

// Define messages (AI SDK format)
const messages: ModelMessage[] = [
  { role: "system", content: "You are a helpful assistant." },
  { role: "user", content: "What's the weather in San Francisco?" },
  {
    role: "assistant",
    content: [
      {
        type: "tool-call",
        toolCallId: "call_123",
        toolName: "getWeather",
        input: { location: "San Francisco", units: "celsius" }
      }
    ]
  },
  {
    role: "tool",
    content: [
      {
        type: "tool-result",
        toolCallId: "call_123",
        toolName: "getWeather",
        output: { temperature: 18, conditions: "Partly cloudy" }
      }
    ]
  }
];

// Define tools with Zod schemas
const tools = {
  getWeather: {
    description: "Get current weather for a location",
    inputSchema: z.object({
      location: z.string().describe("City name"),
      units: z.enum(["celsius", "fahrenheit"]).describe("Temperature units")
    })
  },
  searchWeb: {
    description: "Search the web for information",
    inputSchema: z.object({
      query: z.string().describe("Search query"),
      maxResults: z.number().describe("Maximum results to return")
    })
  }
};

// Count tokens with detailed breakdown
const result = count({ tokenizer, model, messages, tools });

console.log("Total tokens:", result.total);
// Output: Total tokens: 187

console.log("Messages breakdown:", result.messages.map(m => ({
  total: m.total,
  content: m.content
})));
// Output: Messages breakdown: [
//   { total: 13, content: [{ type: "text", total: 9 }] },
//   { total: 15, content: [{ type: "text", total: 11 }] },
//   { total: 22, content: [{ type: "tool-call", total: 18, input: 12 }] },
//   { total: 28, content: [{ type: "tool-result", total: 24, output: 18 }] }
// ]

console.log("Tools breakdown:", result.tools);
// Output: Tools breakdown: {
//   total: 109,
//   definitions: {
//     getWeather: { name: 1, description: 7, inputSchema: 42 },
//     searchWeb: { name: 1, description: 6, inputSchema: 35 }
//   }
// }

// Calculate cost
const inputCostUSD = result.total * model.pricing.input;
console.log(`Estimated input cost: $${inputCostUSD.toFixed(6)}`);
// Output: Estimated input cost: $0.000468
```

--------------------------------

### Estimate tokens using an encoding

Source: https://github.com/coder/ai-tokenizer/blob/main/README.md

Directly estimate token counts for a given text string using a specific encoding. Import only the necessary encoding to manage bundle size, as each encoding can be large.

```ts
import { Tokenizer } from "ai-tokenizer"
import * as o200k_base from "ai-tokenizer/encoding/o200k_base"

const tokenizer = new Tokenizer(o200k_base);
const total = tokenizer.count("some text input");
```

--------------------------------

### Configure LRU Merge Cache Size

Source: https://context7.com/coder/ai-tokenizer/llms.txt

Adjust the merge cache size to balance memory usage and performance for repeated text patterns. Larger caches improve performance for high-throughput applications.

```typescript
import Tokenizer from "ai-tokenizer";
import * as o200k_base from "ai-tokenizer/encoding/o200k_base";

// Default cache size: 100,000 entries
const defaultTokenizer = new Tokenizer(o200k_base);

// Smaller cache for memory-constrained environments
const smallCacheTokenizer = new Tokenizer(
  o200k_base,
  undefined,  // extendedSpecialTokens
  10000       // mergeCacheSize
);

// Larger cache for high-throughput applications
const largeCacheTokenizer = new Tokenizer(
  o200k_base,
  undefined,
  500000
);

// Extended special tokens for custom applications
const customTokenizer = new Tokenizer(
  o200k_base,
  {
    "<|custom_start|>": 200100,
    "<|custom_end|>": 200101
  },
  100000
);

// Benchmark repeated encoding (cache benefits)
const text = "This is a repeated phrase that benefits from caching.";
const iterations = 10000;

const start = performance.now();
for (let i = 0; i < iterations; i++) {
  largeCacheTokenizer.encode(text);
}
const elapsed = performance.now() - start;
console.log(`${iterations} encodings in ${elapsed.toFixed(2)}ms`);
// Output: 10000 encodings in ~15ms (cached)
```

--------------------------------

### Perform Round-Trip Encoding and Decoding

Source: https://context7.com/coder/ai-tokenizer/llms.txt

Verifies that the tokenizer can encode and decode various text formats, including Unicode, code, and JSON, without data loss.

```typescript
import Tokenizer from "ai-tokenizer";
import * as o200k_base from "ai-tokenizer/encoding/o200k_base";

const tokenizer = new Tokenizer(o200k_base);

// Unicode text round-trip
const unicodeText = "Hello 你好 مرحبا 🌍 🚀";
const unicodeTokens = tokenizer.encode(unicodeText);
const unicodeDecoded = tokenizer.decode(unicodeTokens);
console.log(unicodeDecoded === unicodeText);
// Output: true

// Code round-trip
const codeText = `
function greet(name: string): string {
  return \`Hello, \${name}! 👋\`;
}
`;
const codeTokens = tokenizer.encode(codeText);
const codeDecoded = tokenizer.decode(codeTokens);
console.log(codeDecoded === codeText);
// Output: true

// JSON round-trip
const jsonText = '{"name":"John","emoji":"🎉","tags":["a","b"]}';
const jsonTokens = tokenizer.encode(jsonText);
const jsonDecoded = tokenizer.decode(jsonTokens);
console.log(jsonDecoded === jsonText);
// Output: true

// Verify token counts match tiktoken
console.log("Unicode tokens:", unicodeTokens.length);   // 13
console.log("Code tokens:", codeTokens.length);         // 26
console.log("JSON tokens:", jsonTokens.length);         // 19
```

--------------------------------

### Retrieve Token Count Result Types

Source: https://context7.com/coder/ai-tokenizer/llms.txt

Access detailed token count breakdowns for messages, tool calls, and tool definitions using the SDK's strongly-typed result objects.

```typescript
import Tokenizer, { models } from "ai-tokenizer";
import { count, type CountResult, type CountResultMessage } from "ai-tokenizer/sdk";
import * as encoding from "ai-tokenizer/encoding";
import type { ModelMessage } from "ai";
import { z } from "zod";

const model = models["anthropic/claude-sonnet-4.5"];
const tokenizer = new Tokenizer(encoding[model.encoding]);

const messages: ModelMessage[] = [
  { role: "user", content: "Analyze this data" },
  {
    role: "assistant",
    content: [
      { type: "text", text: "I'll analyze the data for you." },
      {
        type: "tool-call",
        toolCallId: "call_456",
        toolName: "analyzeData",
        input: { dataset: "sales_2024", metrics: ["revenue", "growth"] }
      }
    ]
  }
];

const tools = {
  analyzeData: {
    description: "Analyze a dataset with specified metrics",
    inputSchema: z.object({
      dataset: z.string(),
      metrics: z.array(z.string())
    })
  }
};

const result: CountResult<typeof tools> = count({ tokenizer, model, messages, tools });

// Access typed tool definitions
const analyzeDataTokens = result.tools.definitions.analyzeData;
console.log("analyzeData tool tokens:", {
  name: analyzeDataTokens.name,
  description: analyzeDataTokens.description,
  inputSchema: analyzeDataTokens.inputSchema
});
// Output: analyzeData tool tokens: { name: 1, description: 8, inputSchema: 24 }

// Filter messages by content type
const toolCallMessages = result.messages.filter(m =>
  m.content.some(c => c.type === "tool-call")
);
console.log("Messages with tool calls:", toolCallMessages.length);
// Output: Messages with tool calls: 1

// Find large tool results for potential truncation
const largeToolResults = result.messages
  .flatMap(m => m.content)
  .filter(c => c.type === "tool-result" && c.output > 1000);
console.log("Large tool results:", largeToolResults.length);
// Output: Large tool results: 0
```

=== COMPLETE CONTENT === This response contains all available snippets from this library. No additional content exists. Do not make further requests.