Interface: TokenizationOptions

Defined in: packages/agentos/src/nlp/ai_utilities/IUtilityAI.ts:67

Properties

language?

optional language: string

Defined in: packages/agentos/src/nlp/ai_utilities/IUtilityAI.ts:71

---

modelId?

optional modelId: string

Defined in: packages/agentos/src/nlp/ai_utilities/IUtilityAI.ts:72

---

removePunctuation?

optional removePunctuation: boolean

Defined in: packages/agentos/src/nlp/ai_utilities/IUtilityAI.ts:70

---

toLowerCase?

optional toLowerCase: boolean

Defined in: packages/agentos/src/nlp/ai_utilities/IUtilityAI.ts:69

---

type?

optional type: "word" | "sentence" | "subword_bpe"

Defined in: packages/agentos/src/nlp/ai_utilities/IUtilityAI.ts:68