class Tiktoken::Encoding

Defined in:

tiktoken/encoding.cr

Class Method Summary

Instance Method Summary

Class Method Detail

def self.cl100k_base #

[View source]
def self.for_model(model_name : String) #

[View source]
def self.o200k_base #

[View source]
def self.o200k_harmony #

[View source]
def self.p50k_base #

[View source]
def self.p50k_edit #

[View source]
def self.r50k_base #

[View source]

Instance Method Detail

def decode(tokens) : String #

[View source]
def encode(text : String, allowed_special : Set(String) = Set(String).new) #

[View source]
def encode_ordinary(text : String) #

[View source]
def encode_with_special_tokens(text : String) #

[View source]
def finalize #

[View source]