import numpy as np
import matplotlib.pyplot as plt
 
# matplotlib defaults
import matplotlib_inline.backend_inline
matplotlib_inline.backend_inline.set_matplotlib_formats('svg')

# need to install the tiktoken library to get OpenAI's tokenizer
# note: it's tik-token, not tiktok-en :P
!pip install tiktoken
import tiktoken

# GPT-4's tokenizer
tokenizer = tiktoken.get_encoding('cl100k_base')
dir(tokenizer)

# get help
tokenizer??

# vocab size
tokenizer.n_vocab

tokenizer.decode([tokenizer.eot_token])

# but not all tokens are valid, e.g.,
print(tokenizer.n_vocab)
tokenizer.decode([100261])

# list of all tokens:
# https://github.com/vnglst/gpt4-tokens/blob/main/decode-tokens.ipynb

for i in range(1000,1050):
  print(f'{i} = {tokenizer.decode([i])}')

text = "My name is Mike and I like toothpaste-flavored chocolate."
tokens = tokenizer.encode(text)
print(tokens)

text.split()

for word in text.split():
  print(f'"{word}" comprises token(s) {tokenizer.encode(word)}')

for t in tokens:
  print(f'Token {t:>6} is "{tokenizer.decode([t])}"')

# with special (non-ASCII) characters
tokenizer.encode('â')

# initialize lengths vector
token_lengths = np.zeros(tokenizer.n_vocab)
 
# get the number of characters in each token
for idx in range(tokenizer.n_vocab):
  try:
    token_lengths[idx] = len(tokenizer.decode([idx]))
  except:
    token_lengths[idx] = np.nan
 
# count unique lengths
uniqueLengths,tokenCount = np.unique(token_lengths,return_counts=True)
 
 
 
# visualize
_,axs = plt.subplots(1,2,figsize=(12,4))
axs[0].plot(token_lengths,'k.',markersize=3,alpha=.4)
axs[0].set(xlim=[0,tokenizer.n_vocab],xlabel='Token index',ylabel='Token length (characters)',
           title='GPT4 token lengths')
 
axs[1].bar(uniqueLengths,tokenCount,color='k',edgecolor='gray')
axs[1].set(xlim=[0,max(uniqueLengths)],xlabel='Token length (chars)',ylabel='Token count (log scale)',
           title='Distribution of token lengths')
 
plt.tight_layout()
plt.show()

# single-token words with vs. without spaces
print( tokenizer.encode(' Michael') )
print( tokenizer.encode('Michael') )

# multi-token words without a space
print( tokenizer.encode(' Peach') )
print( tokenizer.encode('Peach') )

peach = tokenizer.encode('Peach')
[tokenizer.decode([p]) for p in peach]

import requests
import re
text = requests.get('https://www.gutenberg.org/files/35/35-0.txt').text
 
# split by punctuation
words = re.split(r'([,.:;—?_!"“()\']|--|\s)',text)
words = [item.strip() for item in words if item.strip()]
print(f'There are {len(words)} words.')
words[10000:10050]

# tokens of a random word in the text
someRandomWord = np.random.choice(words)
print(f'"{someRandomWord}" has token {tokenizer.encode(someRandomWord)}')

for t in words[:20]:
  print(f'"{t}" has {len(tokenizer.encode(t))} tokens')

for spelling in ['book','Book','bOok']:
  print(f'"{spelling}" has tokens {tokenizer.encode(spelling)}')

# what happens if we just tokenize the raw (unprocessed) text?
tmTokens = tokenizer.encode(text)
print(f'The text has {len(tmTokens):,} tokens and {len(words):,} words.')

# check out some tokens
 
for t in tmTokens[9990:10020]:
  print(f'Token {t:>6}: "{tokenizer.decode([t])}"')

print(tokenizer.decode(tmTokens[9990:10020]))