What is Tokenization Drift and How to Fix It?

words     = [p[1] for p in pairs]
ids_ws    = [tokenizer.encode(" " + w,  add_special_tokens=False)[0] for w in words]
ids_nws   = [tokenizer.encode(w, add_special_tokens=False)[0] for w in words]
delta     = [abs(a - b) for a, b in zip(ids_ws, ids_nws)]
 
x = np.arange(len(words))
width = 0.35
 
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
fig.patch.set_facecolor("#FAFAF8")
 
# Left: side-by-side token IDs
ax = axes[0]
ax.set_facecolor("#FAFAF8")
bars1 = ax.bar(x - width/2, ids_ws,  width, label="With leading space",    color="#3B6FE0", alpha=0.85)
bars2 = ax.bar(x + width/2, ids_nws, width, label="Without leading space",  color="#E05C3B", alpha=0.85)
ax.set_xticks(x)
ax.set_xticklabels(words, rotation=30, ha="right", fontsize=9)
ax.set_ylabel("Token ID", fontsize=10)
ax.set_title("Token IDs: ' word'  vs  'word'", fontsize=12, fontweight="bold", pad=12)
ax.legend(fontsize=9)
ax.spines[["top", "right"]].set_visible(False)
ax.grid(axis="y", alpha=0.3)
 
for bar in bars1:
    ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 50,
            str(int(bar.get_height())), ha="center", va="bottom", fontsize=7, color="#3B6FE0")
for bar in bars2:
    ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 50,
            str(int(bar.get_height())), ha="center", va="bottom", fontsize=7, color="#E05C3B")
 
# Right: delta
ax2 = axes[1]
ax2.set_facecolor("#FAFAF8")
color_bars = ["#E05C3B" if d > 500 else "#F0A070" if d > 100 else "#A8C4F0" for d in delta]
bars3 = ax2.bar(words, delta, color=color_bars, alpha=0.9)
ax2.set_ylabel("Absolute Token ID Distance", fontsize=10)
ax2.set_title("How Far Apart Are the Token IDs?", fontsize=12, fontweight="bold", pad=12)
ax2.set_xticklabels(words, rotation=30, ha="right", fontsize=9)
ax2.spines[["top", "right"]].set_visible(False)
ax2.grid(axis="y", alpha=0.3)
 
for bar, d in zip(bars3, delta):
    ax2.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 10,
             str(d), ha="center", va="bottom", fontsize=9, fontweight="bold")
 
high  = mpatches.Patch(color="#E05C3B", alpha=0.9, label="> 500 apart")
med   = mpatches.Patch(color="#F0A070", alpha=0.9, label="100-500 apart")
low   = mpatches.Patch(color="#A8C4F0", alpha=0.9, label="< 100 apart")
ax2.legend(handles=[high, med, low], fontsize=8)
 
plt.tight_layout(pad=2)
plt.suptitle("Tokenization Artifacts: One Space, Completely Different Token", 
             fontsize=14, fontweight="bold", y=1.02)
plt.savefig("tokenization_artifact.png", dpi=150, bbox_inches="tight", facecolor="#FAFAF8")
plt.show()