feat(megatron): support sentencepiece

dmarx · dmarx · commit 157f68c380d3 · 2024-10-18T11:55:49.000-07:00
diff --git a/megatron/requirements.txt b/megatron/requirements.txt
@@ -7,3 +7,4 @@ tensorboard-data-server==0.7.2
 transformers==4.45.2
 triton==3.0.0
 wandb==0.18.3
+sentencepiece==0.2.0
