Install the Transformers, Datasets, and Evaluate libraries to run this notebook.
!pip install datasets evaluate transformers[sentencepiece]
!apt install git-lfs
You will need to setup git, adapt your email and name in the following cell.
!git config --global user.email "you@example.com"
!git config --global user.name "Your Name"
You will also need to be logged in to the Hugging Face Hub. Execute the following and enter your credentials.
from huggingface_hub import notebook_login
notebook_login()
from huggingface_hub import notebook_login
notebook_login()
from transformers import PushToHubCallback
callback = PushToHubCallback(
"bert-finetuned-mrpc", save_strategy="epoch", tokenizer=tokenizer
)
from transformers import TFAutoModelForMaskedLM, AutoTokenizer
checkpoint = "camembert-base"
model = TFAutoModelForMaskedLM.from_pretrained(checkpoint)
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model.push_to_hub("dummy-model")
tokenizer.push_to_hub("dummy-model")
tokenizer.push_to_hub("dummy-model", organization="huggingface")
tokenizer.push_to_hub("dummy-model", organization="huggingface", use_auth_token="<TOKEN>")
from huggingface_hub import (
# Gestão de usuários
login,
logout,
whoami,
# Criação e gestão de repositório
create_repo,
delete_repo,
update_repo_visibility,
#E alguns métodos para recuperar/trocar informações sobre o conteúdo
list_models,
list_datasets,
list_metrics,
list_repo_files,
upload_file,
delete_file,
)
from huggingface_hub import create_repo
create_repo("dummy-model")
from huggingface_hub import create_repo
create_repo("dummy-model", organization="huggingface")
from huggingface_hub import upload_file
upload_file(
"<path_to_file>/config.json",
path_in_repo="config.json",
repo_id="<namespace>/dummy-model",
)
from huggingface_hub import Repository
repo = Repository("<path_to_dummy_folder>", clone_from="<namespace>/dummy-model")
repo.git_pull()
repo.git_add()
repo.git_commit()
repo.git_push()
repo.git_tag()
repo.git_pull()
model.save_pretrained("<path_to_dummy_folder>")
tokenizer.save_pretrained("<path_to_dummy_folder>")
repo.git_add()
repo.git_commit("Add model and tokenizer files")
repo.git_push()
from transformers import TFAutoModelForMaskedLM, AutoTokenizer
checkpoint = "camembert-base"
model = TFAutoModelForMaskedLM.from_pretrained(checkpoint)
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
# Do whatever with the model, train it, fine-tune it...
model.save_pretrained("<path_to_dummy_folder>")
tokenizer.save_pretrained("<path_to_dummy_folder>")