#@title from IPython.display import HTML HTML('') ! pip install datasets transformers[sentencepiece] from transformers import pipeline token_classifier = pipeline("token-classification") token_classifier("My name is Sylvain and I work at Hugging Face in Brooklyn.") token_classifier = pipeline("token-classification", aggregation_strategy="simple") token_classifier("My name is Sylvain and I work at Hugging Face in Brooklyn.") from transformers import AutoTokenizer, AutoModelForTokenClassification model_checkpoint = "" tokenizer = AutoTokenizer.from_pretrained(model_checkpoint) model = AutoModelForTokenClassification.from_pretrained(model_checkpoint) example = "My name is Sylvain and I work at Hugging Face in Brooklyn." inputs = tokenizer(example, return_tensors="pt") outputs = model(**inputs) print(inputs["input_ids"].shape) print(outputs.logits.shape) import torch probabilities = torch.nn.functional.softmax(outputs.logits, dim=-1)[0].tolist() predictions = probabilities.argmax(dim=-1)[0].tolist() print(predictions) model.config.id2label results = [] inputs_with_offsets = tokenizer(example, return_offsets_mapping=True) tokens = inputs_with_offsets.tokens() offsets = inputs_with_offsets["offset_mapping"] for idx, pred in enumerate(predictions): label = model.config.id2label[pred] if label != "O": start, end = offsets[idx] results.append( {"entity": label, "score": probabilities[idx][pred], "word": tokens[idx], "start": start, "end": end} ) print(results) import numpy as np label_map = model.config.id2label results = [] idx = 0 while idx < len(predictions): pred = predictions[idx] label = label_map[pred] if label != "O": # Remove the B- or I- label = label[2:] start, _ = offsets[idx] # Grab all the tokens labeled with I-label all_scores = [] while idx < len(predictions) and label_map[predictions[idx]] == f"I-{label}": all_scores.append(probabilities[idx][pred]) _, end = offsets[idx] idx += 1 # The score is the mean of all the scores of the token in that grouped entity. score = np.mean(all_scores).item() word = example[start:end] results.append( {"entity_group": label, "score": score, "word": word, "start": start, "end": end} ) idx += 1