我使用 spaCy 从事文本分类项目。我非常关注 spaCy 代码示例。唯一重要的区别是我在示例中使用了两个类别而不是一个类别。我不明白哪里出了问题，因为我检查过并且我正在加载的数据的格式与原始示例中的格式相同。这是相关代码(下面附上完整代码):
def load_data(limit=0, split=0.8):
"""Load the patents data."""
# Partition off part of the train data for evaluation
temp=pd.read_csv(excel + 'patents_text_class.csv',header = None)
new_cols = ['id' , 'class' , 'patent_text']
temp.columns = new_cols
print(temp)
train_data = list(zip(temp["patent_text"], temp["class"]))
random.shuffle(train_data)
train_data = train_data[-limit:]
texts, labels = zip(*train_data)
cats = [{"A01D": bool(y) , "A01B": operator.not_(bool(y))} for y in labels]
split = int(len(train_data) * split)
return (texts[:split], cats[:split]), (texts[split:], cats[split:])
这是日志:
Loaded model 'en_core_web_lg'
Loading patents data...
id class patent_text
0 1317 0 Improvement n revolving harrows <div itemprop=...
1 2476 1 Machine for cutting meat and other substances ...
2 2650 0 Improvement in cultivators fob vines <div item...
3 3311 0 Improvement in plows <div itemprop="content" h...
4 4544 0 Improvement in plow-clevises <div itemprop="co...
5 7277 1 Improvement in machines for raking and loading...
6 8721 0 Improvement in shovel-plows <div itemprop="con...
7 8844 0 Improvement in gang-plows <div itemprop="conte...
8 9069 0 Improvement in potato-diggers and stone-gather...
9 10624 0 Improvement in rotary cultivators <div itempro...
10 12057 0 Improvement in hoes <div itemprop="content" ht...
[70000 rows x 3 columns]
Using 10000 examples (8000 training, 2000 evaluation)
Training the model...
LOSS P R F
Traceback (most recent call last):
File "process/task_classification.py", line 150, in <module>
plac.call(main)
File "/anaconda/lib/python3.6/site-packages/plac_core.py", line 328, in call
cmd, result = parser.consume(arglist)
File "/anaconda/lib/python3.6/site-packages/plac_core.py", line 207, in consume
return cmd, self.func(*(args + varargs + extraopts), **kwargs)
File "process/task_classification.py", line 78, in main
losses=losses)
File "/anaconda/lib/python3.6/site-packages/spacy/language.py", line 405, in update
gold = GoldParse(doc, **gold)
File "gold.pyx", line 409, in spacy.gold.GoldParse.__init__
TypeError: 'float' object is not iterable
知道我为什么会收到此错误吗？
完整代码供引用:
#!/usr/bin/env python
# coding: utf8
"""Train a convolutional neural network text classifier on the
IMDB dataset, using the TextCategorizer component. The dataset will be loaded
automatically via Thinc's built-in dataset loader. The model is added to
spacy.pipeline, and predictions are available via `doc.cats`. For more details,
see the documentation:
* Training: https://spacy.io/usage/training
Compatible with: spaCy v2.0.0+
"""
from __future__ import unicode_literals, print_function
import plac
import random
from pathlib import Path
import thinc.extra.datasets
import os
import pandas as pd
import operator
import spacy
from spacy.util import minibatch, compounding
root = 'path/to/folder'
output = root + 'output/'
process = root + 'process/'
excel = root + 'excel/'
@plac.annotations(
model=("Model name. Defaults to blank 'en' model.", "option", "m", str),
output_dir=("Optional output directory", "option", "o", Path),
n_texts=("Number of texts to train from", "option", "t", int),
n_iter=("Number of training iterations", "option", "n", int))
def main(model='en_core_web_lg', output_dir=output, n_iter=5, n_texts=10000):
if output_dir is not None:
output_dir = Path(output_dir)
if not output_dir.exists():
output_dir.mkdir()
if model is not None:
nlp = spacy.load(model) # load existing spaCy model
print("Loaded model '%s'" % model)
else:
nlp = spacy.blank('en') # create blank Language class
print("Created blank 'en' model")
# add the text classifier to the pipeline if it doesn't exist
# nlp.create_pipe works for built-ins that are registered with spaCy
if 'textcat' not in nlp.pipe_names:
textcat = nlp.create_pipe('textcat')
nlp.add_pipe(textcat, last=True)
# otherwise, get it, so we can add labels to it
else:
textcat = nlp.get_pipe('textcat')
# add label to text classifier
textcat.add_label("A01B")
textcat.add_label("A01D")
# load the patents dataset
print("Loading patents data...")
(train_texts, train_cats), (dev_texts, dev_cats) = load_data(limit=n_texts)
print("Using {} examples ({} training, {} evaluation)"
.format(n_texts, len(train_texts), len(dev_texts)))
train_data = list(zip(train_texts,
[{'cats': cats} for cats in train_cats]))
# get names of other pipes to disable them during training
other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'textcat']
with nlp.disable_pipes(*other_pipes): # only train textcat
optimizer = nlp.begin_training()
print("Training the model...")
print('{:^5}\t{:^5}\t{:^5}\t{:^5}'.format('LOSS', 'P', 'R', 'F'))
for i in range(n_iter):
losses = {}
# batch up the examples using spaCy's minibatch
batches = minibatch(train_data, size=compounding(4., 32., 1.001))
for batch in batches:
texts, annotations = zip(*batch)
nlp.update(texts, annotations, sgd=optimizer, drop=0.2,
losses=losses)
with textcat.model.use_params(optimizer.averages):
# evaluate on the dev data split off in load_data()
scores = evaluate(nlp.tokenizer, textcat, dev_texts, dev_cats)
print('{0:.3f}\t{1:.3f}\t{2:.3f}\t{3:.3f}' # print a simple table
.format(losses['textcat'], scores['textcat_p'],
scores['textcat_r'], scores['textcat_f']))
# test the trained model
test_text = "Harvesting"
doc = nlp(test_text)
print(test_text, doc.cats)
test_text = "Plowing"
doc = nlp(test_text)
print(test_text, doc.cats)
if output_dir is not None:
with nlp.use_params(optimizer.averages):
nlp.to_disk(output_dir)
print("Saved model to", output_dir)
# test the saved model
print("Loading from", output_dir)
nlp2 = spacy.load(output_dir)
doc2 = nlp2(test_text)
print(test_text, doc2.cats)
def load_data(limit=0, split=0.8):
"""Load the patents data."""
# Partition off part of the train data for evaluation
temp=pd.read_csv(excel + 'patents_text_class.csv',header = None)
new_cols = ['id' , 'class' , 'patent_text']
temp.columns = new_cols
train_data = list(zip(temp["patent_text"], temp["class"]))
random.shuffle(train_data)
train_data = train_data[-limit:]
texts, labels = zip(*train_data)
cats = [{"A01D": bool(y) , "A01B": operator.not_(bool(y))} for y in labels]
split = int(len(train_data) * split)
return (texts[:split], cats[:split]), (texts[split:], cats[split:])
def evaluate(tokenizer, textcat, texts, cats):
docs = (tokenizer(text) for text in texts)
tp = 0.0 # True positives
fp = 1e-8 # False positives
fn = 1e-8 # False negatives
tn = 0.0 # True negatives
for i, doc in enumerate(textcat.pipe(docs)):
gold = cats[i]
print(i)
for label, score in doc.cats.items():
if label not in gold:
continue
if score >= 0.5 and gold[label] >= 0.5:
tp += 1.
elif score >= 0.5 and gold[label] < 0.5:
fp += 1.
elif score < 0.5 and gold[label] < 0.5:
tn += 1
elif score < 0.5 and gold[label] >= 0.5:
fn += 1
precision = tp / (tp + fp)
recall = tp / (tp + fn)
f_score = 2 * (precision * recall) / (precision + recall)
return {'textcat_p': precision, 'textcat_r': recall, 'textcat_f': f_score}
if __name__ == '__main__':
plac.call(main)
最佳答案
根据文档，Language.update 的第一个参数接受一批 unicode 或 Doc。 Probalby texts 包含一些 NaN 值，其类型为 float。相关代码:
batches = minibatch(train_data, size=compounding(4., 32., 1.001))
for batch in batches:
texts, annotations = zip(*batch) # check texts for NaN
nlp.update(texts, annotations, sgd=optimizer, drop=0.2,
losses=losses)
spacy 尝试迭代一个 NaN( float )，它会导致一个
...
TypeError: 'float' object is not iterable
- 因此，您可以删除所有 NaN 值或用空字符串替换它们。
另外，这种错误在 NLP(但不仅限于 NLP)任务中非常常见。始终检查 NaN 的文本数据并替换它们，尤其是当您收到类似的错误消息时。
关于python - Spacy 文本分类 : getting the error massage "' float' object is not iterable"，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/54354431/