我跟着这个git https://github.com/ajmanser/Yelp一切正常，但是当我尝试从头开始训练模型时，我坚持使用step2：
Use the json_converter.py script on the business and review datasets to convert them into csv files. This script requires Python version 2 and simple json (I took this from another repo and made a few quick attempts to get it working with Python 3, but it was becoming a bottleneck for me and it works fine if you use Python 2 + pip2 install simplejson).
，使用我遇到此错误的脚本将我的json转换为csv。我不知道问题是什么。
Traceback (most recent call last):
File "json_converter.py", line 115, in <module>
column_names = get_superset_of_column_names_from_file(json_file)
File "json_converter.py", line 28, in get_superset_of_column_names_from_file
line_contents = json.loads(line)
File "D:\Python27\lib\site-packages\simplejson\__init__.py", line 518, in loads
return _default_decoder.decode(s)
File "D:\Python27\lib\site-packages\simplejson\decoder.py", line 370, in decode
obj, end = self.raw_decode(s)
File "D:\Python27\lib\site-packages\simplejson\decoder.py", line 400, in raw_decode
return self.scan_once(s, idx=_w(s, idx).end())
File "D:\Python27\lib\site-packages\simplejson\scanner.py", line 79, in scan_once
return _scan_once(string, idx)
File "D:\Python27\lib\site-packages\simplejson\scanner.py", line 70, in _scan_once
raise JSONDecodeError(errmsg, string, idx)
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
测试json
[
{
"review_id": "1",
"business_id": "1",
"stars": 5,
"text" : "It was so much fun to read that I readed it again"
},
{
"review_id": "2",
"business_id": "1",
"stars": 5,
"text" : "A classic How can you not like this one? The characters are very memorable, and we all liked it."
},
{
"review_id": "3",
"business_id": "2",
"stars": 5,
"text" : " pretty nice story. and very interesting characters"
},
{
"review_id": "4",
"business_id": "1",
"stars": 5,
"text" : "Awesome! for children and a time travel for elders, really a simple language and beautiful descriptions makes the work very interesting."
},
{
"review_id": "5",
"business_id": "1",
"stars": 5,
"text" : "A fascinating read for anyone who would think to breed a horse for want of a another for whatever purpose that desired and so realize that the blood line means little if the sire or dame should not be suited for breeding purposes in case they should pass on unwanted traits"
},
{
"review_id": "6",
"business_id": "1",
"stars": 5,
"text" : "The Arabian Nights I read when I was young were like disney-fied. I'm excited to read the real version of the tales."
},
{
"review_id": "7",
"business_id": "2",
"stars": 5,
"text" : "Just a string of short boring stories. It looks like some Sindbad is also in there, but I got bored before I got to it."
}
]
我还从yelp下载了一个数据集，在github中这是他们使用的数据
代码转换器
# -*- coding: utf-8 -*-
#!/usr/bin/python2
"""Convert the Yelp Dataset Challenge dataset from json format to csv.
For more information on the Yelp Dataset Challenge please visit http://yelp.com/dataset_challenge
"""
import argparse
import collections
import csv
import simplejson as json
def read_and_write_file(json_file_path, csv_file_path, column_names):
"""Read in the json dataset file and write it out to a csv file, given the column names."""
with open(csv_file_path, 'wb+') as fout:
csv_file = csv.writer(fout)
csv_file.writerow(list(column_names))
with open(json_file_path) as fin:
for line in fin:
line_contents = json.loads(line)
csv_file.writerow(get_row(line_contents, column_names))
def get_superset_of_column_names_from_file(json_file_path):
"""Read in the json dataset file and return the superset of column names."""
column_names = set()
with open(json_file_path) as fin:
for line in fin:
line_contents = json.loads(line)
column_names.update(
set(get_column_names(line_contents).keys())
)
return column_names
def get_column_names(line_contents, parent_key=''):
"""Return a list of flattened key names given a dict.
Example:
line_contents = {
'a': {
'b': 2,
'c': 3,
},
}
will return: ['a.b', 'a.c']
These will be the column names for the eventual csv file.
"""
column_names = []
for k, v in line_contents.iteritems():
column_name = "{0}.{1}".format(parent_key, k) if parent_key else k
if isinstance(v, collections.MutableMapping):
column_names.extend(
get_column_names(v, column_name).items()
)
else:
column_names.append((column_name, v))
return dict(column_names)
def get_nested_value(d, key):
"""Return a dictionary item given a dictionary `d` and a flattened key from `get_column_names`.
Example:
d = {
'a': {
'b': 2,
'c': 3,
},
}
key = 'a.b'
will return: 2
"""
if '.' not in key:
if key not in d:
return None
return d[key]
base_key, sub_key = key.split('.', 1)
if base_key not in d:
return None
sub_dict = d[base_key]
return get_nested_value(sub_dict, sub_key)
def get_row(line_contents, column_names):
"""Return a csv compatible row given column names and a dict."""
row = []
for column_name in column_names:
line_value = get_nested_value(
line_contents,
column_name,
)
if isinstance(line_value, unicode):
row.append('{0}'.format(line_value.encode('utf-8')))
elif line_value is not None:
row.append('{0}'.format(line_value))
else:
row.append('')
return row
if __name__ == '__main__':
"""Convert a yelp dataset file from json to csv."""
parser = argparse.ArgumentParser(
description='Convert Yelp Dataset Challenge data from JSON format to CSV.',
)
parser.add_argument(
'json_file',
type=str,
help='The json file to convert.',
)
args = parser.parse_args()
json_file = args.json_file
csv_file = '{0}.csv'.format(json_file.split('.json')[0])
column_names = get_superset_of_column_names_from_file(json_file)
read_and_write_file(json_file, csv_file, column_names)
您正在将每行文件发送到json.loads，这会导致错误。
json.loads()期望整个json字符串，所以你必须使用fin.read()使用整个文件内容并将其发送到json.loads()，见下面的解决方案：
def get_superset_of_column_names_from_file(json_file_path):
"""Read in the json dataset file and return the superset of column names."""
column_names = set()
with open(json_file_path) as fin:
line_contents = json.loads(fin.read())
column_names.update(
set(get_column_names(line_contents).keys())
)
return column_names