|
|
@@ -418,9 +418,11 @@ def crawl_spider(
|
|
|
break
|
|
|
else:
|
|
|
'''记录采集页码,已记录页码不在访问'''
|
|
|
- finished_pages = CRAWL_RECORDS[category]['pages']
|
|
|
+ _records = CRAWL_RECORDS[category]
|
|
|
+ finished_pages = _records['pages']
|
|
|
finished_pages.append(str(page_num))
|
|
|
- CRAWL_RECORDS.update({category: finished_pages})
|
|
|
+ _records.update({'pages': finished_pages})
|
|
|
+ CRAWL_RECORDS.update({category: _records})
|
|
|
else:
|
|
|
break
|
|
|
|
萤火也是火