update

ybw/crawler/crawl_record.py

@@ -1,99 +0,0 @@
-import datetime
-import json
-import threading
-from pathlib import Path
-
-LOCK = threading.RLock()
-ROOT_PATH = Path(__file__).parent.parent
-CRAWL_RECORD = (ROOT_PATH / 'crawl_records.json').resolve()
-
-
-def _open_file():
-    try:
-        fp = open(CRAWL_RECORD, encoding='utf-8')
-    except FileNotFoundError:
-        fp = open(CRAWL_RECORD, 'w+', encoding='utf-8')
-    return fp
-
-
-def load_records(account: str, date_str: str):
-    """
-    读取当天详情页采集数量
-
-    Args:
-        account: 账号
-        date_str: 采集日期 ‘YYYY-mm-dd‘
-
-    Returns:
-        采集数量
-    """
-    fp = _open_file()
-    try:
-        records: dict = json.load(fp)
-        return int(records.get(account, {}).get(date_str, '0'))
-    except json.decoder.JSONDecodeError:
-        return 0
-
-
-def reset_records(account: str, date_str=None):
-    """
-    重置当天采集记录
-
-    Args:
-        account: 账号
-        date_str: 采集日期 ‘YYYY-mm-dd‘
-
-    """
-    if date_str is None:
-        today = datetime.datetime.today().strftime('%Y-%m-%d')
-    else:
-        today = date_str
-
-    with LOCK:
-        fp = _open_file()
-        try:
-            records: dict = json.load(fp)
-            records[account][today] = '0'
-        except json.decoder.JSONDecodeError:
-            records = {account: {today: '0'}}
-
-        wp = open(CRAWL_RECORD, 'w+', encoding='utf-8')
-        wp.write(json.dumps(records, indent=4))
-        wp.close()
-        fp.close()
-    # print(f"{today}采集记录重置完成")
-
-
-def update_records(account: str, number: int):
-    """
-    更新详情页采集数量记录
-
-    Args:
-        account: 账号
-        number: 采集数量
-
-    """
-    today = datetime.datetime.today().strftime('%Y-%m-%d')
-    with LOCK:
-        fp = _open_file()
-        try:
-            records: dict = json.load(fp)
-        except json.decoder.JSONDecodeError:
-            records = {account: {today: '0'}}
-
-        if account not in records:
-            info = {today: str(number)}
-        else:
-            info: dict = records.get(account)
-            record = int(info.get(today, '0'))
-            if len(info) > 7:
-                '''采集记录保存7天'''
-                info = {today: str(record)}
-            record += number
-            info.update({today: str(record)})
-
-        records.update({account: info})
-        wp = open(CRAWL_RECORD, 'w+', encoding='utf-8')
-        wp.write(json.dumps(records, indent=4))
-        wp.close()
-        fp.close()