topic_spider/find_source/crawler/Task.py

import time
from collections import UserDict

from common.databases import int2long


class Task(UserDict):

    """
    # >>> import json
    # >>> task = Task(name='剑鱼')
    # >>> task_str = json.dumps(task, default=lambda obj: obj.__dict__['data'], ensure_ascii=False)
    # >>> task_obj = json.loads(task_str, object_hook=lambda d: Task(**d))
    """

    def __init__(
            self,
            name='',
            url='',
            domain='',
            base_url='',
            search='',
            classify='',
            groups='',
            origin='',
            weight=1,
            depth=None,
            status_code=None,
            err_reason='',
            create_at=int2long(int(time.time())),
            update_at=int2long(int(time.time())),
    ):
        super(Task, self).__init__(
            name=name,
            url=url,
            domain=domain,
            base_url=base_url,
            search=search,
            classify=classify,
            groups=groups,
            origin=origin,
            weight=weight,
            status_code=status_code,
            depth=depth,
            err_reason=err_reason,
            create_at=create_at,
            update_at=update_at,
        )

    def _update_at(self):
        super(Task, self).__setitem__('update_at', int2long(int(time.time())))

    def __setitem__(self, key, value):
        self._update_at()
        super(Task, self).__setitem__(str(key), value)

    def __getitem__(self, item):
        self._update_at()
        return super(Task, self).__getitem__(item)

    @property
    def retrieve_task_to_dict(self):
        item = {
            'name': self.data['name'],
            'url': self.data['url'],
            'domain': self.data['domain'],
            'origin': self.data['origin'],
            'groups': self.data['groups'],
            'create_at': self.data['create_at'],
            'update_at': self.data['update_at'],
        }
        return item

    @property
    def excavate_task_to_dict(self):
        item = {
            'name': self.data['name'],
            'url': self.data['url'],
            'domain': self.data['domain'],
            'depth': self.data['depth'],
            'origin': self.data['origin'],
            'groups': self.data['groups'],
            'create_at': self.data['create_at'],
            'update_at': self.data['update_at'],
        }
        return item

    @property
    def validate_task_to_dict(self):
        item = {
            'domain': self.data['domain'],
            'origin': self.data['origin'],
            'create_at': self.data['update_at'],
        }
        return item