topic_spider/find_source/crawler/services/channel.py

import json
import os
import pathlib
from urllib.parse import urljoin

from lxml.html import etree, HtmlElement

from common.tools import sha1, detect_encoding
from crawler.analysis import TimeExtractor
from crawler.defaults import PAGE_TEXTS, LOGIN_TEXTS, NAV_TEXTS
from crawler.download import Downloader
from crawler.utils import (
    element2html,
    html2element,
    iter_node,
    drop_tag,
    remove_node,
    pre_parse,
    is_empty_element,
    check_text_by_words,
)

_base_path = pathlib.Path(__file__).parent


def analysis(origin_lst, target_lst):
    results = []
    for target_ in target_lst:
        source: str = target_['contenthtml']
        _c = 0
        for item in origin_lst:
            href, channel = item['href'], item['channel']
            if source.count(channel) > 0 or source.count(href) > 0:
                _c += 1

        if _c > 0:
            results.append({
                'similarity': _c,
                'contenthtml': source,
                'depth': target_['depth']
            })

    results = sorted(results, key=lambda x: x['similarity'], reverse=True)
    _t = max(results, key=lambda dic: dic['depth'])
    return _t


def extract_text(node: HtmlElement):
    text = (node.text or node.tail or node.xpath('string(.)'))
    return "".join(f"{text}".split())


def tag_date_total(node: HtmlElement, tag=None):
    count = 0
    # 先辈节点与目标节点名称相同并且包含时间文本的个数
    contains_date_nodes = []
    if tag is not None:
        descendants = list(node.iterdescendants(tag))
    else:
        descendants = list(node.iterdescendants())

    for descendant in descendants:
        pt = TimeExtractor().extractor(descendant)
        children = len(list(descendant.iterchildren())) > 0
        if pt != '' and not children and descendant not in contains_date_nodes:
            contains_date_nodes.append(descendant)
            count += 1
    return count


def remove_ancestors_date_tag(node: HtmlElement):
    prev_node = node
    # 情况1: 先辈节点下直接放置全部的时间节点，直接删除
    remove_count = 0
    for ancestor in node.iterancestors():
        if ancestor.tag.lower() in ['body', 'html']:
            continue
        is_remove = False
        total = tag_date_total(ancestor)
        # print("ancestors_date_tag >>> ", ancestor.tag, ancestor.attrib, total)
        if total > 3:
            # 情况2: ancestor节点下面不直接放置全部的时间节点；
            # 首先查询出时间文本大于3及以上的节点，此时直接删除ancestor节点，会导致
            # ancestor节点边界变大，包括不想期望保留节点一起被删除。此处遍历查询prev_node
            # 兄弟节点（ancestor后裔节点）寻找时间文本大于3及以上的兄弟节点，若不存
            # 时间文本大于3及以上的情况，此时再处理ancestor节点，达到增大边界效果，删除即可
            for sibling in prev_node.itersiblings():
                sibling_tag_date_total = tag_date_total(sibling)
                if sibling_tag_date_total > 3:
                    remove_node(sibling)
                    is_remove = True
                    # print("remove sibling tag >>> ", sibling.tag, sibling.attrib, )
            # 查询出时间文本大于3及以上，若prev_node的没有兄弟节点，此时直接删除该ancestor
            if not is_remove:
                remove_node(ancestor)
                # print("remove ancestor tag >>> ", ancestor.tag, ancestor.attrib, )
                is_remove = True
        elif 1 < total <= 2:
            # 逐条删除（查询出时间文本条数，从子节点边界范围进行查询、剔除，防止）
            for child in ancestor.iterchildren():
                child_tag_date_total = tag_date_total(child)
                if child_tag_date_total > 0:
                    remove_node(child)
                    # print("remove child tag >>> ", child.tag, child.attrib, )
                    is_remove = True
            if not is_remove:
                remove_node(ancestor)
                # print("remove ancestor tag >>> ", ancestor.tag, ancestor.attrib, )
                is_remove = True
        else:
            # 保存上一次查询的先辈节点（时间文本小于等于3）
            prev_node = ancestor

        if is_remove:
            remove_count += 1

    return True if remove_count > 0 else False


def show_html(page, *, file=None, bash_url=None):
    if bash_url is None:
        bash_url = ''

    if isinstance(page, HtmlElement):
        source = element2html(page)
    elif isinstance(page, bytes):
        source = page.decode(detect_encoding(page), 'surrogatepass')
    else:
        source = page

    if file is None:
        dirname = 'html'
        os.makedirs(dirname, mode=0o777, exist_ok=True)
        file = f'{dirname}/{sha1(bash_url)}.html'
    else:
        file = file

    with open(file, 'w') as fp:
        fp.write(source)


def trim_node(element: HtmlElement):
    """
        整理节点 （body子节点更新为div,在更新过程中子节点会合并文本）

    :param element:
    :return:
    """
    children = element.xpath('/html/body/child::*')
    for child in children:
        for node, _ in iter_node(child):
            # print('trim_node >>> ', node.tag, node.attrib)
            if node.tag.lower() == 'div':
                break
            drop_tag(node)
    return element


def strip_node(element: HtmlElement):
    """
        剔除节点，若不是标签a且无时间文本,关键词标题,标签a个数,移除或者剔除

    :param element:
    :return:
    """
    for node, _ in iter_node(element):
        # 删除掉没有文本、发布时间、发布标题、href属性的节点及父节点
        if node.tag.lower() != 'a':
            # 节点文本（剔除左右空白、换行、回车符号）
            text = "".join("".join(node.xpath('./text()')).strip())
            # 关键词文本
            non_title = check_text_by_words(text) is False
            # 后裔a节点数量
            sub_tag_gt_0 = len(list(node.iterdescendants('a'))) == 0
            # 时间文本
            publish_time = TimeExtractor().extractor(node)
            # print('>>> ', node.tag, node.attrib, text)
            if non_title and sub_tag_gt_0 and publish_time == '':
                # print('strip_node >>> ', node.tag, node.attrib, text)
                parent = node.getparent()
                if parent is not None:
                    if parent.tag.lower() == 'a':
                        etree.strip_tags(parent, node.tag)
                    elif parent.tag.lower() == 'td':
                        if not node.getchildren():
                            if len(text) == 0:
                                remove_node(parent)
                            else:
                                etree.strip_tags(parent, node.tag)
                        else:
                            name = [child.tag for child in node.getchildren()]
                            etree.strip_tags(parent, *name)
                else:
                    remove_node(node)


def remove_nav_node(element: HtmlElement):
    for node, _ in iter_node(element):
        text = extract_text(node)
        # print('nav_node >>> ', node.tag, text)
        # （板块文本|页脚文本|导航文本）
        for item in NAV_TEXTS:
            if item in text:
                remove_node(node)
        # 登录标签
        if text in LOGIN_TEXTS:
            remove_node(node)
        # 翻页标签栏
        if text in PAGE_TEXTS:
            tag = node.tag.lower()
            siblings = list(node.itersiblings(tag))
            # 通过查询相同标签兄弟节点,兄弟节点包含在一个先辈节点内；翻页标签不包含在同一先辈节点
            if len(siblings) > 3 and node.tag.lower() == tag:
                remove_node(node.getparent())
            else:
                for ancestor in node.iterancestors():
                    # 先辈节点包含两个以上同类节点，则认为该先辈节点包含全部想要删除标签
                    if len(list(ancestor.iterdescendants(tag))) > 2:
                        remove_node(ancestor)
                        break
        # 位置标签
        if '位置' in text:
            remove_node(node)


def remove_date_node(element: HtmlElement):
    retrieve = False
    for node, _ in iter_node(element):
        if retrieve:
            break
        publish_time = TimeExtractor().extractor(node)
        # print('date_node >>> ', node.tag, node.attrib, publish_time)
        # 首先找拥有时间文本最深层级节点（拥有时间文本，不存在子节点）
        if publish_time != '' and len(list(node.iterchildren())) == 0:
            # print("date_node >>> ", node.tag, node.attrib, len(list(node.itersiblings())))
            # 时间文本有可能来自兄弟节点，而不是自身
            if len(list(node.itersiblings())) > 0:
                # 存在多个兄弟节点，所以通过分析共同的父节点
                parent = node.getparent()
                # 统计父节点的拥有时间文本的后裔节点个数
                total = tag_date_total(parent)
                if total > 3:
                    # 简单场景：仅仅单个兄弟节点拥有时间文本(发布时间)
                    remove_node(parent)
                else:
                    # 复杂场景：多个兄弟节点拥有时间文本(开始时间，截止时间...)
                    # print("parent_date_node >>> ", parent.tag, parent.attrib, len(list(parent.itersiblings())))
                    # 从父节点开始，查询且删除先辈节点中拥有时间文本的先辈节点
                    retrieve = remove_ancestors_date_tag(parent)
            else:
                # 情况2：无兄弟节点，从自身开始，查询且删除先辈节点中拥有时间文本的先辈节点
                retrieve = remove_ancestors_date_tag(node)


def clean_node(element):
    for node, _ in iter_node(element):
        if is_empty_element(node):
            # print(' clean_node >>> ', node.tag, node.attrib, extract_text(node))
            remove_node(node)

        if node.tag.lower() == 'a' and list(node.iterchildren()):
            # 剔除a标签包含的单个或多个短语标签、文本标签，保留内部文本
            for child in node.iterchildren():
                etree.strip_tags(node, child.tag)


def extract_data(source, base_url):
    element = html2element(source)
    children = element.xpath('/html/body/child::*')
    result = {}
    for index, child in enumerate(children):
        data = []
        for node in child.iterdescendants('a'):
            title = extract_text(node)
            href = urljoin(base_url, node.attrib.get('href'))
            if check_text_by_words(title) and len(title) <= 15:
                item = (title, href)
                data.append(item)
        key = "{}_{}".format(child.tag.lower(), index)
        result[key] = data

    print(result)
    for key, items in result.items():
        print(f"=============== {base_url} && {key} ===============")
        for val in items:
            print(val)
        print()
    return result


def process_page(source):
    element = html2element(source)
    # web网页预处理（web页面去噪，会改变原始dom结构）
    element = pre_parse(element)
    # show_html(element, file='2预处理.html')
    # 整理节点
    element = trim_node(element)
    # show_html(element, file='3整理body节点.html')
    # 剔除节点
    strip_node(element)
    # show_html(element, file='4剔除节点.html')
    # 删除导航节点
    remove_nav_node(element)
    # show_html(element, file='5删除导航条.html')
    # 删除时间节点
    remove_date_node(element)
    # show_html(element, file='6删除时间块.html')
    # 清理节点
    clean_node(element)
    # show_html(element, file='7清理dom.html')
    return element2html(element)


def bfs(response, base_url):
    try:
        source = response.json().get('html', '')
    except json.decoder.JSONDecodeError:
        source = response.text
    # show_html(source, file='1原始页.html')
    if len(source) == 0:
        return {}
    source = process_page(source)
    items = extract_data(source, base_url)
    return items