]*>': '
', # 分割 '

': '', # 分割 division '|]*>|': '' # OFFICE微软WORD段落 } '''其他''' OTHER = { ']*>|]*>|': '', '': '', '|]*>': '', '【关闭】|关闭': '', '【打印】|打印本页': '', '【字体：[\s\S]*】': '', '文章来源：[\u4e00-\u9fa5]+': '', '浏览次数：.*[<]+': '', '（责任编辑：.*?）': '', '分享到[：]': '', } '''样式''' CSS_STYLE = { 'style="[\s\S]*?"|style ="[\s\S]*?"': '', 'bgcolor="[\s\S]*?"|bgcolor ="[\s\S]*?"': '', 'bordercolor="[\s\S]*?"|bordercolor ="[\s\S]*?"': '', 'class="[\s\S]*?"|class ="[\s\S]*?"': '', 'align="[\s\S]*?"|align ="[\s\S]*?"': '', 'cellpadding="(\d+)"|cellspacing="(\d+)"': '', } '''空白符''' BLANKS = { '\n\s*\n': '\n', '\s*\n\s*': '\n', '[^\S\n]': ' ', '\s+': ' ', } '''css标签集合''' TAGS = {'table', 'tr', 'td', 'div', 'span', 'p'} '''css属性集合''' ATTRS = {'id', 'class', 'style', 'width'} '''特殊样式的标签''' SPECIAL_TAGS = { re.compile('(?i)<[^>]+style="display: none".*[^>]+>'): '
', } def _repair_tag(): """异常的标签组合,用来替换非标准页面的标签""" _repairs = {} for tag in TAGS: for attr in ATTRS: key = '{}{}'.format(tag, attr) val = '{} {}'.format(tag, attr) _repairs[key] = val return _repairs def _escape_character(html): """转义字符""" html = html.replace('<', '<') html = html.replace('>', '>') html = html.replace('"', '"') html = html.replace('&', '&') return html def _lowercase_tag(html): """元素标签转成小写，不影响页面文本""" tags = re.findall("<[^>]+>", html) tag_sets = set(tags) if len(tag_sets) > 10000: from bs4 import BeautifulSoup soup = BeautifulSoup(html, 'lxml') html = str(soup.body.next_element) else: for tag in tag_sets: html = html.replace(tag, str(tag).lower()) repair_tags = _repair_tag() # 标签修复 for err, right in repair_tags.items(): html = html.replace(err, right) return html def _clear_special_tag(html): """删除特殊元素标签""" for tag, repl in SPECIAL_TAGS.items(): html = tag.sub(repl, html) return html def _clear_input_tag(html, display=False): """提取value值，替换input标签""" if not display: html = html.replace('', re.S) value = re.compile(r'value=["|\'](.*?)["|\']') lst = re.findall(tag, html) or [] for ipt in lst: val = re.findall(value, ipt) if val and 'hidden' not in ipt and 'hide' not in ipt and 'display: none' not in ipt: html = html.replace(ipt, val[0]) return html def drop_tree_by_lxml(html, feature): tree: HtmlElement = fromstring(html) tag_lst = tree.xpath(feature) for tag in tag_lst: tag.drop_tree() html = tostring(tree, encoding='utf8').decode('utf8') return html def cleaner(html, special=None, completely=False, del_tag=False, **kwargs): """ 源码清洗 :param html: 清洗的页面 :param special: 额外指定页面清洗规则 :param completely: 是否完全清洗页面 :param del_tag: 删除标签 :return: 页面源码 """ special = set() if special is None else special OTHER.update(special) remove_tags = { **INDEPENDENT_TAGS, **INLINE_TAGS, **BLOCK_TAGS, **OTHER, **CSS_STYLE, **BLANKS, } html = _lowercase_tag(html) if del_tag: html = _clear_special_tag(html) for tag, repl in remove_tags.items(): html = re.sub(tag, repl, html) if completely: html = re.sub(r']*>[\s\S]*?', '', html) # 画布 html = re.sub(r']*>[\s\S]*?', '', html) # 内框架 html = re.sub('<([^<>\u4e00-\u9fa5]|微软雅黑|宋体|仿宋)+>', '', html) html = _escape_character(html) html = _clear_input_tag(html, **kwargs) return html