browserSpider/program/spider/scripts/模版/2层URL翻页模版.lua

-- 采集参数设置
-- 二层级,单位信息列表/信息
-- 拼接列表页地址，需要指定模版
-- 列表中含有附件
require "scripts/inc/util"
-- 列表页模版
local href = "https://czj.xinyang.gov.cn/c/113/1.html"
local href_tpl = "https://czj.xinyang.gov.cn/c/113/%d.html"
-- 列表页CSS选择期，可以避免检测垃圾数据，不想定义，直接写a也行
local info_list_css_selector = "div.zfxxgk_zdgkc ul a"
-- 列表页页数
local first_page_no,last_page_no,page_step = 1,100,1

local timeout = 1000 * 60 * 120
-- TODO 
-- 检查是否有下一页
local is_has_next_page_js = [[
	var ret="false";
	//var obj = %s;
	//if(obj){ret="true"};
	ret="true";
	ret
]]
is_has_next_page_js = string.format(is_has_next_page_js, next_page_css_selector)
-- 找信息
local find_info_list_js = [[
    var ns = [];
    document.querySelectorAll("%s").forEach((v,i)=>{
        v.setAttribute("target","_blank");
        var linkText = v.innerText;
	 if((linkText.indexOf("预算")>-1 ||linkText.indexOf("预决算")>-1||linkText.indexOf("绩效")>-1 ||linkText.indexOf("目标")>-1 ) &&
            (linkText.indexOf("2024年")>-1)) { //linkText.indexOf("2022年")>-1 || linkText.indexOf("2023年")>-1 || 
            ns.push({"index":i,"title":linkText,"href":v.href})
        }
    });
    ns
]]
find_info_list_js = string.format(find_info_list_js, info_list_css_selector)
--找附件
local find_info_attach_file_js = [[
       var ns = [];
       var extensions = ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'zip', 'rar','PDF','DOC','DOCX','XLS','XLSX','ZIP','RAR'];
       var extensionRegex = new RegExp('\\.(' + extensions.join('|') + ')$', 'i');
       document.querySelectorAll("a").forEach((v,i)=>{
        v.setAttribute("target","_blank");
        var linkText = v.innerText;
        var href = v.href
	if(extensionRegex.test(linkText.toLowerCase())||extensionRegex.test(href.toLowerCase())){
        	ns.push({"index":i,"title":linkText,"href":v.href})
        }
    });
    ns
]]

--TODO 1. 打开主页
local page_no=first_page_no
local current_page_has_data=true
repeat
 	if page_no>first_page_no then
		href = string.format(href_tpl,page_no)
	end
	browser_navagite(1000*120, href)
	browser_sleep(1000 * 1)
		
	current_page_has_data=false
    local ok, list = browser_executejs(1000*30, 1, find_info_list_js)
    if ok == "ok" then
        print("信息列表信息", list, table_length(list))
        for _, v in pairs(list) do
            print(v.title, v.href, v.index)
			browser_navagite(true,1000*30, v.href)
            local path = v.href
			local ok2, list2 = browser_executejs(v.href, 1000*30, 1, find_info_attach_file_js)
			if ok2~="ok" then
				path = browser_url_last_segs(2,v.href)
				ok2, list2 = browser_executejs(path, 1000*3, 1, find_info_attach_file_js)
			end
            print('查找详情页附件链接', ok2, list2, table_length(list2))
            if ok2 == "ok" then
                for _, v2 in pairs(list2) do
                    -- 存储数据
                    current_page_has_data=true
                    print(v2.title, v2.href)
                    browser_save("", "", "", "", { ["department"] = v['title'], ["info_title"] = v2['title'], ["attach_href"] = v2["href"] ,["source"]=v.href})
                end
            end
			
			-- 找不到附件，保存主文档区域到文件
			if table_length(list2)==0 then
				print("找不到附件，准备生成网页pdf")
				clear_page_el("",path,1000*2,"div.article_txt")
				save_page_to_pdf("",path,"/Users/taozhang/Downloads/docs2/",v.title)
				browser_save("", "", "", "", { ["department"] = v["title"], ["info_title"] =  v["title"], ["attach_href"] = "",["source"]=v.href })
			end
			
            browser_closetabs(path, 500)
        end
    end
    browser_reset()
    page_no = page_no + page_step
	print("翻页：：",page_no,last_page_no)
until  page_no>last_page_no --not current_page_has_data or

print("所有链接都爬完了")
browser_sleep(1000 * 5)