browserSpider/program/spider/scripts/模版/2层URL翻页模版.lua_bak

-- 采集参数设置
-- 二层级,单位信息列表/信息
-- 拼接列表页地址，需要指定模版
-- 列表中含有附件

-- 列表页地址
local href = "https://www.nanning.gov.cn/zwgk/fdzdgknr/czxx/sbjyjs/sbjbmys/2023bmys/index.html"
-- 列表页模版
local href_tpl = "https://www.nanning.gov.cn/zwgk/fdzdgknr/czxx/sbjyjs/sbjbmys/2023bmys/index_%d.html"
-- 列表页CSS选择期，可以避免检测垃圾数据，不想定义，直接写a也行
local info_list_css_selector = "a"
-- 三级页特征，用于找到tab页，准备在这个tab页执行js，关闭tab页
local info_url_path_tpl = "t20240306"
-- 列表页页数
local first_page_no,last_page_no,page_step = 0,6,1


local timeout = 1000 * 60 * 120
-- TODO 
-- 检查是否有下一页
local is_has_next_page_js = [[
	var ret="false";
	//var obj = %s;
	//if(obj){ret="true"};
	ret="true";
	ret
]]
is_has_next_page_js = string.format(is_has_next_page_js, next_page_css_selector)
-- 找信息
local find_info_list_js = [[
    var ns = [];
    document.querySelectorAll("%s").forEach((v,i)=>{
        v.setAttribute("target","_blank");
        var linkText = v.innerText;
        ns.push({"index":i,"title":linkText,"href":v.href})
        if((linkText.indexOf("预算")>-1 ||linkText.indexOf("预决算")>-1) &&
            (linkText.indexOf("2022年")>-1 || linkText.indexOf("2023年")>-1 || linkText.indexOf("2024年")>-1)) {
            ns.push({"index":i,"title":linkText,"href":v.href})
        }
    });
    ns
]]
find_info_list_js = string.format(find_info_list_js, info_list_css_selector)
--找附件
local find_info_attach_file_js = [[
       var ns = [];
       var extensions = ['pdf', 'doc', 'docx', 'xls', 'xlsx', 'zip', 'rar','PDF','DOC','DOCX','XLS','XLSX','ZIP','RAR'];
       var extensionRegex = new RegExp('\\.(' + extensions.join('|') + ')', 'i');
       document.querySelectorAll("a").forEach((v,i)=>{
        v.setAttribute("target","_blank");
        var linkText = v.innerText;
        var href = v.href
	if(extensionRegex.test(linkText.toLowerCase())||extensionRegex.test(href.toLowerCase())){
        	ns.push({"index":i,"title":linkText,"href":v.href})
        }
    });
    ns
]]

--TODO 1. 打开主页
local page_no=first_page_no
local current_page_has_data=true
repeat
	if page_no>first_page_no then
		href = string.format(href_tpl,page_no)
	end
	browser_navagite(1000*120, href)
	browser_sleep(1000 * 1)
		
	current_page_has_data=false
    local ok, list = browser_executejs(1000*30, 1, find_info_list_js)
    if ok == "ok" then
        print("信息列表信息", list, #list)
        for _, v in pairs(list) do
            print(v.title, v.href, v.index)
            --browser_click(1000*5, 3, string.format('document.querySelectorAll("%s")[', info_list_css_selector) .. v['index'] .. ']')
            --browser_sleep(1000 * 1)
			browser_navagite(true,1000*30, v.href)
            --local path = string.match(v.href, info_url_path_tpl)
            --print("path::",path)
            local path = v.href
			local ok2, list2 = browser_executejs(v.href, 1000*30, 1, find_info_attach_file_js)
			if ok2~="ok" then
				path = browser_url_last_segs(2,v.href)
				ok2, list2 = browser_executejs(path, 1000*3, 1, find_info_attach_file_js)
			end
            print('查找详情页附件链接', ok2, list2, #list2)
            if ok2 == "ok" then
                for _, v2 in pairs(list2) do
                    -- 存储数据
                    current_page_has_data=true
                    print(v2.title, v2.href)
                    browser_save("", "", "", "", { ["department"] = v['title'], ["info_title"] = v2['title'], ["attach_href"] = v2["href"] })
                end
            end
            browser_closetabs(path, 500)
        end
    end
    -- browser_reset()
    page_no = page_no + page_step
	--关闭其他页面，莫名其妙有tab关不掉
	browser_closetabs_without("",href,1000*1)
	print("翻页：：",page_no,last_page_no)
	browser_sleep(1000 * 2)
until  page_no>last_page_no --not current_page_has_data or

print("所有链接都爬完了")
browser_sleep(1000 * 5)