
local json=require "json"
local com=require "res.util.comm"
spiderCode="spider_demo";
spiderName="中国政府采购网";
spiderChannel="地方公告";
spiderDownDetailPage=true;
spiderStartPage=1;
spiderMaxPage=12000;
spiderRunRate=30;
spider2Collection="bidding";
spiderPageEncoding="utf8";
spiderStoreMode=1;
spiderStoreToMsgEvent=4002;
spiderTargetChannelUrl="http://search.ccgp.gov.cn/bxsearch?searchtype=2&page_index=1&bidSort=2&buyerName=&projectId=&pinMu=0&bidType=0&dbselect=bidx&kw=&start_time=2019%3A02%3A15&end_time=2019%3A02%3A18&timeType=1&displayZone=&zoneId=&pppStatus=0&agentName=";
spiderLastDownloadTime="2020-01-17 13:15:46";
spiderIsHistoricalMend=false;
spiderIsMustDownload=true;
spiderUserName="jiaoyubo";
spiderUserEmail="jiaoyubo@topnet.net.cn";
spiderUploadTime="2020-03-31 15:42:53";
spiderCoverAttr="title";
spiderSleepBase=1000;
spiderSleepRand=5000;
spiderTimeout=60;
	 
			
function getLastPublishTime()

		local head = {
			['Accept']='application/json,text/plain,*/*',
			['Referer']='http://zbb.snnu.edu.cn/provider/',
			['User-Agent']='Mozilla/5.0(WindowsNT10.0;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/83.0.4103.97Safari/537.36'
		}
print("ssssssssss")
		cont,_ = downloadAdv("https://www.yuncaitong.cn/api/enquiry/20KBFYBD5L0KZYHB","get",{},head,"")
		cont = json.decode(cont)
		-- printf(cont)
 printf(cont.winBid)
			return cont.winBid
		 
	 
end
	
function downloadAndParseListPage(pageno)
	for i=1,5 do 	--5次下载任务不成功，退出
		local update="" 
		local page={}
		--2019年1月1-2019年3月31
		local href = "http://search.ccgp.gov.cn/bxsearch?searchtype=2&page_index="..pageno.."&bidSort=2&buyerName=&projectId=&pinMu=0&bidType=0&dbselect=bidx&kw=&start_time=2019%3A01%3A01&end_time=2019%3A03%3A31&timeType=6&displayZone=&zoneId=&pppStatus=0&agentName="
		local head = {
			["Accept"]="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
			["Accept-Encoding"]="deflate",
			["Accept-Language"]="zh-CN,zh;q=0.9",
			["Cache-Control"]="no-cache",
			["Connection"]="keep-alive",
			["Host"]="search.ccgp.gov.cn",
			["Pragma"]="no-cache",
			["Referer"]=href,
			["Upgrade-Insecure-Requests"]="1",
			["User-Agent"]="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"
		}
		
		content,_ = downloadAdv(href,"get",{},head,"")
		
		local list = findListHtml("ul.vT-srch-result-list-bid li",content)--信息块规则
	
	
		if list~=nil then
			for k, v in pairs(list) do
				local item={}
				
				local bo = {}
				item["href"]="a:attr(href)"--信息地址
				item["title"]="a:eq(0)"--信息标题
				item["publishtime"]="span"--信息时间
				bo["areas"] = "a:eq(1)"
				item=findMap(item,"<table><tr>"..v.."</tr></table>")
				bo=findMap(bo,"<table><tr>"..v.."</tr></table>")
			
				local aa = ""
				if bo["areas"]~="" then
					aa = bo["areas"]
				else
					aa = "全国"
				end
				if item["title"]~=nil and item["title"]~="" then
					item["title"]=com.trim(item["title"])
					local timeType="yyyyMMddHHmmss"
					item["publishtime"]=com.parseDate(item["publishtime"],timeType)--时间格式
					item["href"]=com.gethref(spiderTargetChannelUrl,item["href"])
					
					item["spidercode"]="a_zgzfcgw_dfgg_new_1_3";item["site"]="中国政府采购网";item["channel"]="地方公告"; --通用数据补充
					item["publishdept"]="";item["type"]="";item["area"]=aa;item["city"]="";  --模型数据补充
			        table.insert(page,item) 
				end	
			end
		end
		if table.getn(page)>0 then
			return page
		end
	end
end
function strToTimestamp(str)
	local Y = tonumber(string.sub(str,1,4))
	local M = tonumber(string.sub(str,6,7))
	local D = tonumber(string.sub(str,9,10))
	return os.time{year=Y, month=M, day=D}
end
function downloadDetailPage(data)
	local update=""
	for i=1,5 do 	--5次下载任务不成功，退出
		local  head = {
			["Accept"]="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
			["Accept-Encoding"]="deflate",
			["Accept-Language"]="zh-CN,zh;q=0.9",
			["Cache-Control"]="no-cache",
			["Connection"]="keep-alive",
			["Host"]="www.ccgp.gov.cn",
			["Pragma"]="no-cache",
			["Referer"]="http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=1&bidSort=&buyerName=&projectId=&pinMu=&bidType=&dbselect=bidx&kw=&start_time=2019%3A11%3A24&end_time=2019%3A11%3A27&timeType=1&displayZone=&zoneId=&pppStatus=0&agentName=",
			["Upgrade-Insecure-Requests"]="1",
			["User-Agent"]="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"
		}
		content,_ = downloadAdv(data["href"],"get",{},head,"")
		local cont = findContentText("table[width='600']",content)
		
		--项目名称
		local projectname = regexp(cont,"采购项目名称(.+?)</",2)
		projectname = regexp(projectname[1],'">(.+)',2)[1]
		
		--品目
		local item = ""
		if cont:find("品目")~=nil then
			local it = regexp(cont,"品目(.+)",2)[1]
			it = regexp(it,'\">(.+?)<',2)[1]
			if it=='<br/>' then
				item = ""
			else
				item = it
			end
		end
	
		--采购单位
		local buyer = regexp(cont,"采购单位(.+)",2)[1]
		buyer = regexp(buyer,'\">(.+?)<',2)[1]
		
		
		--中标金额
		local bidamount = ""
		if cont:find("总中标金额")~=nil then
			local bdt = regexp(cont,"总中标金额(.+)",2)[1]
			if bdt~=nil then
				bidamount = regexp(bdt,'\">(.+?)<',2)[1]
			end
		elseif cont:find("总成交金额")~=nil then
			local bdt = regexp(cont,"总成交金额(.+)",2)[1]
			if bdt~=nil then
				bidamount = regexp(bdt,'\">(.+?)<',2)[1]
			end
		end
		
		--采购单位电话
		local buyer_info = ""
		if cont:find("采购单位联系方式")~=nil then
			local bytl = regexp(cont,"采购单位联系方式(.+)",2)[1]
			if bytl:find("电    话")~=nil then
				buyer_info = regexp(bytl,'电    话：(.+?)<',2)[1]
			else
				buyer_info = regexp(bytl,'\">(.+?)<',2)[1]
			end
		end
		
		--采购单位地址
		local buyeraddr = regexp(cont,"采购单位地址(.+)",2)[1]
		buyeraddr = regexp(buyeraddr,'\">(.+?)<',2)[1]
		
		--招标代理机构
		local agency = regexp(cont,"代理机构名称(.+)",2)[1]
		agency = regexp(agency,'\">(.+?)<',2)[1]
	
		
		--招标机构联系方式
		local agency_info = regexp(cont,"代理机构联系方式(.+)",2)[1]		
		if agency_info~=nil then
			agency_info = regexp(agency_info,'">(.+?)<',2)[1]
		end
		
		--招标机构地址
		local agencyaddr = regexp(cont,"代理机构地址(.+)",2)[1]
		agencyaddr = regexp(agencyaddr,'">(.+?)<',2)[1]
		--开标日期
		local bpt = ""
		if cont:find("开标时间")~=nil then
			local bidopentime=regexp(cont,'开标时间</td><td colspan=\"3\">(.+?)</td>',2)[1]
			if bidopentime:find("年")~=nil or bidopentime:find("月")~=nil or bidopentime:find("日")~=nil then
				bidopentime = replace(bidopentime,"年","/")
				bidopentime = replace(bidopentime,"月","/")
				bidopentime = replace(bidopentime,"日","/")
				bpt = strToTimestamp(bidopentime)
			else
				bpt = strToTimestamp(bidopentime)
			end
		end
		--预算金额
		local bt = ""
		if cont:find("预算金额")~=nil then
			bt = regexp(cont,"预算金额</td><td colspan=\"3\">(.+?)</td>",2)[1]
		end
		--行政区域
		local acd = ""
		if cont:find("行政区域")~=nil then
			acd = regexp(cont,"行政区域</td><td>(.+?)</td>",2)[1]
		end
		data["jsondata"] = {
				["projectname"] =projectname ,
				["item"] = item,
				["buyer"] = buyer,
				["bidamount"] =bidamount,
				["buyeraddr"] = buyeraddr,
				["buyer_info"] =buyer_info,
				["agency"] =agency,
				["agency_info"] =agency_info,
				["agencyaddr"] =agencyaddr,
				["bidopentime"]=bpt,
				["budget"]=bt,
				["area_city_district"]=acd
			}
		local file_list = findListHtml("table[width='600'] tr",content)
		local files = ""
		for k,v in pairs(file_list) do
			local name_1 = findOneText("a:attr(title)",v)
			if name_1=="点击下载" then
				local name = findOneText("a",v)
				local link = findOneText("a:attr(id)",v)
				link="<a href='".."http://www.ccgp.gov.cn/oss/download?uuid="..link.."'>"..name.."</a>"
				files = files.."<br/>"..link
			end
		end
		data["summary"]=findOneHtml("div.table",content)
		
		data["title"]= com.trim(findOneText("h2.tc",content))
		data["detail"]=findContentText("div.vF_detail_content",content)
		data["contenthtml"]=findOneHtml("div.vF_detail_content",content)
		data["l_np_publishtime"]=com.strToTimestamp(data["publishtime"])
		data["_d"]="comeintime"
		local checkAttr={"title","href","publishtime","detail","contenthtml"}
		local b,err=com.checkData(checkAttr,data)
		if b then
			local attachments = com.getFileAttachmentsArrayWithTag(data["href"],"dl","<dl>"..files.."</dl>",false) 
			if table.getn(attachments)>0 then
				data["projectinfo"]={
					["attachments"]=attachments
				}
			end
			return data
		else
			timeSleep(60)--延时60秒再次请求
			if i==5 then
				--print("下载失败")
				saveErrLog(spiderCode,spiderName,data["href"],err)
			end
		end
	end
end