--山东省公示系统爬虫  
--http://218.57.139.24/

--引用公用包
local com=require "res.util.comm"
local ecps=require "res.util.ecps"
local json=require "json"
--名称
spiderName="山东公示爬虫";
--代码
spiderCode="ecps_sd";
--是否下载3级页
spiderDownDetailPage=true;
--开始下载页
spiderStartPage=1;
--最大下载页
spiderMaxPage=500;
--上次下载时间
spiderLastDownloadTime="2016-01-01 01:01:01";
--执行频率30分钟
spiderRunRate=1;

--下载内容写入表名
spider2Collection="ecps";
--下载页面时使用的编码
spiderPageEncoding="UTF8";
--是否使用代理
spiderUserProxy=false;
--是否是安全协议
spiderUserHttps=false;
--下载详细页线程数
spiderThread=1
--存储模式 1 直接存储，2 调用消息总线 ...
spiderStoreMode=1
spiderStoreToMsgEvent=1 --消息总线event
--延时毫秒 基本延时(spiderSleepBase)+随机延时(spiderSleepRand)
spiderSleepBase=100
spiderSleepRand=100
--判重字段 空默认不判重，spiderCoverAttr="title" 按title判重覆盖
spiderCoverAttr="title"
--取得对方网站最后发布时间 必须返回yyyy-MM-dd HH:mm:ss 格式
function getLastPublishTime()
--	local content=[[var wdxxliststr ='';
--    var czxxliststr ='[{"acconcurrency":"156","accondate":{"date":2,"day":4,"hours":0,"minutes":0,"month":5,"seconds":0,"time":1306944000000,"timezoneOffset":-480,"year":111},"acconform":"1|货币","ancheid":"046C247C8D725FB7E05014AC8A8370B7","ifpub":"2","inv":"王立国","liacconam":70,"lisubconam":70,"pid":"565c156c13e64afe85bb3dbd53635a21","subconcurrency":"156","subcondate":{"date":2,"day":4,"hours":0,"minutes":0,"month":5,"seconds":0,"time":1306944000000,"timezoneOffset":-480,"year":111},"subconform":"1|货币","updatetime":{"date":30,"day":2,"hours":8,"minutes":57,"month":11,"seconds":42,"time":1419901062000,"timezoneOffset":-480,"year":114}},{"acconcurrency":"156","accondate":{"date":2,"day":4,"hours":0,"minutes":0,"month":5,"seconds":0,"time":1306944000000,"timezoneOffset":-480,"year":111},"acconform":"1|货币","ancheid":"046C247C8D725FB7E05014AC8A8370B7","ifpub":"2","inv":"王立军","liacconam":66,"lisubconam":66,"pid":"a31221eaeb7a4b8b8a39bcfbb70fa3de","subconcurrency":"156","subcondate":{"date":2,"day":4,"hours":0,"minutes":0,"month":5,"seconds":0,"time":1306944000000,"timezoneOffset":-480,"year":111},"subconform":"1|货币","updatetime":{"date":30,"day":2,"hours":8,"minutes":57,"month":11,"seconds":42,"time":1419901062000,"timezoneOffset":-480,"year":114}},{"acconcurrency":"156","accondate":{"date":2,"day":4,"hours":0,"minutes":0,"month":5,"seconds":0,"time":1306944000000,"timezoneOffset":-480,"year":111},"acconform":"1|货币","ancheid":"046C247C8D725FB7E05014AC8A8370B7","ifpub":"2","inv":"王洋","liacconam":64,"lisubconam":64,"pid":"94fa6ceb758b4d41be263fdd71a63930","subconcurrency":"156","subcondate":{"date":2,"day":4,"hours":0,"minutes":0,"month":5,"seconds":0,"time":1306944000000,"timezoneOffset":-480,"year":111},"subconform":"1|货币","updatetime":{"date":30,"day":2,"hours":8,"minutes":57,"month":11,"seconds":42,"time":1419901062000,"timezoneOffset":-480,"year":114}}]';
--    var dwtzliststr ='';
--    var nbgqbgsstr='';
--    var nbdwdbstr='[]';
--    var nbalthisstr='[]';
--    var althistlist=null;
--    var wdxxlist=null;
--    var czxxlist=null;
--    var dwtzlist=null;
--    var gqbglist=null;
--    var dwdblist=null;]]
--	local czxxliststr=string.match(content,"var czxxliststr%s*=%'(%[.*%])%';")
--	print(czxxliststr)
--	os.exit()
	return os.date("%Y-%m-%d %H:%M:%S", os.time())
end



--下载分析列表页
function downloadAndParseListPage(pageno) 
	
	
	local entMl=getEntNames("ecps_sd")
	--local entMl={}
	print("山东企业名录长度:",tostring(table.getn(entMl)))
	if table.getn(entMl)==0 then
		print("开始下山东异常名录")
		local downloadId=changeDownloader()
		print("山东切换完下载点:",downloadId)
		local head={
				["Host"]="218.57.139.24",
				["Connection"]="keep-alive",
				["Accept"]="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
				["Accept-Encoding"]="gzip,deflate,sdch",
				["Accept-Language"]="zh-CN,zh;q=0.8",
				["Upgrade-Insecure-Requests"]="1",
				["User-Agent"]="Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"	
		}
		local con,cookies=downloadAdv("http://218.57.139.24/pub/abnormalrecordindex","get",{},head,"")
		if cookies~="null" then
			local token=string.match(con,"<meta name=\"_csrf\".*content=\"(.*)\".*/>.*<link")
			--print("token:"..token)
			local head3={
				["Host"]="218.57.139.24",
				["Connection"]="keep-alive",
				["Cache-Control"]="no-cache",
				["Accept"]="application/json,text/javascript,*/*;q=0.01",
				["Origin"]="http://218.57.139.24/",
				["X-CSRF-TOKEN"]=token,
				["X-Requested-With"]="XMLHttpRequest",
				["Accept-Encoding"]="gzip,deflate",
				["Accept-Language"]="zh-CN,zh;q=0.8",
				["Content-Type"]="application/x-www-form-urlencoded; charset=UTF-8",
				["Referer"]="http://218.57.139.24/pub/abnormalrecordindex",
				--["Cookie"]=cookies3,
				["User-Agent"]="Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"	
			}
			local param={
				["page"]=tostring(pageno)
			}
			local con3,_=downloadAdv("http://218.57.139.24/pub/jyyc","post",param,head3,cookies)
			print(con3)
			if con3~="" then
				local dataArray=json.decode(con3)
				if table.getn(dataArray)>0 then
					for k,v in pairs(dataArray) do
						item={}
						--print(v["entname"])
						item["title"]=v["entname"]
						item["href"]="http://218.57.139.24/securitycode?"..tostring(os.time()) --验证码
						table.insert(entMl,item)
					end
				end
			end
		end
	end
return entMl
--测试代码
--	local entMl={}
--	for i=1,1 do
--				item={}
--				--item["title"]="中绿（白城）农业开发有限公司"
--				item["title"]="山东高速篮球俱乐部有限公司"
--				item["href"]="http://218.57.139.24/securitycode?"..tostring(os.time()) --验证码
--				table.insert(entMl,item)
				
--	end
--	return entMl
--测试代码结束
	
end

function downloadDetailPage(data)
	data.href="http://218.57.139.24/securitycode?"..tostring(os.time())
	for i=1,50 do
		local downloadId=changeDownloader()
		print("山东切换完下载点:",downloadId)
		for j=1,1 do--死循环
			local cookies,_csrf=getSpecialCookies()
				--os.exit()
				print("开始下验证码: "..data["title"])
				local code=getCode(data["href"],cookies)
				if code=="" then
					--timeSleep(3)
					break
				else
					--print("开始校验:",code,_csrf)
					local isSuccess,enckeyword=checkCode(data["title"],_csrf,code,cookies)
					--print(isSuccess,enckeyword)
					if isSuccess=="true" and enckeyword~=nil then
						local entTmp=getEntTmp(_csrf,cookies,enckeyword)
						--os.exit()
						if entTmp~=nil then
							local entTmpObj=json.decode(entTmp)
							if table.getn(entTmpObj)>0 then 
								local entType=entTmpObj[1]["enttype"]
								local encrptpripid=entTmpObj[1]["encrptpripid"]
								local detailUrl="http://218.57.139.24/pub/gsgsdetail/"..entType.."/"..encrptpripid
								local entInfo={}
								getEntDetail(encrptpripid,entType,_csrf,detailUrl,cookies,entInfo)
								if entInfo["Area"]~="SD" then
									print("下载",data["title"]," 详细信息不完整，丢弃")
									return nil
								else
									--去冗余字段
									entInfo["pripid"]=nil
									entInfo["href"]=nil
									entInfo["publishtime"]=nil
									data["href"]=nil
									local b=checkData(data["title"],entInfo)
									--print("b:",b)
										if b	 then
											return entInfo
										else
											break
										end
								end
							else
								print(data["title"].."  无此企业!")
								saveErrLog(spiderCode,spiderName.."_title",data["title"],err)
								return nil
							end
							
							
						end
					else
						print("验证码错误..")
						--timeSleep(3)
						break
					end
				end
			
		end
	end
end
function getEntDetail(encrptpripid,entType,_csrf,detailUrl,cookies,entInfo)
	--print("detailUrl:"..detailUrl)
	local head={
			["Accept"]="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
			["Accept-Encoding"]="gzip, deflate, sdch",
			["Accept-Language"]="zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4",
			["Connection"]="keep-alive",
			["Host"]="218.57.139.24",
			["Origin"]="http://218.57.139.24/",
			["Referer"]="http://218.57.139.24/pub/indsearch",
			["User-Agent"]="Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36"
			--["Cookie"]=cookies
	}
	local content,_=downloadAdv(detailUrl,"get",{},head,cookies)
	--content=unGzip(content)
	--print(content)
	--os.exit()
	entInfo["Area"]="SD"
	--基本信息
	entInfo=getBaseInfo(content,entInfo)
	--os.exit()
	--股东信息
	local invertors=getInvertor(content)
	entInfo["invertor"]=invertors
	
	--变更信息
	local alterInfos=getAlterInfo(content)
	entInfo["alterInfo"]=alterInfos
	
	--主要人员
	local staffinfos=getStaffinfo(encrptpripid,entType,_csrf,detailUrl,cookies)
	entInfo["staffinfo"]=staffinfos
	
	--分支机构
	local childents=getChildent(encrptpripid,entType,_csrf,detailUrl,cookies)
	entInfo["childent"]=childents
	
	--行政处罚
	local punishInfos=getPunishInfo(encrptpripid,entType,_csrf,detailUrl,cookies)
	entInfo["punishInfo"]=punishInfos
	
	--经营异常
	local excDirecs=getExcDirec(encrptpripid,entType,_csrf,detailUrl,cookies)
	entInfo["excDirec"]=excDirecs
	
	--年报
	local nbtab=getNbList(encrptpripid,entType,cookies)
	if nbtab=="error" then
		print("查询年报列表信息请求出错")
		return "error"
	end
	entInfo["qynb"]=nbtab
	
	
end

function getBaseInfo(content,entInfo)
	local th=findListText("div#jibenxinxi table.detailsList:eq(0) tr th",content)
	local td=findListText("div#jibenxinxi table.detailsList:eq(0) tr td",content)
	local tab={}
	
	for k,v in pairs(th) do
		if k>1 then
			local tdv=com.trim(td[k-1])
			tab[com.trim(v)]=tdv
			--print(com.trim(v),tab[com.trim(v)])
		end
	end
	local baseinfo=ecps.reversalFormat(tab,ecps.baseFm,ecps.baseMap)
	for k,v in pairs(baseinfo) do
		entInfo[k]=v
		--print(k,v)
	end
	return entInfo
end

function getInvertor(content)
	local investors={}
	--print("投资人.....")
	if string.find(content,"czxxliststr")~=nil then
		local czxxliststr=string.match(content,"var czxxliststr%s*=%'(%[.-%])%';")
		--print("czxxliststr:"..czxxliststr)
		if czxxliststr~="" then
			local czxxlist=json.decode(czxxliststr)
			local changdu=table.getn(czxxlist)
			--print("changdu:"..changdu)
			if changdu>0 then
				for k,v in pairs(czxxlist) do
					local inverstor={}
					inverstor["Inv"]=v["inv"]
					inverstor["InvTypeName"]=v["invtype"]
					--print(inverstor["Inv"],inverstor["InvTypeName"])
					table.insert(investors,inverstor)
				end
			end
		end
	end
	
	
	return investors
end


function getAlterInfo(content)
	--print(content)
	local alterInfos={}
	--print("变更信息")
	--if string.find(content,"czxxlist")~=nil then
		local bgsxliststr=string.match(content,"var bgsxliststr%s*=%'(%[.-%])%';")
		--print("bgsxliststr:"..bgsxliststr)
		if bgsxliststr~="" then
			local bgsxlist=json.decode(bgsxliststr)
			local changdu=table.getn(bgsxlist)
			if changdu>0 then
				for k,v in pairs(bgsxlist) do
					local alterInfo={}
					alterInfo["AltItemName"]=v["altitem"]
					alterInfo["AltAf"]=v["altaf"]
					alterInfo["AltBe"]=v["altbe"]
					local str=tostring(v["altdate"]["time"])
					--print(str)
					local str2=string.sub(str,0,string.len(str)-3)
					--print(str2)
					local time=tonumber(str2)
					--print(time)
					alterInfo["AltDate"]=os.date("%Y-%m-%d",time)
					--print(alterInfo["AltItemName"],alterInfo["AltBe"],alterInfo["AltAf"],alterInfo["AltDate"])
					--print(alterInfo["AltDate"])
					table.insert(alterInfos, alterInfo)
				end
			end
		end
	--end
	
	
	return alterInfos
end

function getStaffinfo(encrpripid,entType,_csrf,detailUrl,cookies)
	local staffinfos={}
	local head={
			["Accept"]="application/json, text/javascript, */*; q=0.01",
			["Accept-Encoding"]="gzip,deflate",
			["Accept-Language"]="zh-CN,zh;q=0.8",
			["Connection"]="keep-alive",
			["X-CSRF-TOKEN"]=_csrf,
			["X-Requested-With"]="XMLHttpRequest",
			["Host"]="218.57.139.24",
			["Origin"]="http://218.57.139.24",
			["Content-Type"]="application/x-www-form-urlencoded; charset=UTF-8",
			["Referer"]=detailUrl,
			["User-Agent"]="Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36"
			--["Cookie"]=cookies
	}
	local param={
		["encrpripid"]=encrpripid
	}
	local content,_= downloadAdv("http://218.57.139.24/pub/gsryxx/"..entType,"post",param,head,cookies)
	--print("content:",content)
	local list=json.decode(content)
	local changdu=table.getn(list)
	if changdu>0 then
		for k,v in pairs(list) do
			local staffinfo={}
			staffinfo["Name"]=v["name"]
			if v["position"]~=nil then
				staffinfo["Position"]=v["position"]
			end
			--print(staffinfo["Name"],staffinfo["Position"])
			table.insert(staffinfos,staffinfo)
		end
	end
	return staffinfos
end


function getChildent(encrptpripid,entType,_csrf,detailUrl,cookies)
	local childents={}
	local head={
			["Accept"]="application/json, text/javascript, */*; q=0.01",
			["Accept-Encoding"]="gzip,deflate",
			["Accept-Language"]="zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4",
			["Connection"]="keep-alive",
			["X-CSRF-TOKEN"]=_csrf,
			["X-Requested-With"]="XMLHttpRequest",
			["Host"]="218.57.139.24",
			["Origin"]="http://218.57.139.24",
			["Content-Type"]="application/x-www-form-urlencoded; charset=UTF-8",
			["Referer"]=detailUrl,
			["User-Agent"]="Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36"
			--["Cookie"]=cookies
	}
	local param={
		["encrpripid"]=encrptpripid
	}
	--print("encrpripid:"..encrptpripid)
	local content,_= downloadAdv("http://218.57.139.24/pub/gsfzjg/"..entType,"post",param,head,cookies)
	--print("content:",content)
	local list=json.decode(content)
	local changdu=table.getn(list)
	if changdu>0 then
		for k,v in pairs(list) do
			local childent={}
			childent["BrName"]=v["brname"]--分支名称
			childent["RegNO"]=v["regno"]--注册号
			childent["RegOrgName"]=v["regorg"]--分支机构登记机关
			--print(childent["BrName"],childent["RegNO"],childent["RegOrgName"])
			table.insert(childents, childent)
		end
	end
	return childents
end


function getPunishInfo(encrpripid,entType,_csrf,detailUrl,cookies)
	--print("行政处罚........")
	local punishInfos={}
	local head={
			["Accept"]="application/json, text/javascript, */*; q=0.01",
			["Accept-Encoding"]="gzip,deflate",
			["Accept-Language"]="zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4",
			["Connection"]="keep-alive",
			["X-CSRF-TOKEN"]=_csrf,
			["X-Requested-With"]="XMLHttpRequest",
			["Host"]="218.57.139.24",
			["Origin"]="http://218.57.139.24",
			["Content-Type"]="application/x-www-form-urlencoded; charset=UTF-8",
			["Referer"]=detailUrl,
			["User-Agent"]="Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36"
			--["Cookie"]=cookies
	}
	local param={
		["encrpripid"]=encrpripid
	}
	local content,_= downloadAdv("http://218.57.139.24/pub/gsxzcfxx/","post",param,head,cookies)
	--print("content:",content)
	local list=json.decode(content)
	local changdu=table.getn(list)
	if changdu>0 then
		for k,v in pairs(list) do
			local punishInfo={}
			punishInfo["PenDecNo"]=v["pendecno"]--行政处罚号
			punishInfo["IllegActTypeName"]=v["illegacttype"]--行政处罚类型名称
			punishInfo["PenResult"]="罚款金额:"..v["penam"].."万元,没收金额:"..v["forfam"].."万元"--处罚结果
			punishInfo["OrgName"]=v["penauth"]--机构
			local str=tostring(v["pendecissdate"]["time"])
			--print(str)
			local str2=string.sub(str,0,string.len(str)-3)
			--print(str2)
			local time=tonumber(str2)
			--print(time)
			punishInfo["PenDecIssDate"]=os.date("%Y-%m-%d",time)
			--print(punishInfo["PenDecNo"],punishInfo["IllegActTypeName"],punishInfo["PenResult"],punishInfo["OrgName"],punishInfo["PenDecIssDate"])
			table.insert(punishInfos, punishInfo)
		end
	end
	--print("行政处罚完毕...")
	return punishInfos
end

function getExcDirec(encrpripid,entType,_csrf,detailUrl,cookies)
	      
	local excDirecs={}
	local head={
			["Accept"]="application/json, text/javascript, */*; q=0.01",
			["Accept-Encoding"]="gzip,deflate",
			["Accept-Language"]="zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4",
			["Cache-Control"]="no-cache",
			["Connection"]="keep-alive",
			["Pragma"]="no-cache",
			["X-CSRF-TOKEN"]=_csrf,
			["X-Requested-With"]="XMLHttpRequest",
			["Host"]="218.57.139.24",
			["Origin"]="http://218.57.139.24",
			["Content-Type"]="application/x-www-form-urlencoded; charset=UTF-8",
			["Referer"]=detailUrl,
			["User-Agent"]="Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36"
			--["Cookie"]=cookies
	}
	local param={
		["encrpripid"]=encrpripid
	}
	local content,_= downloadAdv("http://218.57.139.24/pub/jyyc/"..entType,"post",param,head,cookies)
	--print("content:",content)
	local list=json.decode(content)
	local changdu=table.getn(list)
	if changdu>0 then
		for k,v in pairs(list) do
			local excDirec={}
			excDirec["DecOrg"]=v["decorg"]
			excDirec["SpeCause"]=v["specause"]
			excDirec["OutSpeCause"]=v["remexcpres"]
			
			
			local str=tostring(v["abntime"]["time"])
			local str2=string.sub(str,0,string.len(str)-3)
			local time=tonumber(str2)
			excDirec["AbnTime"]=os.date("%Y-%m-%d",time)
			
			
			
--			local str2=tostring(v["remdate"]["time"])
--			local str3=string.sub(str2,0,string.len(str2)-3)
--			local time2=tonumber(str3)
--			excDirec["OutSpeTim"]=os.date("%Y-%m-%d",time2)
			
			
			
			--print(excDirec["DecOrg"],excDirec["SpeCause"],excDirec["AbnTime"],excDirec["OutSpeTim"],excDirec["OutSpeCause"])
			table.insert(excDirecs, excDirec)
		end
	end
	--print("经营异常完毕")
	return excDirecs
end

function getNbList(encrptpripid,entType,cookies)
	local  nburl="http://218.57.139.24/pub/qygsdetail/"..entType.."/"..encrptpripid
	--print("nburl:"..nburl)
	local head={
			["Accept"]="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
			["Accept-Encoding"]="gzip, deflate, sdch",
			["Accept-Language"]="zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4",
			["Cache-Control"]="no-cache",
			["Connection"]="keep-alive",
			["Host"]="218.57.139.24",
			["Origin"]="218.57.139.24",
			["Pragma"]="no-cache",
			["Upgrade-Insecure-Requests"]="1",
			["Referer"]="http://218.57.139.24/pub/qygsdetail/"..entType.."/"..encrptpripid,
			["User-Agent"]="Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36"
			--["Cookie"]=cookies
	}
	local content,_=downloadAdv(nburl,"get",{},head,cookies)
	
	local nbinfos={}
	local list=findListHtml("div#qiyenianbao table.detailsList:eq(0) tr",content)
	
	if table.getn(list)>0 then
		for k,v in pairs(list) do
			if k>2 then
				local item={}
				item["href"]="td:eq(1) a:eq(0):attr(href)"
				item["nd"]="td:eq(1) a:eq(0)"
				item=findMap(item,"<table>><tr>"..v.."</tr></table>")
				--print(item["href"])
				local _content,_=downloadAdv("http://218.57.139.24/"..item["href"],"get",{},head,cookies)
				local nbinfo=getExYear(_content)
				nbinfo["ancheyear"]=com.trim(item["nd"])
				table.insert(nbinfos, 1, nbinfo)
			end
			
		end
	end
	--print("年报彻底完成")
	return nbinfos
	
end

function getExYear(content)
	local nb={}
	--年报基本信息
	local th=findListText("div#qufenkuang:eq(0) table.detailsList:eq(0) tr th",content)
	local td=findListText("div#qufenkuang:eq(0) table.detailsList:eq(0) tr td",content)
	local tab={}
	for k,v in pairs(th) do
		if k>2 then
			local tdv=com.trim(td[k-2])
			tab[com.trim(v)]=tdv
			--print(com.trim(v),tab[com.trim(v)])
		end
	end
	local nbbase=ecps.reversalFormat(tab,ecps.baseNbFm,ecps.baseNbMap)
	nb["base"]=nbbase
	
	
	--股东及出资信息
	--print("股东及出资信息.............")
	local czxxliststr=string.match(content,"var czxxliststr%s*=%'(%[.-%])%';")
	local gdczs={}
	--print("czxxliststr:"..czxxliststr)
	if czxxliststr~="" then
		local czxxlist=json.decode(czxxliststr)
		local changdu=table.getn(czxxlist)
		if changdu>0 then
			for k,v in pairs(czxxlist) do
				local gdcz={}
				gdcz["inv"]=v["inv"]--股东
				gdcz["acConAm"]=v["liacconam"] --实缴出资额
				gdcz["realConDate"]=dealTime(v["accondate"]["time"])--实缴出资时间
				
				local realConFormArr=com.split(v["acconform"],"|")
				gdcz["realConForm"]=realConFormArr[2]--实缴出资方式
				
				
				
				gdcz["subConAm"]=v["lisubconam"]--认缴出资额
				
				
				
				local subconformArr=com.split(v["subconform"],"|")
				--print("subconformArr[2]"..subconformArr[2])
				gdcz["subConAmForm"]=subconformArr[2]--认缴出资方式
				
				gdcz["subConAmDate"]=dealTime(v["subcondate"]["time"])--认缴出资时间
				--print(gdcz["inv"],gdcz["subConAm"],gdcz["subConAmDate"],gdcz["subConAmForm"],gdcz["acConAm"],gdcz["realConDate"],gdcz["realConForm"])
				table.insert(gdczs,gdcz)
			end
		end
	end
	nb["invs"]=gdczs
	
	--企业资产状况信息
	local content2=""
	local list=findListHtml("table.detailsList",content)
	for k,v in pairs(list) do
		local tmp=findOneText("tr:eq(0) th:eq(0)","<table>"..v.."</table>")
		if tmp=="企业资产状况信息" then
			content2=v
		end
	end
	--print(content2)
	
	local qyzc={}
	local th=findListText("tr th","<table>"..content2.."</table>")
	local td=findListText("tr td","<table>"..content2.."</table>")
	for k,v in pairs(th) do
		if k>1 then
			--print(v)
			qyzc[com.trim(v)]=com.trim(td[k-1])
			--print(com.trim(v),com.trim(td[k-1]))
		end
	end
	nb["assetstatus"]=qyzc
	
	
	
	--股权变更信息
	if string.find(content,"var nbgqbgsstr=''")~=nil then
		
	else
		local nbgqbgsstr=string.match(content,"var nbgqbgsstr%s*=%'(%[.-%])%';")
		--print("nbgqbgsstr:"..nbgqbgsstr)
		local bgs={}
		if nbgqbgsstr~="" then
			local gqbglist=json.decode(nbgqbgsstr)
			local changdu=table.getn(gqbglist)
			if changdu>0 then
				for k,v in pairs(gqbglist) do
					local bg={}
					bg["inv"]=v["inv"] --股东
					bg["transbmpr"]=v["transamprpre"] --变更前股权比例
					bg["transampr"]=v["transampraf"] --变更后股权比例				
					bg["altdate"]=dealTime(v["altdate"]["time"]) --股权变更日期
					--print(bg["inv"],bg["transbmpr"],bg["transampr"],bg["altdate"])
					table.insert(bgs, bg)     
				end
			end
		end
		nb["equitychange"]=bgs
	end
	
	--print("年报完成")
	return nb
end


function dealTime(paramTime)
	
	local returnTime=""
	local str=tostring(paramTime)
	--print(str)
	local str2=string.sub(str,0,string.len(str)-3)
	--print(str2)
	local time=tonumber(str2)
	--print(time)
	returnTime=os.date("%Y-%m-%d",time)
	return returnTime
end

function getEntTmp(_csrf,cookies,enckeyword)
	local head={
			["Accept"]="application/json, text/javascript, */*; q=0.01",
			["Accept-Encoding"]="gzip,deflate",
			["Accept-Language"]="zh-CN,zh;q=0.8",
			["Connection"]="keep-alive",
			["X-CSRF-TOKEN"]=_csrf,
			["X-Requested-With"]="XMLHttpRequest",
			["Host"]="218.57.139.24",
			["Origin"]="http://218.57.139.24",
			["Content-Type"]="application/x-www-form-urlencoded; charset=UTF-8",
			["Referer"]="http://218.57.139.24/pub/indsearch",
			["User-Agent"]="Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36",
			--["Cookie"]=cookies
	}
	local param={
		["param"]=com.trim(enckeyword)
	}
	local content,_= downloadAdv("http://218.57.139.24/pub/search","post",param,head,cookies)
	--print("content:",content)
	return content
end

function checkCode(title,_csrf,code,cookies)
	local head={
			["Accept"]="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
			["Accept-Encoding"]="gzip,deflate",
			["Accept-Language"]="zh-CN,zh;q=0.8",
			["Cache-Control"]="no-cache",
			["Host"]="218.57.139.24",
			["Origin"]="http://211.141.74.198:8081",
			["Content-Type"]="application/x-www-form-urlencoded;charset=UTF-8",
			["Referer"]="http://218.57.139.24/",
			["Upgrade-Insecure-Requests"]="1",
			["User-Agent"]="Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36",
			--["Cookie"]=cookies
	}
	local param={
		["kw"]=title,
		["_csrf"]=com.trim(_csrf),
		["secode"]=com.trim(transCode("encodeMd5",code))
	}
	local content,_= downloadAdv("http://218.57.139.24/pub/indsearch","post",param,head,cookies)
	--content=unGzip(content)
	--print("content:",content)
	local enckeyword=string.match(content,"enckeyword=%'(%w*)%'")
	--print("enckeyword:"..enckeyword)
	if enckeyword~=nil and enckeyword~="" then
		return "true",enckeyword
	end
	return "false",nil
end

function getCode(href,cookies)

	local head={
			["Accept"]="image/webp,image/*,*/*;q=0.8",
			["Accept-Encoding"]="gzip,deflate,sdch",
			["Accept-Language"]="zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4",
			["Cache-Control"]="no-cache",
			["Host"]="218.57.139.24",
			["Pragma"]="no-cache",
			["Referer"]="http://218.57.139.24/",
			["User-Agent"]="Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36",
			--["Cookie"]=cookies
	}
	local content,_= downloadAdv("http://218.57.139.24/securitycode","get",{},head,cookies)
	--print("content:",content)
	local code= getEcpsCode("sd",content)
	print("code:"..code)
	return code
	
end

function getSpecialCookies()
	
	local head={
				["Host"]="218.57.139.24",
				["Connection"]="keep-alive",
				["Accept"]="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
				["Accept-Encoding"]="gzip,deflate,sdch",
				["Accept-Language"]="zh-CN,zh;q=0.8",
				["Upgrade-Insecure-Requests"]="1",
				["User-Agent"]="Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"	
		}
		local con,cookies=downloadAdv("http://218.57.139.24/","get",{},head,"")	
		--print("cookies:"..cookies)
--		local rebot_cookieid=""
--		local index=0
--		string.gsub(con,'[0-9a-z]+',function(s)
--			 if string.len(s)>=40 then
--				index=index+1
--				if index==1 then
--					rebot_cookieid=s
--				end
--			 end
--			end
--		)
		--print("rebot_cookieid:",rebot_cookieid)
--		local head2={
--				["Host"]="211.141.74.198:8081",
--				["Connection"]="keep-alive",
--				["Cache-Control"]="no-cache",
--				["Accept"]="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
--				["Accept-Encoding"]="gzip,deflate,sdch",
--				["Accept-Language"]="zh-CN,zh;q=0.8",
--				["Upgrade-Insecure-Requests"]="1",
--				["Referer"]="http://211.141.74.198:8081/aiccips/",
--				["Cookie"]="ROBOTCOOKIEID="..rebot_cookieid,
--				["User-Agent"]="Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36"	
--		}
		
--		local con2,cookies2=downloadAdv("http://211.141.74.198:8081/aiccips/","get",{},head2,"")
		--print("cookies2:",cookies2)
		if cookies~="null" then
			
			--print(con)
			local token=string.match(con,"<meta name=\"_csrf\".*content=\"(.*)\".*/>.*<link")
			--print("token:"..token)
--			local obj2=json.decode(cookies2)
--			local cookies3="ROBOTCOOKIEID="..rebot_cookieid
--			if obj2~=nil then
--				for k,v in pairs(obj2) do
--					cookies3=v["Name"].."="..v["Value"]..";"..cookies3
--				end
--			end
			--print("cookies3:",cookies3)
			return cookies,token
		end
	
end



function strDel(str)
	str=string.gsub(str,"\"","")
	return str
end



function checkData(title,entInfo)
	--print("检查数据....")
	local entName=entInfo["EntName"]
	local regNo=entInfo["RegNo"]
	if entName=="" or entName==nil or title~=entName then
		print(title.."山东数据下载出错或者不完整重新下载")
		return false
	end
	if regNo=="" or regNo==nil then
		print(title.."山东数据下载出错或者不完整重新下载")
		return false
	end
	--print(entInfo["EntName"].."数据完整")
	return true
end