xg

src/spider/download.go

@@ -82,7 +82,7 @@ func Download(downloaderid, url, method string, head map[string]interface{}, enc
 }
 
 //下载页面，发送消息，等待下载
-func DownloadAdv(downloaderid, url, method string, reqparam, head map[string]interface{}, mycookie []*http.Cookie, encoding string, useproxy, ishttps bool, code string, timeout int64) (string, []*http.Cookie) {
+func DownloadAdv(downloaderid, url, method string, reqparam, head map[string]interface{}, mycookie []*http.Cookie, encoding string, useproxy, ishttps bool, code string, timeout int64) (string, []*http.Cookie, map[string]interface{}) {
 	defer mu.Catch()
 	msgid := mu.UUID(8)
 	if len(head) < 1 {
@@ -127,16 +127,17 @@ func DownloadAdv(downloaderid, url, method string, reqparam, head map[string]int
 	tmp := map[string]interface{}{}
 	json.Unmarshal(ret, &tmp)
 	cooks := lu.ParseHttpCookie(tmp["cookie"])
+	headers, _ := tmp["header"].(map[string]interface{})
 	if v, ok := tmp["code"].(string); ok && v == "200" {
 		if isImg {
 			bs, _ := tmp["content"].(string)
-			return string(bs), cooks
+			return string(bs), cooks, headers
 		} else {
 			bs, _ := base64.StdEncoding.DecodeString(tmp["content"].(string))
-			return string(bs), cooks
+			return string(bs), cooks, headers
 		}
 	} else {
-		return "", nil
+		return "", nil, nil
 	}
 }
 

src/spider/script.go

@@ -120,18 +120,21 @@ func (s *Script) LoadScript(code, script_file string, newstate bool) string {
 		json.Unmarshal([]byte(cookie), &mycookie)
 		var ret string
 		var retcookie []*http.Cookie
+		var headers = map[string]interface{}{}
 		if param == nil {
 			ptext := map[string]interface{}{"text": S.ToString(-3)}
-			ret, retcookie = DownloadAdv(s.Downloader, url, method, ptext, util.GetTable(head), mycookie, charset, s.Userproxy, ishttps, s.SCode, s.Timeout)
+			ret, retcookie, headers = DownloadAdv(s.Downloader, url, method, ptext, util.GetTable(head), mycookie, charset, s.Userproxy, ishttps, s.SCode, s.Timeout)
 		} else {
-			ret, retcookie = DownloadAdv(s.Downloader, url, method, util.GetTable(param), util.GetTable(head), mycookie, charset, s.Userproxy, ishttps, s.SCode, s.Timeout)
+			ret, retcookie, headers = DownloadAdv(s.Downloader, url, method, util.GetTable(param), util.GetTable(head), mycookie, charset, s.Userproxy, ishttps, s.SCode, s.Timeout)
 		}
 		S.Push(lua.LString(ret))
 		scookie, _ := json.Marshal(retcookie)
 		S.Push(lua.LString(scookie))
+		hTable := util.MapToLuaTable(S, headers)
+		S.Push(hTable)
 		atomic.AddInt32(&s.ToDayRequestNum, 1)
 		atomic.AddInt32(&s.TotalRequestNum, 1)
-		return 2
+		return 3
 	}))
 	//保存验证错误日志
 	s.L.SetGlobal("saveErrLog", s.L.NewFunction(func(S *lua.LState) int {

src/spider/spider.go

@@ -190,7 +190,8 @@ func (s *Spider) DownloadDetailItem(paramdata, tmp map[string]interface{}) {
 				data["state"] = 3
 				UpdateState(id, data)
 				//下载成功根据href更新spider_highlistdata中state
-				Mgo.Update("spider_highlistdata", map[string]interface{}{"href": tmp["href"]}, map[string]interface{}{"$set": map[string]interface{}{"state": 1}}, false, true)
+				Mgo.Update("spider_highlistdata", map[string]interface{}{"href": tmp["href"]}, map[string]interface{}{"$set": map[string]interface{}{"state": 1, "by": "regather", "updatetime": time.Now().Unix()}}, false, true)
+				Mgo.Update("spider_listdata", map[string]interface{}{"href": tmp["href"]}, map[string]interface{}{"$set": map[string]interface{}{"state": 1, "by": "regather", "updatetime": time.Now().Unix()}}, false, true)
 			} else {
 				logger.Debug("Save Failed：", id, "	", s.Code)
 			}