Merge branch 'master' of http://192.168.3.207:8080/data_processing/DataMaintenance

src/service/repair_service.go

@@ -126,9 +126,12 @@ func (jy *RepairRule) RepairNewSave() {
 	db := HexToBigIntMod(href)
 	hashHref := HexText(href)
 	shaid := Sha(detail)
-	PutRedis("title_repeat_fulljudgement", db, hashHref, mongodb.BsonIdToSId((*updata)["_id"]), -1) // 全量redis
-	PutRedis("shaid", 0, shaid, "", 5184000)                                                        //	增量reids
-	PutRedis("title_repeat_listpagehref", 0, href, "", 3600*24*30*24)                               // 列表页增量redis
+	if !strings.Contains(href, "https://www.jianyu360.cn/") {
+		// 剑鱼链接 不需要存redis
+		PutRedis("title_repeat_fulljudgement", db, hashHref, mongodb.BsonIdToSId((*updata)["_id"]), -1) // 全量redis
+	}
+	PutRedis("shaid", 0, shaid, mongodb.BsonIdToSId((*updata)["_id"]), 5184000) //	增量reids
+	//PutRedis("title_repeat_listpagehref", 0, href, "", 3600*24*30*24)                               // 列表页增量redis
 	if b {
 		log.Println("当前新增id：", mongodb.BsonIdToSId((*updata)["_id"]))
 		//日志记录
@@ -185,6 +188,9 @@ func (jy *RepairRule) RepairPub() {
 		} else if k == "href" && v == "#" {
 			var Url = "https://www.jianyu360.cn/article/content/%s.html"
 			save["href"] = fmt.Sprintf(Url, qu.CommonEncodeArticle("content", mongodb.BsonIdToSId(save["_id"])))
+		} else if k == "detail" {
+			v = strings.Replace(qu.ObjToString(v), "&#60;", "<", -1)
+			v = strings.Replace(qu.ObjToString(v), "&#62;", ">", -1)
 		} else {
 			save[k] = v
 		}
@@ -207,7 +213,10 @@ func (jy *RepairRule) RepairPub() {
 		db := HexToBigIntMod(href)
 		hashHref := HexText(href)
 		shaid := Sha(qu.ObjToString(save["detail"]))
-		PutRedis("title_repeat_fulljudgement", db, hashHref, mongodb.BsonIdToSId(save["_id"]), -1)
+		if !strings.Contains(href, "https://www.jianyu360.cn/") {
+			// 剑鱼链接 不需要存redis
+			PutRedis("title_repeat_fulljudgement", db, hashHref, mongodb.BsonIdToSId(save["_id"]), -1)
+		}
 		PutRedis("shaid", 0, shaid, "", 5184000)
 		log.Println("当前新增id：", mongodb.BsonIdToSId(save["_id"]))
 		//日志记录
@@ -389,6 +398,11 @@ func (jy *RepairRule) RepairSave() {
 			detail = strings.Replace(detail, "&#62;", ">", -1)
 			updata["detail"] = detail
 		}
+		if contenthtml := qu.ObjToString(updata["contenthtml"]); contenthtml != "" {
+			contenthtml = strings.Replace(contenthtml, "&#60;", "<", -1)
+			contenthtml = strings.Replace(contenthtml, "&#62;", ">", -1)
+			updata["contenthtml"] = contenthtml
+		}
 		//变更字段
 		if len(modifyinfo) == 0 {
 			jy.ServeJson(map[string]interface{}{

src/util/config.go

@@ -108,6 +108,7 @@ func initJYRedis() {
 }
 
 func InitOther() {
+	initCitys()
 	initInfoType()
 	initBuyerClass()
 	initScopeClass()
@@ -116,6 +117,33 @@ func InitOther() {
 	initJYRedis()
 }
 
+func initCitys() {
+	//map初始化
+	if ProvinceCitys == nil {
+		ProvinceCitys = make(map[string][]string)
+	}
+	if CityDistricts == nil {
+		CityDistricts = make(map[string][]string)
+	}
+	datas, _ := Mgo.Find("code_city", `{}`, `{"_id":1}`, `{}`, false, -1, -1)
+	for _, data := range *datas {
+		brief := data["brief"].(string)
+		Province = append(Province, brief)
+		citys := data["citys"].([]interface{})
+		citytmp := []string{}
+		for _, city := range citys {
+			cityMap := city.(map[string]interface{})
+			for cname, districts := range cityMap {
+				citytmp = append(citytmp, cname)
+				tmp := districts.([]interface{})
+				districtarr := qu.ObjArrToStringArr(tmp)
+				CityDistricts[cname] = districtarr
+			}
+		}
+		ProvinceCitys[brief] = citytmp
+	}
+}
+
 func initInfoType() {
 	SubTypeMap = make(map[string][]string)
 	datas, _ := Mgo.Find("code_infotype", `{}`, `{"_id":1}`, `{}`, false, -1, -1)