xg

src/jy/extract/clearesult.go

@@ -19,7 +19,7 @@ func (c *ClearTask) ClearProcess(doc *map[string]interface{}) {
 		kvMap := make(map[string][]map[string]interface{})
 		tmpmaparr := make(map[string]interface{})
 		set := make(map[string]interface{})
-		if len(*data) > 0 {
+		if data != nil && len(*data) > 0 {
 			list := (*data)[0]["list"].([]interface{})
 			listArr := qu.ObjArrToMapArr(list)
 			kvMap["list"] = listArr
@@ -103,27 +103,25 @@ func (c *ClearTask) UpdateResultVal() {
 	c.ClearChannel = make(chan bool, 5)
 	c.UpdateResult = [][]map[string]interface{}{}
 	for {
-		if len(c.UpdateResult) > 2 {
+		if len(c.UpdateResult) > 500 {
 			c.ClearChannel <- true
-			arr := c.UpdateResult[:2]
+			arr := c.UpdateResult[:500]
 			go func(tmp *[][]map[string]interface{}) {
 				qu.Try(func() {
 					db.Mgo.UpdateBulk(c.ClearTaskInfo.FromColl, *tmp...)
-					//db.Mgo.UpSertBulk("extract_result", *tmp...)
 					<-c.ClearChannel
 				}, func(err interface{}) {
 					log.Println(err)
 					<-c.ClearChannel
 				})
 			}(&arr)
-			c.UpdateResult = c.UpdateResult[2:]
+			c.UpdateResult = c.UpdateResult[500:]
 		} else {
 			c.ClearChannel <- true
 			arr := c.UpdateResult
 			go func(tmp *[][]map[string]interface{}) {
 				qu.Try(func() {
 					db.Mgo.UpdateBulk(c.ClearTaskInfo.FromColl, *tmp...)
-					//db.Mgo.UpSertBulk("extract_result", *tmp...)
 					<-c.ClearChannel
 				}, func(err interface{}) {
 					log.Println(err)