敏感词数据过滤

src/jy/admin/distribution/distribution.go

@@ -27,9 +27,10 @@ func init() {
 		s_table, _ := c.GetPostForm("s_table")
 		instanceid, _ := c.GetPostForm("instanceid")
 		if s_table != "" { //id划段
-			endate, _ := c.GetPostForm("s_date")
-			log.Println(s_table, endate)
-			ecs.IdsRange(s_table, endate)
+			sdate, _ := c.GetPostForm("s_date")
+			edate, _ := c.GetPostForm("e_date")
+			log.Println(s_table, sdate, edate)
+			ecs.IdsRange(s_table, sdate, edate)
 		} else if instanceid != "" { //实例自动释放时间
 			hours, _ := c.GetPostForm("hours")
 			hour := qu.IntAll(hours)

src/jy/admin/task/task.go

@@ -8,6 +8,7 @@ import (
 	. "jy/mongodbutil"
 	"net/http"
 	qu "qfw/util"
+	"strings"
 	"time"
 
 	"github.com/gin-contrib/sessions"
@@ -37,6 +38,13 @@ func init() {
 
 	//新增任务、编辑任务
 	Admin.POST("/task/save", func(c *gin.Context) {
+		//判断存储配置是否完整
+		s_mgosavecoll, _ := c.GetPostForm("s_mgosavecoll")
+		strs := strings.Split(s_mgosavecoll, "/")
+		if len(strs) < 3 {
+			c.JSON(200, gin.H{"rep": false, "msg": "保存表输入地址不正确！"})
+			return
+		}
 		data := GetPostForm(c)
 		_id, _ := c.GetPostForm("_id")
 		b := false

src/jy/cluster/distributed.go

@@ -21,9 +21,9 @@ import (
 var EscIds map[string][]string //id区间
 
 //根据esc数量实例数量id划段
-func IdsRange(table, endate string) int {
-	start := time.Date(2015, 11, 3, 0, 0, 0, 0, time.Local)
-	end, _ := time.ParseInLocation(qu.Date_Short_Layout, endate, time.Local)
+func IdsRange(table, sdate, edate string) int {
+	start, _ := time.ParseInLocation(qu.Date_Short_Layout, sdate, time.Local)
+	end, _ := time.ParseInLocation(qu.Date_Short_Layout, edate, time.Local)
 	EscIds = map[string][]string{}
 	list, _ := db.Mgo.Find("ecs", `{"Status":"Running"}`, nil, nil, false, -1, -1)
 	ids := RangeIdsByDate(len(*list), start, end)
@@ -73,13 +73,13 @@ func RunEcsTask() int {
 }
 
 //id分段
-func RangeIdsByDate(escnum int, start, edate time.Time) map[string][]string {
+func RangeIdsByDate(escnum int, start, end time.Time) map[string][]string {
 	ids := map[string][]string{}
 	task, _ := db.Mgo.FindById("task", qu.ObjToString(ju.Config["udptaskid"]), nil)
 	log.Println(qu.ObjToString((*task)["s_mgoaddr"]), qu.ObjToString((*task)["s_mgodb"]))
 	DB := db.MgoFactory(2, 3, 120, qu.ObjToString((*task)["s_mgoaddr"]), qu.ObjToString((*task)["s_mgodb"]))
-	total := DB.Count("bidding", `{"comeintime":{"$lt":`+fmt.Sprint(edate.Unix())+`}}`)
-	total_back := DB.Count("bidding_back", `{}`)
+	total := DB.Count("bidding", bson.M{"_id": bson.M{"$gte": bson.NewObjectIdWithTime(start), "$lt": bson.NewObjectIdWithTime(end)}})
+	total_back := DB.Count("bidding_back", bson.M{"_id": bson.M{"$gte": bson.NewObjectIdWithTime(start), "$lt": bson.NewObjectIdWithTime(end)}})
 	total += total_back
 	pagesize := (total + escnum - 1) / escnum
 	log.Printf("total:%d total_back:%d pagesize:%d escnum:%d", total, total_back, pagesize, escnum)
@@ -93,8 +93,8 @@ func RangeIdsByDate(escnum int, start, edate time.Time) map[string][]string {
 		for {
 			tmpsid := bson.NewObjectIdWithTime(start)
 			end := start.Add(4 * time.Hour)
-			if end.Unix() > edate.Unix() {
-				eid = bson.NewObjectIdWithTime(edate)
+			if end.Unix() > end.Unix() {
+				eid = bson.NewObjectIdWithTime(end)
 			} else {
 				eid = bson.NewObjectIdWithTime(end)
 			}

src/jy/extract/exportask.go

@@ -49,7 +49,7 @@ func extractAndExport(v string, t map[string]interface{}) {
 		IsEtxLog:    false,
 		ProcessPool: make(chan bool, 5),
 	}
-	e.TaskInfo.DB = db.MgoFactory(1, 3, 120, fmt.Sprint(t["dbaddr"]), fmt.Sprint(t["dbname"]))
+	e.TaskInfo.FDB = db.MgoFactory(1, 3, 120, fmt.Sprint(t["dbaddr"]), fmt.Sprint(t["dbname"]))
 	e.InitRulePres()
 	e.InitRuleBacks()
 	e.InitRuleCore()
@@ -57,7 +57,7 @@ func extractAndExport(v string, t map[string]interface{}) {
 	e.InitClearFn()
 	query := t["query"]
 	limit := qu.IntAll(t["limit"])
-	list, _ := e.TaskInfo.DB.Find(e.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
+	list, _ := e.TaskInfo.FDB.Find(e.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
 	for _, v := range *list {
 		if v["check_sensitive2"] != nil { //去除含敏感词数据
 			if v["check_sensitive2"].(string) != "" {

src/jy/extract/extract.go

@@ -13,7 +13,6 @@ import (
 	"reflect"
 	"regexp"
 	"strconv"
-	"strings"
 	"sync"
 	"time"
 
@@ -27,7 +26,7 @@ var (
 	TaskList  map[string]*ExtractTask                //任务列表
 	saveLimit = 200                                  //抽取日志批量保存
 	PageSize  = 5000                                 //查询分页
-	Fields    = `{"title":1,"detail":1,"contenthtml":1,"href":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"check_sensitive2":1}`
+	Fields    = `{"title":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"check_sensitive2":1}`
 )
 
 //启动测试抽取
@@ -37,7 +36,7 @@ func StartExtractTestTask(taskId, startId, num, resultcoll, trackcoll string) bo
 	ext.Id = taskId
 	ext.IsRun = true
 	ext.InitTestTaskInfo(resultcoll, trackcoll)
-	ext.TaskInfo.DB = db.MgoFactory(1, 3, 120, ext.TaskInfo.FromDbAddr, ext.TaskInfo.FromDB)
+	ext.TaskInfo.FDB = db.MgoFactory(1, 3, 120, ext.TaskInfo.FromDbAddr, ext.TaskInfo.FromDB)
 	ext.InitRulePres()
 	ext.InitRuleBacks()
 	ext.InitRuleCore()
@@ -67,7 +66,7 @@ func RunExtractTestTask(ext *ExtractTask, startId, num string) bool {
 	id := IdTrans(startId)
 	if id.Valid() {
 		query := bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(startId)}}
-		list, _ := ext.TaskInfo.DB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, n)
+		list, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, n)
 		for _, v := range *list {
 			//log.Println(v["_id"])
 			j := PreInfo(v)
@@ -93,7 +92,8 @@ func StartExtractTaskId(taskId string) bool {
 		ext.Id = taskId
 		ext.InitTaskInfo()
 	}
-	ext.TaskInfo.DB = db.MgoFactory(2, 3, 120, ext.TaskInfo.FromDbAddr, ext.TaskInfo.FromDB)
+	ext.TaskInfo.FDB = db.MgoFactory(2, 3, 120, ext.TaskInfo.FromDbAddr, ext.TaskInfo.FromDB)
+	ext.TaskInfo.TDB = db.MgoFactory(1, 3, 120, ext.TaskInfo.ToDbAddr, ext.TaskInfo.ToDB)
 	ext.InitRulePres()
 	ext.InitRuleBacks()
 	ext.InitRuleCore()
@@ -136,7 +136,7 @@ func StopExtractTaskId(taskId string) bool {
 func RunExtractTask(taskId string) {
 	ext := TaskList[taskId]
 	query := bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(ext.TaskInfo.LastExtId)}}
-	count := ext.TaskInfo.DB.Count(ext.TaskInfo.FromColl, query)
+	count := ext.TaskInfo.FDB.Count(ext.TaskInfo.FromColl, query)
 	pageNum := (count + PageSize - 1) / PageSize
 	limit := PageSize
 	if count < PageSize {
@@ -146,7 +146,7 @@ func RunExtractTask(taskId string) {
 	for i := 0; i < pageNum; i++ {
 		query = bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(ext.TaskInfo.LastExtId)}}
 		log.Printf("page=%d,query=%v", i+1, query)
-		list, _ := ext.TaskInfo.DB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
+		list, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
 		for _, v := range *list {
 			if v["check_sensitive2"] != nil { //去除含敏感词数据
 				if v["check_sensitive2"].(string) != "" {
@@ -184,17 +184,6 @@ func PreInfo(doc map[string]interface{}) *ju.Job {
 	detail = ju.CutLableStr(detail)
 	detail = cut.ClearHtml(detail)
 	doc["detail"] = detail
-	href := qu.ObjToString(doc["href"])
-	if strings.HasPrefix(href, "http://") {
-		href = href[7:]
-	} else if strings.HasPrefix(href, "https://") {
-		href = href[8:]
-	}
-	pos := strings.Index(href, "/")
-	if pos > 0 {
-		href = href[:pos]
-	}
-	doc["domain"] = href
 	toptype := qu.ObjToString(doc["toptype"])
 	if qu.ObjToString(doc["type"]) == "bid" {
 		toptype = "结果"
@@ -207,14 +196,14 @@ func PreInfo(doc map[string]interface{}) *ju.Job {
 		Category:   toptype,
 		Content:    qu.ObjToString(doc["detail"]),
 		SpiderCode: qu.ObjToString(doc["spidercode"]),
-		Domain:     qu.ObjToString(doc["domain"]),
-		Href:       qu.ObjToString(doc["href"]),
-		Title:      qu.ObjToString(doc["title"]),
-		Data:       &doc,
-		City:       qu.ObjToString(doc["city"]),
-		Province:   qu.ObjToString(doc["area"]),
-		Result:     map[string][]*ju.ExtField{},
-		BuyerAddr:  qu.ObjToString(doc["buyeraddr"]),
+		//Domain:     qu.ObjToString(doc["domain"]),
+		//Href:       qu.ObjToString(doc["href"]),
+		Title:     qu.ObjToString(doc["title"]),
+		Data:      &doc,
+		City:      qu.ObjToString(doc["city"]),
+		Province:  qu.ObjToString(doc["area"]),
+		Result:    map[string][]*ju.ExtField{},
+		BuyerAddr: qu.ObjToString(doc["buyeraddr"]),
 	}
 	qu.Try(func() {
 		pretreated.AnalyStart(j)
@@ -831,7 +820,8 @@ func AnalysisSaveResult(j *ju.Job, e *ExtractTask) {
 		tmp["winnerorder"] = j.Winnerorder
 	}
 	for k, v := range *doc {
-		if k == "detail" || k == "contenthtml" {
+		//去重冗余字段
+		if k == "detail" || k == "contenthtml" || k == "site" || k == "spidercode" {
 			continue
 		}
 		if tmp[k] == nil {

src/jy/extract/extractInit.go

@@ -37,8 +37,10 @@ type RuleCore struct {
 type TaskInfo struct {
 	Name, Version, VersionId, TrackColl string    //名称、版本、版本id、追踪记录表
 	FromDbAddr, FromDB, FromColl        string    //抽取数据库地址、库名、表名
-	SaveColl, TestColl, LastExtId       string    //抽取结果表、测试结果表、上次抽取信息id
-	DB                                  *db.Pool  //数据库连接池
+	ToDbAddr, ToDB, ToColl              string    //结果数据库地址、库名、表名
+	TestColl, LastExtId                 string    //测试结果表、上次抽取信息id
+	FDB                                 *db.Pool  //数据库连接池
+	TDB                                 *db.Pool  //数据库连接池
 	IsEtxLog                            bool      //是否开启抽取日志
 	ProcessPool                         chan bool //任务进程池
 	TestLua                             bool      //检查测试用
@@ -121,21 +123,29 @@ func (e *ExtractTask) InitTaskInfo() {
 	log.Println("task", task)
 	if len(*task) > 1 {
 		v, _ := db.Mgo.FindOne("version", `{"version":"`+(*task)["s_version"].(string)+`","delete":false}`)
-		e.TaskInfo = &TaskInfo{
-			Name:      (*task)["s_taskname"].(string),
-			Version:   (*task)["s_version"].(string),
-			VersionId: qu.BsonIdToSId((*v)["_id"]),
-			//TrackColl:   (*task)["s_trackcoll"].(string),
-			FromDbAddr:  (*task)["s_mgoaddr"].(string),
-			FromDB:      (*task)["s_mgodb"].(string),
-			FromColl:    (*task)["s_mgocoll"].(string),
-			SaveColl:    (*task)["s_mgosavecoll"].(string),
-			IsEtxLog:    false, //qu.If(qu.IntAll((*task)["i_track"]) == 1, true, false).(bool),
-			LastExtId:   qu.ObjToString((*task)["s_extlastid"]),
-			ProcessPool: make(chan bool, qu.IntAllDef((*task)["i_process"], 1)),
-		}
-		if (*v)["isextractcity"] != nil {
-			e.IsExtractCity = (*v)["isextractcity"].(bool)
+		strs := strings.Split((*task)["s_mgosavecoll"].(string), "/")
+		log.Println("s_mgosavecoll", strs)
+		if len(strs) < 3 {
+			return
+		} else {
+			e.TaskInfo = &TaskInfo{
+				Name:      (*task)["s_taskname"].(string),
+				Version:   (*task)["s_version"].(string),
+				VersionId: qu.BsonIdToSId((*v)["_id"]),
+				//TrackColl:   (*task)["s_trackcoll"].(string),
+				FromDbAddr:  (*task)["s_mgoaddr"].(string),
+				FromDB:      (*task)["s_mgodb"].(string),
+				FromColl:    (*task)["s_mgocoll"].(string),
+				ToDbAddr:    strs[0],
+				ToDB:        strs[1],
+				ToColl:      strs[2],
+				IsEtxLog:    false, //qu.If(qu.IntAll((*task)["i_track"]) == 1, true, false).(bool),
+				LastExtId:   qu.ObjToString((*task)["s_extlastid"]),
+				ProcessPool: make(chan bool, qu.IntAllDef((*task)["i_process"], 1)),
+			}
+			if (*v)["isextractcity"] != nil {
+				e.IsExtractCity = (*v)["isextractcity"].(bool)
+			}
 		}
 		log.Println(e.TaskInfo.Name, "thread:", qu.IntAllDef((*task)["i_process"], 1))
 	} else {
@@ -711,7 +721,7 @@ func (e *ExtractTask) BidSave() {
 			arr := e.BidArr[:500]
 			go func(tmp *[][]map[string]interface{}) {
 				qu.Try(func() {
-					db.Mgo.UpSertBulk(e.TaskInfo.SaveColl, *tmp...)
+					db.Mgo.UpSertBulk(e.TaskInfo.ToColl, *tmp...)
 					<-e.BidChanel
 				}, func(err interface{}) {
 					log.Println(err)
@@ -724,7 +734,7 @@ func (e *ExtractTask) BidSave() {
 			arr := e.BidArr
 			go func(tmp *[][]map[string]interface{}) {
 				qu.Try(func() {
-					db.Mgo.UpSertBulk(e.TaskInfo.SaveColl, *tmp...)
+					db.Mgo.UpSertBulk(e.TaskInfo.ToColl, *tmp...)
 					<-e.BidChanel
 				}, func(err interface{}) {
 					log.Println(err)

src/jy/extract/extractudp.go

@@ -87,7 +87,7 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 	ext := &ExtractTask{}
 	ext.Id = qu.ObjToString(ju.Config["udptaskid"])
 	ext.InitTaskInfo()
-	ext.TaskInfo.DB = db.MgoFactory(2, 3, 120, ext.TaskInfo.FromDbAddr, ext.TaskInfo.FromDB)
+	ext.TaskInfo.FDB = db.MgoFactory(2, 3, 120, ext.TaskInfo.FromDbAddr, ext.TaskInfo.FromDB)
 	ext.InitRulePres()
 	ext.InitRuleBacks()
 	ext.InitRuleCore()
@@ -109,8 +109,8 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 
 	if len(instanceId) > 0 { //分布式抽取进度
 		query := bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(sid), "$lte": bson.ObjectIdHex(eid)}}
-		count1 := ext.TaskInfo.DB.Count(ext.TaskInfo.FromColl, query)
-		count2 := ext.TaskInfo.DB.Count(ext.TaskInfo.FromColl+"_back", query)
+		count1 := ext.TaskInfo.FDB.Count(ext.TaskInfo.FromColl, query)
+		count2 := ext.TaskInfo.FDB.Count(ext.TaskInfo.FromColl+"_back", query)
 		count := count1 + count2
 		pageNum := (count + PageSize - 1) / PageSize
 		limit := PageSize
@@ -133,8 +133,8 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 		for i := startI; i < pageNum; i++ {
 			query = bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(sid), "$lte": bson.ObjectIdHex(eid)}}
 			log.Printf("page=%d,query=%v", i+1, query)
-			if ext.TaskInfo.DB.Count(ext.TaskInfo.FromColl, query) > 0 {
-				list, _ := ext.TaskInfo.DB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
+			if ext.TaskInfo.FDB.Count(ext.TaskInfo.FromColl, query) > 0 {
+				list, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
 				for _, v := range *list {
 					if v["check_sensitive2"] != nil { //去除含敏感词数据
 						if v["check_sensitive2"].(string) != "" {
@@ -154,8 +154,8 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 			}
 			queryback := bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(sidback)}}
 			log.Printf("page=%d,queryback=%v", i+1, queryback)
-			if ext.TaskInfo.DB.Count(ext.TaskInfo.FromColl+"_back", queryback) > 0 {
-				list2, _ := ext.TaskInfo.DB.Find(ext.TaskInfo.FromColl+"_back", queryback, nil, Fields, false, 0, limit)
+			if ext.TaskInfo.FDB.Count(ext.TaskInfo.FromColl+"_back", queryback) > 0 {
+				list2, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl+"_back", queryback, nil, Fields, false, 0, limit)
 				for _, v := range *list2 {
 					if v["check_sensitive2"] != nil { //去除含敏感词数据
 						if v["check_sensitive2"].(string) != "" {
@@ -181,17 +181,29 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 		}
 	} else { //普通抽取
 		query := bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(sid), "$lte": bson.ObjectIdHex(eid)}}
-		list, _ := ext.TaskInfo.DB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, -1, -1)
-		for _, v := range *list {
-			if v["check_sensitive2"] != nil { //去除含敏感词数据
-				if v["check_sensitive2"].(string) != "" {
-					continue
+		count := ext.TaskInfo.FDB.Count(ext.TaskInfo.FromColl, query)
+		pageNum := (count + PageSize - 1) / PageSize
+		limit := PageSize
+		if count < PageSize {
+			limit = count
+		}
+		for i := 0; i < pageNum; i++ {
+			query = bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(sid)}}
+			log.Printf("page=%d,query=%v", i+1, query)
+			list, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
+			for _, v := range *list {
+				if v["check_sensitive2"] != nil { //去除含敏感词数据
+					if v["check_sensitive2"].(string) != "" {
+						continue
+					}
 				}
+				//log.Println(v["_id"])
+				j := PreInfo(v)
+				ext.TaskInfo.ProcessPool <- true
+				go ext.ExtractProcess(j)
+				sid = qu.BsonIdToSId(v["_id"])
 			}
-			//log.Println(v["_id"])
-			j := PreInfo(v)
-			ext.TaskInfo.ProcessPool <- true
-			go ext.ExtractProcess(j)
+
 		}
 	}
 }

src/jy/util/script.go

@@ -172,8 +172,12 @@ func LuaTableToMap(param *lua.LTable) map[string]interface{} {
 			} else {
 				tmp[kk] = false
 			}
+		} else if kk == "comeintime" || kk == "publishtime" {
+			tmp[kk] = qu.Int64All(val.String())
+		} else if v, ok := val.(*lua.LNumber); ok {
+			tmp[kk] = qu.Float64All(v.String())
 		} else {
-			tmp[kk] = v
+			tmp[kk] = val
 		}
 	})
 	return tmp

src/web/templates/admin/distribution.html

@@ -132,7 +132,8 @@ $(function () {
                   _tit="id划段"
     				tag=[
     					{label:"表名",s_label:"s_table",placeholder:"信息表名",must:true},
-                        {label:"截止日期",s_label:"s_date",placeholder:"2019-01-01",must:true},
+                        {label:"开始日期",s_label:"s_date",placeholder:"2018-01-01",must:true},
+                        {label:"截止日期",s_label:"e_date",placeholder:"2019-01-01",must:true},
     				]   
                 }else if (n=="releasetime"){
                     _tit="释放设置"

src/web/templates/admin/task_list.html

@@ -96,10 +96,10 @@ $(function () {
 				/*表单*/
 				addtask=[
 					{label:"任务名称",s_label:"s_taskname",placeholder:"剑鱼抽取",must:true},
-					{label:"数据库连接",s_label:"s_mgoaddr",must:true},
-					{label:"数据库",s_label:"s_mgodb",must:true},
-					{label:"数据库表",s_label:"s_mgocoll",must:true},
-					{label:"保存表",s_label:"s_mgosavecoll",must:true},
+					{label:"源库连接",s_label:"s_mgoaddr",must:true},
+					{label:"源数据库",s_label:"s_mgodb",must:true},
+					{label:"源表",s_label:"s_mgocoll",must:true},
+					{label:"保存表",s_label:"s_mgosavecoll",placeholder:"127.0.0.1:27080/extract/bidding(数据库地址/数据库/表)",must:true},
 					{label:"描述",s_label:"s_descript",type:"tpl_text"},
 					/*
 					{label:"是否追踪",s_label:"i_track",type:"tpl_list_local",must:true,list:[{"s_name":"是","_id":1},{"s_name":"否","_id":0}],default:0,fun:function(){

udps/main.go

@@ -24,16 +24,15 @@ func main() {
 	//2018-06-01,2019-02-20
 	flag.StringVar(&startDate, "start", "", "开始日期2006-01-02")
 	flag.StringVar(&endDate, "end", "", "结束日期2006-01-02")
-	flag.StringVar(&ip, "ip", "127.0.0.1", "dup端口")
+	flag.StringVar(&ip, "ip", "127.0.0.1", "ip")
 	flag.StringVar(&port, "port", "", "dup端口")
 	flag.StringVar(&stype, "stype", "", "stype")
 	flag.Parse()
-	log.Println(startDate, endDate, ip, port)
+	log.Println(startDate, endDate, ip, port, stype)
 	start, _ := time.ParseInLocation(qu.Date_Short_Layout, startDate, time.Local)
 	end, _ := time.ParseInLocation(qu.Date_Short_Layout, endDate, time.Local)
 	sid := bson.NewObjectIdWithTime(start)
 	eid := bson.NewObjectIdWithTime(end)
-	log.Println(sid, eid)
 	udpclient = mu.UdpClient{Local: ":1470", BufSize: 1024}
 	udpclient.Listen(processUdpMsg)
 	by, _ := json.Marshal(map[string]interface{}{