新增站点管理

src/config.json

@@ -3,6 +3,7 @@
     "dbaddr": "192.168.3.207:27092",
     "dbname": "editor",
     "dbname2": "spider",
+    "sitecoll": "site",
     "udport": 1499,
     "udpaddr": "127.0.0.1",
     "localudport": ":1498",
@@ -38,6 +39,9 @@
     	"keyword":"(抽签|中标|招标|成交|合同|中标候选人|资格预审|拟建|邀请|询价|比选|议价|竞价|磋商|采购|招投标|答疑|变更公告|更正公告|竞争性谈判|竞谈|意见征询|澄清|单一来源|流标|废标|验收公告|中止|终止|违规|处罚|征集公告|开标结果|评审结果|监理|招租|租赁|评判结果|项目|遴选|补遗|竞标|征求意见|标段|定点结果|项目评审公示|采购项目违规|采购活动中违规|项目行政处罚|采购行政处罚|项目审批公示)",
     	"notkeyword":"(招聘|拍卖|出租|出让|使用权|资产)"
     },
+    "sitetype": [
+        "政府采购","公共资源","人民政府网","政府门户","学校","银行","保险","证券","医疗机构","代理机构","企业门户/平台","工程建设类","社会采购类","在线审批平台","军队","其他"
+    ],
     "model": {
         "bid": {
             "type": "公告类型",

src/front/site.go

@@ -0,0 +1,239 @@
+package front
+
+import (
+	"fmt"
+	"io/ioutil"
+	mgo "mongodb"
+	qu "qfw/util"
+	"sort"
+	sp "spiderutil"
+	"strings"
+	"time"
+	"util"
+
+	"github.com/go-xweb/xweb"
+	"github.com/tealeg/xlsx"
+	//"github.com/tealeg/xlsx"
+)
+
+type Site struct {
+	*xweb.Action
+	loadIndex     xweb.Mapper `xweb:"/center/site/sitelist"`      //控制中心
+	saveSite      xweb.Mapper `xweb:"/center/site/savesite"`      //新增站点
+	importFile    xweb.Mapper `xweb:"/center/site/importfile"`    //导入站点
+	deleteSite    xweb.Mapper `xweb:"/center/site/delete"`        //删除站点
+	getLuas       xweb.Mapper `xweb:"/center/site/getluas/(.*)"`  //删除站点
+	getLuasBySite xweb.Mapper `xweb:"/center/site/getluasbysite"` //删除站点
+}
+
+func (s *Site) LoadIndex() {
+	if s.Method() == "POST" {
+		platform := s.GetString("platform")
+		event := s.GetString("event")
+		sitestype := s.GetString("sitestype")
+		area := s.GetString("area")
+		query := map[string]interface{}{"delete": false}
+		if platform != "-1" && platform != "" {
+			query["platform"] = map[string]interface{}{"$regex": platform}
+		}
+		if event != "-1" && event != "" {
+			query["event"] = map[string]interface{}{"$regex": event}
+		}
+		if sitestype != "-1" && sitestype != "" {
+			query["site_type"] = sitestype
+		}
+		if area != "-1" && area != "" {
+			query["area"] = area
+		}
+		start, _ := s.GetInteger("start")
+		limit, _ := s.GetInteger("length")
+		draw, _ := s.GetInteger("draw")
+		searchStr := s.GetString("search[value]")
+		search := strings.TrimSpace(searchStr)
+		if search != "" {
+			query["$or"] = []interface{}{
+				map[string]interface{}{"site": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"industry": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"city": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"district": map[string]interface{}{"$regex": search}},
+			}
+		}
+		sort := `{"%s":%d}`
+		orderIndex := s.GetString("order[0][column]")
+		orderName := s.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
+		orderType := 1
+		if s.GetString("order[0][dir]") != "asc" {
+			orderType = -1
+		}
+		sort = fmt.Sprintf(sort, orderName, orderType)
+		sites, _ := util.MgoE.Find(sp.Config.SiteColl, query, sort, ``, false, start, limit)
+		count := util.MgoE.Count(sp.Config.SiteColl, query)
+		for _, s := range *sites {
+			remarktime := qu.Int64All(s["remarktime"])
+			s["remarktime"] = qu.FormatDateByInt64(&remarktime, qu.Date_Full_Layout)
+		}
+		s.ServeJson(map[string]interface{}{"draw": draw, "data": sites, "recordsFiltered": count, "recordsTotal": count})
+	} else {
+		events := []string{}
+		for k, _ := range sp.Config.Uploadevents {
+			events = append(events, k)
+		}
+		sort.Strings(events)
+		s.T["events"] = events
+		s.T["sitetypes"] = sp.Config.SiteType
+		s.T["areas"] = util.Area
+		s.Render("sitelist.html", &s.T)
+	}
+}
+
+func (s *Site) SaveSite() {
+	defer qu.Catch()
+	id := s.GetString("id")
+	site := s.GetString("site")
+	if id == "" { //新增
+		data, _ := util.MgoE.FindOne(sp.Config.SiteColl, `{"site":"`+site+`"}`)
+		if len(*data) > 0 {
+			s.ServeJson(map[string]interface{}{"ok": false})
+			return
+		}
+	}
+	othername := s.GetString("othername")
+	domain := s.GetString("domain")
+	area := s.GetString("area")
+	city := s.GetString("city")
+	district := s.GetString("district")
+	industry := s.GetString("industry")
+	sitetype := s.GetString("sitetype")
+	psite := s.GetString("psite")
+	ssite := s.GetString("ssite")
+	tmpdomain, status, event, platform, remarktime := util.GetLuasInfoBySite(site)
+	if domain == "" {
+		domain = tmpdomain
+	}
+	update := map[string]interface{}{
+		"site":         site,
+		"another_name": othername,
+		"domain":       domain,
+		"area":         area,
+		"city":         city,
+		"district":     district,
+		"industry":     industry,
+		"site_type":    sitetype,
+		"p_site":       psite,
+		"s_site":       ssite,
+	}
+	b := false
+	if id == "" { //新增
+		update["spider_status"] = status
+		update["event"] = event
+		update["remarktime"] = remarktime
+		update["platform"] = platform
+		update["delete"] = false
+		update["comeintime"] = time.Now().Unix()
+		if id := util.MgoE.Save(sp.Config.SiteColl, update); id != "" {
+			b = true
+		} else {
+			b = false
+		}
+	} else { //更新
+		update["updatetime"] = time.Now().Unix()
+		b = util.MgoE.Update(sp.Config.SiteColl, map[string]interface{}{"_id": mgo.StringTOBsonId(id)}, map[string]interface{}{"$set": update}, true, false)
+	}
+	s.ServeJson(map[string]interface{}{"ok": b})
+}
+
+func (s *Site) ImportFile() {
+	defer qu.Catch()
+	msg := ""
+	file, _, err := s.GetFile("xlsx")
+	if err != nil {
+		msg = "导入文件错误"
+	} else {
+		binary, _ := ioutil.ReadAll(file)
+		xls, _ := xlsx.OpenBinary(binary)
+		sheet := xls.Sheets[0]
+		for i, r := range sheet.Rows {
+			if i != 0 {
+				site := r.Cells[0].Value
+				if site == "" {
+					msg = "站点名称为空"
+					break
+				}
+				data, _ := util.MgoE.FindOne(sp.Config.SiteColl, `{"site":"`+site+`"}`)
+				if len(*data) > 0 {
+					msg = site + "：站点已存在"
+					break
+				}
+				another_name := r.Cells[1].Value
+				domain := r.Cells[2].Value
+				area := r.Cells[3].Value
+				city := r.Cells[4].Value
+				district := r.Cells[5].Value
+				industry := r.Cells[6].Value
+				site_type := r.Cells[7].Value
+				p_site := r.Cells[8].Value
+				s_site := r.Cells[9].Value
+				tmpdomain, status, event, platform, remarktime := util.GetLuasInfoBySite(site)
+				if domain == "" {
+					domain = tmpdomain
+				}
+				save := map[string]interface{}{
+					"site":          site,
+					"another_name":  another_name,
+					"domain":        domain,
+					"area":          area,
+					"city":          city,
+					"district":      district,
+					"industry":      industry,
+					"site_type":     site_type,
+					"p_site":        p_site,
+					"s_site":        s_site,
+					"spider_status": status,
+					"event":         event,
+					"remarktime":    remarktime,
+					"platform":      platform,
+					"delete":        false,
+					"comeintime":    time.Now().Unix(),
+				}
+				if id := util.MgoE.Save(sp.Config.SiteColl, save); id == "" {
+					msg = "导入失败"
+				}
+			}
+		}
+	}
+	s.ServeJson(msg)
+}
+
+func (s *Site) DeleteSite() {
+	defer qu.Catch()
+	id := s.GetString("id")
+	set := map[string]interface{}{
+		"$set": map[string]interface{}{
+			"delete":     true,
+			"deletetime": time.Now().Unix(),
+		},
+	}
+	if util.MgoE.UpdateById(sp.Config.SiteColl, id, set) {
+		s.ServeJson(map[string]interface{}{"d": true})
+	} else {
+		s.ServeJson(map[string]interface{}{"d": false})
+	}
+}
+
+func (s *Site) GetLuas(site string) {
+	defer qu.Catch()
+	s.T["site"] = site
+	s.Render("sitelualist.html", &s.T)
+
+}
+
+func (s *Site) GetLuasBySite() {
+	site := s.GetString("site")
+	qu.Debug(site)
+	luas, _ := util.MgoE.Find("luaconfig", `{"param_common.1":"`+site+`"}`, ``, ``, false, -1, -1)
+	for _, l := range *luas {
+		modifytime := qu.Int64All(l["modifytime"])
+		l["modifytime"] = qu.FormatDateByInt64(&modifytime, qu.Date_Full_Layout)
+	}
+	s.ServeJson(map[string]interface{}{"data": luas})
+}

src/main.go

@@ -9,7 +9,7 @@ import (
 	//. "luaweb/task"
 	"luaerrdata"
 	"taskManager"
-	"tomail"
+	"timetask"
 	"udp"
 
 	"net/http"
@@ -51,7 +51,7 @@ func init() {
 	//初始化区域信息
 	u.InitAreaCity()
 	//初始化mgo
-	//u.InitMgo()
+	u.InitMgo()
 	//xweb框架配置
 	xweb.Config.RecoverPanic = true
 	xweb.Config.Profiler = true
@@ -64,6 +64,7 @@ func init() {
 	xweb.RootApp().AppConfig.Mode = xweb.Product
 	xweb.RootApp().AppConfig.CacheTemplates = false
 	xweb.AddAction(&front.Front{})
+	xweb.AddAction(&front.Site{})
 	xweb.AddAction(&taskManager.TaskM{})
 	xweb.AddAction(&quesManager.QuesM{})
 	xweb.AddAction(&luaerrdata.ErrorData{})
@@ -98,7 +99,7 @@ func main() {
 	//定时查询任务发送邮件
 	//go tomail.SendToMail()
 	//定时任务
-	go tomail.TimeTask()
+	go timetask.TimeTask()
 	//提供接口，接收其他数据
 	http.HandleFunc("/spider/infos", func(w http.ResponseWriter, req *http.Request) {
 		data := req.FormValue("data")

src/tomail/sendmail.go

@@ -1,157 +0,0 @@
-package tomail
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/smtp"
-	"qfw/util"
-	mgdb "qfw/util/mongodb"
-	sp "spiderutil"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/cron"
-
-	"gopkg.in/mgo.v2/bson"
-	. "gopkg.in/mgo.v2/bson"
-)
-
-var Mail map[string]interface{}
-
-func TimeTask() {
-	defer util.Catch()
-	c := cron.New()
-	c.Start()
-	c.AddFunc("0 20 9 ? * MON-FRI", CheckCreateTask)
-	c.AddFunc("0 0 */1 ? * *", CheckLuaMove)
-	c.Start()
-}
-
-//监测爬虫由历史转增量时未成功的
-func CheckLuaMove() {
-	defer util.Catch()
-	util.Debug("开始检测爬虫节点移动...")
-	query := map[string]interface{}{
-		"comeintime": map[string]interface{}{
-			"$gte": time.Now().Add(-(time.Hour * 1)).Unix(),
-			"$lte": time.Now().Unix(),
-		},
-		"ok": false,
-	}
-	util.Debug("query:", query)
-	list := *mgdb.Find("luamovelog", query, nil, nil, false, -1, -1)
-	text := ""
-	if len(list) > 0 {
-		for _, l := range list {
-			stype := util.ObjToString(l["type"])
-			code := util.ObjToString(l["code"])
-			text += code + "：" + stype + "；"
-		}
-	}
-	if text != "" {
-		for i := 1; i <= 3; i++ {
-			res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", sp.Config.JkMail["api"], sp.Config.JkMail["to"], "lua-move-fail", text))
-			if err == nil {
-				res.Body.Close()
-				read, err := ioutil.ReadAll(res.Body)
-				util.Debug("邮件发送：", string(read), err)
-				break
-			}
-		}
-	}
-}
-
-//检测创建任务失败的爬虫
-func CheckCreateTask() {
-	defer util.Catch()
-	util.Debug("开始检测任务创建...")
-	query := map[string]interface{}{
-		"comeintime": map[string]interface{}{
-			"$gte": GetTime(0),
-		},
-	}
-	codes := []string{}
-	list := *mgdb.Find("luacreatetaskerr", query, nil, nil, false, -1, -1)
-	if len(list) > 0 {
-		for _, l := range list {
-			code := util.ObjToString(l["code"])
-			codes = append(codes, code)
-		}
-	}
-	if len(codes) > 0 {
-		for i := 1; i <= 3; i++ {
-			res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", sp.Config.JkMail["api"], sp.Config.JkMail["to"], "lua-createtask-err", "爬虫："+strings.Join(codes, "；")))
-			if err == nil {
-				res.Body.Close()
-				read, err := ioutil.ReadAll(res.Body)
-				util.Debug("邮件发送：", string(read), err)
-				break
-			}
-		}
-	}
-}
-
-func SendToMail() {
-	mailInfo := *(util.ObjToMap(Mail["smtp"]))
-	host := mailInfo["host"].(string)
-	from := mailInfo["from"].(string)
-	pwd := mailInfo["password"].(string)
-	subject := mailInfo["subject"].(string)
-
-	hour := time.Now().Hour()
-	if hour == 8 {
-		//定时查询数据库 查询需要发邮件的人和相关信息
-		timeStr := time.Now().Format("2006-01-02")
-		the_time, _ := time.ParseInLocation("2006-01-02", timeStr, time.Local)
-		time_zero := the_time.Unix()         //当日凌晨的时间戳
-		time_twentyFour := time_zero + 86399 //当日24时的时间戳
-		//聚合查询数据
-		//mgdb.InitMongodbPool(5, "192.168.3.207:27080", "editor")
-		sess := mgdb.GetMgoConn()
-		defer mgdb.DestoryMongoConn(sess)
-		var res []M
-		sess.DB("editor").C("task").Pipe([]M{M{"$match": M{"l_complete": M{"$gte": time_zero, "$lte": time_twentyFour}, "i_state": M{"$gte": 1, "$lte": 2}}},
-			M{"$group": M{"_id": "$s_modifyid", "count": M{"$sum": 1}}}}).All(&res)
-		//遍历数据进行发邮件
-		for _, v := range res {
-			_id, ok := v["_id"].(string)
-			if ok {
-				query := bson.M{
-					"_id": bson.ObjectIdHex(_id),
-				}
-				user := *mgdb.FindOne("user", query)
-				if user["s_email"] == nil {
-					continue
-				}
-				num := strconv.Itoa(v["count"].(int))
-				body := `<html><body><h3>你有` + num + `条任务需要今天完成</h3></body></html>`
-				hp := strings.Split(host, ":")
-				auth := smtp.PlainAuth("", from, pwd, hp[0])
-				content_type := "Content-Type: text/html; charset=UTF-8"
-				msg := []byte("To: " + user["s_email"].(string) + "\r\nFrom: " + from + "\r\nSubject: " + subject + "\r\n" + content_type + "\r\n\r\n" + body)
-				send_to := strings.Split(user["s_email"].(string), ";")
-				err := smtp.SendMail(host, auth, from, send_to, msg)
-				if err == nil {
-					log.Println(user["s_email"].(string), "  sendMail   success")
-				} else {
-					log.Println(user["s_email"].(string), "  sendMail   fail")
-				}
-			}
-		}
-		time.Sleep(1 * time.Hour)
-	}
-	//time.AfterFunc(30*time.Minute, func() { SendToMail(to, num) })
-	time.AfterFunc(30*time.Minute, SendToMail)
-}
-
-//获取第day天凌晨的时间戳
-func GetTime(day int) int64 {
-	defer util.Catch()
-	nowTime := time.Now().AddDate(0, 0, day)
-	timeStr := util.FormatDate(&nowTime, util.Date_Short_Layout)
-	t, _ := time.ParseInLocation(util.Date_Short_Layout, timeStr, time.Local)
-	return t.Unix()
-}

src/util/util.go

@@ -20,6 +20,9 @@ var (
 	CodeTypeReg     = regexp.MustCompile(`(utf8|utf-8|gbk)`)
 	TitleFilterReg1 = regexp.MustCompile(`[\p{Han}]`)
 	TitleFilterReg2 = regexp.MustCompile(`((上|下)一页|阅读次数)`)
+	Area            []string //省份
+	DomainReg       = regexp.MustCompile(`(?://).+?(?:[:：/])`)
+	ReplaceReg      = regexp.MustCompile(`[]:：/]+`)
 	CheckText       = `item["spidercode"]="%s";item["site"]="%s";item["channel"]="%s"`
 	JsonDataMap     = map[string]bool{ //jsondata
 		"extweight":          true,
@@ -88,6 +91,12 @@ func InitMgo() {
 //初始化省市行政区划信息
 func InitAreaCity() {
 	qu.ReadConfig("areacity.json", &Province)
+	for area, _ := range Province {
+		if area == "全国" {
+			continue
+		}
+		Area = append(Area, area)
+	}
 }
 
 //爬虫整体测试时校验爬虫代码
@@ -188,3 +197,56 @@ func SpiderPassCheckListAndDetail(list []map[string]interface{}, data map[string
 	}
 	return strings.Join(msg, ",")
 }
+
+func GetLuasInfoBySite(site string) (domain, status, event, platform string, remarktime int64) {
+	shelveUp := 0
+	eventMap, platformMap := map[int]interface{}{}, map[string]interface{}{}
+	eventArr, platformArr := []string{}, []string{}
+	domainMap := map[string]int{}
+	remarktime = time.Now().Unix()
+	luas, _ := MgoE.Find("luaconfig", `{"param_common.1":"`+site+`"}`, ``, `{"event":1,"state":1,"platform":1,"param_common":1,"comeintime":1}`, false, -1, -1)
+	for _, l := range *luas {
+		//remarktime
+		if comeintime := qu.Int64All(l["comeintime"]); comeintime != int64(0) && comeintime < remarktime {
+			remarktime = comeintime
+		}
+		//domain
+		paramCommon := l["param_common"].([]interface{})
+		if len(paramCommon) >= 12 {
+			href := qu.ObjToString(paramCommon[11])
+			domain := DomainReg.FindString(href)
+			if domain != "" {
+				domain = ReplaceReg.ReplaceAllString(domain, "")
+				domainMap[domain] = domainMap[domain] + 1
+			}
+		}
+		//state、event、platform
+		state := qu.IntAll(l["state"])
+		event := qu.IntAll(l["event"])
+		platform := qu.ObjToString(l["platform"])
+		if state == 5 {
+			shelveUp++
+		}
+		eventMap[event] = true
+		platformMap[platform] = true
+	}
+	//
+	for e, _ := range eventMap {
+		eventArr = append(eventArr, fmt.Sprint(e))
+	}
+	event = strings.Join(eventArr, ",")
+	for p, _ := range platformMap {
+		platformArr = append(platformArr, p)
+	}
+	platform = strings.Join(platformArr, ",")
+	//
+	n := 0
+	for tmpDomain, num := range domainMap {
+		if num > n {
+			n = num
+			domain = tmpDomain
+		}
+	}
+	status = fmt.Sprintf("%d%s%d", shelveUp, "/", len(*luas))
+	return
+}

src/web/templates/head.html

@@ -556,6 +556,7 @@
 			<li data="index_rwgl"><a href="/center/managerTask"><i class="glyphicon glyphicon glyphicon-tasks"></i> <span>任务管理</span></a></li>
 			<li data="index_fbgl"><a href="/center/lualist.html"><i class="glyphicon glyphicon-credit-card"></i> <span>运行监控中心</span></a></li>
       <li data="index_pcdr"><a href="/center/importdata"><i class="glyphicon glyphicon-import"></i> <span>爬虫导入</span></a></li>
+      <li data="index_site"><a href="/center/site/sitelist"><i class="glyphicon glyphicon-home"></i> <span>站点列表</span></a></li>
       <li data="index_pcqy"><a href="/center/luamove"><i class="glyphicon glyphicon-move"></i> <span>爬虫迁移管理</span></a></li>
 		{{else if eq (session "auth") 2}}
 			<li data="index"><a href="/center"><i class="glyphicon glyphicon-eye-open"></i> <span>爬虫审核</span></a></li>

src/web/templates/sitelist.html

@@ -0,0 +1,527 @@
+{{include "head.html"}}
+<div class="modal fade" id="editsite" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <div class="modal-header">
+                    <button type="button" class="close" data-dismiss="modal" aria-hidden="true" style="margin-top: -19px">&times;</button>
+                    <div class="edit-form">
+                        <div class="edit-info">
+                            <span class="glyphicon glyphicon-edit" aria-hidden="true" style="margin-left: 20px;"></span>
+                            <span class="info">编辑站点</span>
+                        </div>
+                        <form class="form-horizontal" role="form">
+                            <div class="form-group">
+                                <label for="site" class="col-sm-2 control-label">网站名称:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="edit-site" placeholder="请输入网站名称">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="othername" class="col-sm-2 control-label">网站别名:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="edit-othername" placeholder="请输入网站别名">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="domain" class="col-sm-2 control-label">网站域名:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="edit-domain" placeholder="请输入网站域名">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="area" class="col-sm-2 control-label">省份:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="edit-area" placeholder="请输入省份">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="city" class="col-sm-2 control-label">城市:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="edit-city" placeholder="请输入城市">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="district" class="col-sm-2 control-label">区/县:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="edit-district" placeholder="请输入区/县">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="industry" class="col-sm-2 control-label">行业:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="edit-industry" placeholder="请输入行业">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="sitetype" class="col-sm-2 control-label">网站类型:</label>
+                                <div class="col-sm-10">
+                                    <select class="form-control" id="edit-sitetype"></select>
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="psite" class="col-sm-2 control-label">父站:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="edit-psite" placeholder="请输入父站">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="ssite" class="col-sm-2 control-label">子站:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="edit-ssite" placeholder="请输入子站">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <div class="col-sm-offset-2 col-sm-10 operateStyle">
+                                    <input type="button" id="saveSiteId" onclick="updateSite()" class="btn btn-primary" value="保存">
+                                    <input type="button" onclick="cancelAdd()" class="btn btn-default" value="取消">
+                                </div>
+                            </div>
+                        </form>
+                    </div>
+                </div>
+            </div>
+        </div><!-- /.modal-content -->
+    </div><!-- /.modal -->
+</div>
+<div class="modal fade" id="addsite" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <div class="modal-header">
+                    <button type="button" class="close" data-dismiss="modal" aria-hidden="true" style="margin-top: -19px">&times;</button>
+                    <div class="edit-form">
+                        <div class="edit-info">
+                            <span class="glyphicon glyphicon-save" aria-hidden="true" style="margin-left: 20px;"></span>
+                            <span class="info">新增站点</span>
+                        </div>
+                        <form class="form-horizontal" role="form">
+                            <div class="form-group">
+                                <label for="site" class="col-sm-2 control-label">网站名称:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="site" placeholder="请输入网站名称">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="othername" class="col-sm-2 control-label">网站别名:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="othername" placeholder="请输入网站别名">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="domain" class="col-sm-2 control-label">网站域名:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="domain" placeholder="请输入网站域名">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="area" class="col-sm-2 control-label">省份:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="area" placeholder="请输入省份">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="city" class="col-sm-2 control-label">城市:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="city" placeholder="请输入城市">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="district" class="col-sm-2 control-label">区/县:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="district" placeholder="请输入区/县">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="industry" class="col-sm-2 control-label">行业:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="industry" placeholder="请输入行业">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="sitetype" class="col-sm-2 control-label">网站类型:</label>
+                                <div class="col-sm-10">
+                                    <select class="form-control" id="sitetype"></select>
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="psite" class="col-sm-2 control-label">父站:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="psite" placeholder="请输入父站">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label for="ssite" class="col-sm-2 control-label">子站:</label>
+                                <div class="col-sm-10">
+                                    <input type="text" class="form-control" id="ssite" placeholder="请输入子站">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <div class="col-sm-offset-2 col-sm-10 operateStyle">
+                                    <input type="button" onclick="saveSite('')" class="btn btn-primary" value="保存">
+                                    <input type="button" onclick="cancelAdd()" class="btn btn-default" value="取消">
+                                </div>
+                            </div>
+                        </form>
+                    </div>
+                </div>
+            </div>
+        </div><!-- /.modal-content -->
+    </div><!-- /.modal -->
+</div>
+<div class="content-wrapper">
+<section class="content-header">
+   <h1>
+     站点列表
+     <small>
+<!--			{{if gt (session "auth") 2}}-->
+<!--			<a class="btn btn-default btn-sm" href="/center/spider">新建爬虫</a>-->
+         <a class="btn btn-default btn-sm" id="import" onclick="importSite()">导入</a>
+         <a class="btn btn-default btn-sm" id="add" onclick="addSite()">新增</a>
+<!--			{{end}}-->
+         <iframe srcdoc="<form id='uploadform' method='post' enctype='multipart/form-data' action='/center/site/importf
+				ile'><input type='file' name='xlsx' /></form>" height=0 scrolling=no class="hide"  id="fileframe">
+         </iframe>
+	 </small>
+   </h1>
+   <ol class="breadcrumb">
+     <li><a href="#"><i class="fa fa-dashboard"></i> 站点</a></li>
+     <li class="active">列表</li>
+   </ol>
+ </section>
+ <section class="content">
+	<div class="box">
+            <!-- /.box-header -->
+            <div class="box-body">
+              <table id="sitelist" class="table table-bordered table-striped">
+                <thead>
+                <tr>
+                    <th>站点</th>
+                    <th>别名</th>
+                    <th>域名</th>
+                    <th>省份</th>
+                    <th>城市</th>
+                    <th>区县</th>
+                    <th>行业</th>
+                    <th>类型</th>
+                    <th>父站</th>
+                    <th>子站</th>
+                    <th>网站状态</th>
+                    <th>节点</th>
+                    <th>时间</th>
+                    <th>平台</th>
+                    <th class="hidden-xs">操作</th>
+                </tr>
+                </thead>
+              </table>
+            </div>
+            <!-- /.box-body -->
+          </div>
+</section>
+
+<script>
+$(function(){
+    var sitetypes={{.T.sitetypes}};
+    $("#sitetype").append("<option value=''>--请选择网站类型--</option>");
+    $("#edit-sitetype").append("<option value=''>--请选择网站类型--</option>");
+    for(var i in sitetypes){
+        $("#sitetype").append("<option value='"+sitetypes[i]+"'>"+sitetypes[i]+"</option>")
+        $("#edit-sitetype").append("<option value='"+sitetypes[i]+"'>"+sitetypes[i]+"</option>")
+    }
+	ttable=$('#sitelist').DataTable({
+		"language": {
+              "url": "/js/dataTables.chinese.lang"
+          },
+		"columnDefs": [
+		     { "orderable": false, "targets": [0,1,2,3,4,5,6,7,8,9,10,11,13,14] }
+		],
+		"order": [[12,"desc"]],
+        "lengthChange":false,
+		"processing":true,
+		"serverSide": true,
+		"searching": true,
+		"ordering": true,
+		"info": false,
+		"autoWidth": true,
+		"ajax": {
+			"url": "/center/site/sitelist",
+			"type": "POST"
+		 },
+		"columns": [
+			{ "data": "site"},
+            { "data": "another_name"},
+            { "data": "domain"},
+            { "data": "area"},
+            { "data": "city"},
+            { "data": "district"},
+            { "data": "industry"},
+            { "data": "site_type"},
+            { "data": "p_site"},
+            { "data": "s_site"},
+            { "data": "spider_status",width:"28px",render:function (val,a,row) {
+                  return  "<a href='/center/site/getluas/"+row.site+"' style='color: #333 !important;' target='_blank'>"+val+"</a>"
+                  //return  '<a onclick="goToCenter(\''+row.site+'\')" style="color: #333 !important;" target="_blank">'+val+'</a>'
+                }},
+            { "data": "event",render:function (val,a,row){
+                   var tmpval =val
+                   if(val.length >9){
+                       val = val.substring(0,9)+"..."
+                   }
+                   return "<span title='"+tmpval+"'>"+val+"</span>"
+                }},
+            { "data": "remarktime",width:"50px"},
+            { "data": "platform"},
+            { "data": "_id",render:function (val,a,row){
+                var div=$("<div><div class=\"btn-group\"></div></div>")
+                var buttonEdit=$('<a type="button" target="_blank" class="btn btn-sm btn-default">编辑</a>');
+                var buttonDelete=$('<a type="button" class="btn btn-sm btn-default">删除</a>');
+                buttonEdit.attr("onclick","siteEdit('"+JSON.stringify(row)+"',)");
+                buttonDelete.attr("onclick","siteDelete('"+val+"')");
+                div.find(".btn-group").append(buttonEdit);
+                div.find(".btn-group").append(buttonDelete);
+                return div.html();
+            }}
+        ],
+		"fnDrawCallback": function(table) {
+		 	$("ul.pagination").prepend("&nbsp;&nbsp;&nbsp;转到第 <input type='text' id='changePage'   style='width:20px;'> 页    <a type='text' href='javascript:void(0);' id='dataTable-btn' style='text-align:center'>GO</a>");
+		 	$('#dataTable-btn').click(function(e) {    
+			    var redirectpage=0
+				if($("#changePage").val() && $("#changePage").val() > 0) {    
+			        var redirectpage = $("#changePage").val() - 1;    
+			    }  
+				ttable.page(redirectpage).draw(false);      
+		    });    
+		}, 
+		"fnServerParams": function (e) {
+		    var platform=$("#platform").val()
+            var event = $("#event").val();
+            var sitestype = $("#sitestype").val();
+            var area = $("#areas").val();
+            if(platform){
+              e.platform=platform
+            }else{
+              e.platform="-1"
+            }
+            if(event){
+                e.event=event
+            }else{
+                e.event="-1"
+            }
+            if(sitestype){
+                e.sitestype = sitestype;
+            }else{
+                e.sitestype="-1";
+            }
+            if(area){
+                e.area = area;
+            }else{
+                e.area="-1";
+            }
+        }
+	});
+	common.setActive("index_site");
+	ttable.on('init.dt', function () {
+	    //平台
+        var urgency="<div class='form-group'><label for='name'>平台：</label>"+
+            "<select id='platform' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+            "<option value='-1'>全部</option>"+
+            "<option value='golua平台'>golua平台</option>"+
+            "<option value='python'>python</option>"+
+            "<option value='通用爬虫'>通用爬虫</option>"+
+            "<option value='chrome插件'>chrome插件</option>"+
+            "</select></div>"
+		$("#sitelist_filter").prepend("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;");
+		$("#sitelist_filter").prepend(urgency);
+		//节点
+        var event="<option value='-1'>全部</option>"
+        var events={{.T.events}}
+        for(k in events){
+            event+="<option value='"+events[k]+"'>"+events[k]+"</option>"
+        }
+        var selectEvent="<div class='form-group'><label for='name'>节点：</label>"+
+            "<select id='event' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+            event+
+            "</select></div>"
+        $("#sitelist_filter").prepend("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;");
+        $("#sitelist_filter").prepend(selectEvent);
+		//网站类型
+        var sitestype ="<option value='-1'>全部</option>";
+        for(k in sitetypes){
+            sitestype+="<option value='"+sitetypes[k]+"'>"+sitetypes[k]+"</option>";
+        }
+        var selectStype="<div class='form-group'><label for='name'>网站类型：</label>"+
+            "<select id='sitestype' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+            sitestype+
+            "</select></div>"
+        $("#sitelist_filter").prepend("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;");
+        $("#sitelist_filter").prepend(selectStype);
+        //省份
+        var area ="<option value='-1'>全部</option>";
+        var areas={{.T.areas}};
+        for(k in areas){
+            area+="<option value='"+areas[k]+"'>"+areas[k]+"</option>";
+        }
+        var selectArea="<div class='form-group'><label for='name'>省份：</label>"+
+            "<select id='areas' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+            area+
+            "</select></div>"
+        $("#sitelist_filter").prepend("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;");
+        $("#sitelist_filter").prepend(selectArea);
+        //
+        $("#sitelist_wrapper .col-sm-6").css({width:"100%"})//样式
+	});
+})
+
+function addSite() {
+    $("#addsite").modal("show");
+}
+function updateSite(){
+    id = $("#saveSiteId").attr("saveid");
+    var site = $("#edit-site").val();
+    var sitetype = $("#edit-sitetype").val();
+    if(site==""||sitetype==""){
+        alert("表单填写不完整！");
+        return
+    }
+    var domain = $("#edit-domain").val();
+    var othername = $("#edit-othername").val();
+    var area = $("#edit-area").val();
+    var city = $("#edit-city").val();
+    var district = $("#edit-district").val();
+    var industry = $("#edit-industry").val();
+    var sitetype = $("#edit-sitetype").val();
+    var psite = $("#edit-psite").val();
+    var ssite = $("#edit-ssite").val();
+    $.ajax({
+        url:"/center/site/savesite",
+        type:"post",
+        data:{"id":id,"site":site,"sitetype":sitetype,"othername":othername,"domain":domain,"area":area,"city":city,"district":district,"industry":industry,"psite":psite,"ssite":ssite},
+        success:function (r){
+            if(r&&r.ok){
+                showTip("保存成功", 1000);
+                $("#editsite").modal("hide");
+                setTimeout(function (){
+                    window.location.reload();
+                }, 1000);
+            }else{
+                showTip("保存失败", 1000);
+            }
+        }
+    })
+}
+//编辑信息
+function siteEdit(rowjson){
+    var row = JSON.parse(rowjson);
+    $("#edit-site").val(row.site);
+    $("#edit-domain").val(row.domain);
+    $("#edit-othername").val(row.another_name);
+    $("#edit-area").val(row.area);
+    $("#edit-city").val(row.city);
+    $("#edit-district").val(row.district);
+    $("#edit-industry").val(row.industry);
+    if(row.site_type!=""){
+        $("#edit-sitetype option").each(function (){
+            var text = $(this).val()
+            if(text == row.site_type){
+                $(this).attr("selected",true);
+            }
+        })
+
+    }
+    $("#psite").val(row.p_site);
+    $("#ssite").val(row.s_site);
+    $("#saveSiteId").attr("saveid",row._id);
+    $("#editsite").modal("show");
+}
+//删除站点
+function siteDelete(id){
+    showConfirm("确定删除？", function() {
+        $.ajax({
+            url:"/center/site/delete",
+            type:"post",
+            data:{"id":id},
+            success:function (r) {
+                if(r&&r.d){
+                    showTip("删除成功", 1000);
+                    ttable.ajax.reload();
+                }else{
+                    showTip("删除失败", 1000);
+                }
+            }
+        })
+    });
+}
+//保存站点
+function saveSite(id){
+    var site = $("#site").val();
+    var sitetype = $("#sitetype").val();
+    if(site==""||sitetype==""){
+        alert("表单填写不完整！");
+        return
+    }
+    var domain = $("#domain").val();
+    var othername = $("#othername").val();
+    var area = $("#area").val();
+    var city = $("#city").val();
+    var district = $("#district").val();
+    var industry = $("#industry").val();
+    var sitetype = $("#sitetype").val();
+    var psite = $("#psite").val();
+    var ssite = $("#ssite").val();
+    $.ajax({
+        url:"/center/site/savesite",
+        type:"post",
+        data:{"id":id,"site":site,"sitetype":sitetype,"othername":othername,"domain":domain,"area":area,"city":city,"district":district,"industry":industry,"psite":psite,"ssite":ssite},
+        success:function (r){
+            if(r&&r.ok){
+                showTip("保存成功", 1000000);
+                $("#addsite").modal("hide");
+                setTimeout(function (){
+                    window.location.reload();
+                }, 1000);
+            }else{
+                showTip("保存失败", 1000);
+            }
+        }
+    })
+}
+//导入站点
+function importSite(){
+    var f=$("#fileframe").contents().find("input");
+    f.get(0).click();
+    f.change(function(){
+        var val=$(this).val()?$(this).val():"";
+        if(val.indexOf(".xlsx")<0){
+            //alert("文件格式非法");
+            showMsg("文件格式非法", function() {});
+        }else{
+            $(this).parent().submit();
+            common.maskShow("正在导入数据");
+            var ret=setInterval(function(){
+                var f=$(window.frames[0].document).find("form");
+                if(f.length==0){
+                    common.maskHide();
+                    var b=$(window.frames[0].document).find("body").text();
+                    $(window.frames[0].document).find("body").append("<form id='uploadform' method='post' enctype='multipart/form-data' action='/center/site/importfile'><input type='file' name='xlsx' /></form>");
+                    if(b!=""&&b!=`""`){
+                        showTip(b, 2000);
+                    }else{
+                        showTip("导入成功", 2000);
+                    }
+                    clearInterval(ret);
+                    ttable.ajax.reload();
+                }
+            },500)
+
+        }
+    })
+}
+function checkclick(me){
+    ttable.ajax.reload();
+}
+function cancelAdd(){
+    $("#addsite").modal("hide");
+}
+</script>
+</div>
+{{include "bottom.html"}}

src/web/templates/sitelualist.html

@@ -0,0 +1,112 @@
+{{include "head.html"}}
+<div class="content-wrapper">
+  <section class="content-header">
+    <h1>
+      {{.T.site}}
+    </h1>
+    <ol class="breadcrumb">
+      <li><a href="/center/site/sitelist"><i class="fa fa-dashboard"></i> 站点</a></li>
+      <li class="active">脚本列表</li>
+    </ol>
+  </section>
+  <section class="content">
+    <div class="box">
+      <!-- /.box-header -->
+      <div class="box-body">
+        <table id="spider" class="table table-bordered table-striped">
+          <thead>
+          <tr>
+            <th>栏目名称</th>
+            <th>爬虫代码</th>
+            <th>紧急度</th>
+            <th>节点</th>
+            <th>作者</th>
+            <th>最后修改时间</th>
+            <th>状态</th>
+            <th>平台</th>
+          </tr>
+          </thead>
+        </table>
+      </div>
+      <!-- /.box-body -->
+    </div>
+  </section>
+
+  <script>
+    console.log()
+    $(function(){
+      lualisttable=$('#spider').DataTable({
+        "language": {
+          "url": "/js/dataTables.chinese.lang"
+        },
+        "processing":true,
+        "serverSide": false,
+        "searching": true,
+        "ordering": false,
+        "info": true,
+        "autoWidth": true,
+        "paging": false,
+        "ajax": {
+          "url": "/center/site/getluasbysite/",
+          "type": "POST",
+          "data": {"site":{{.T.site}}}
+        },
+        "columns": [
+          { "data": "param_common","width":"200px",render:function(val){
+              if(val.length<3){
+                val="暂无";
+              }else{
+                var href="#"
+                if(val[11]){
+                  href=val[11]
+                }
+                val="<a href='"+href+"' title='"+val[2]+"' target='_blank'>"+val[2]+"</a>"
+              }
+              return val
+            }},
+          { "data": "code"},
+          { "data": "urgency",render:function(val){
+              if(val==1){
+                return "紧急"
+              }else{
+                return "普通"
+              }
+            }},
+          { "data": "event","width":"50px"},
+          { "data": "createuser"},
+          { "data": "modifytime"},
+          { "data": "state","width":"70px",render:function(val,a,row){
+              var state=row.state;
+              if(state==0){
+                val="<span class='text-info text-bold'>待完成</span>"
+              }else if(state==1){
+                val="<span class='text-primary text-bold'>待审核</span>"
+              }else if(state==2){
+                val="<span class='text-danger text-bold'>未通过</span>"
+              }else if(state==3){
+                val="<span class='text-success text-bold'>已通过</span>"
+              }else if(state==4){
+                val="<span class='text-danger text-bold'>已作废</span>"
+              }else if(state==5){
+                val="<span class='text-success text-bold'>已上架</span>"
+              }else if(state==6){
+                val="<span class='text-danger text-bold'>已下架</span>"
+              }else if(state==7){
+                val="<span class='text-danger text-bold'>无发布</span>"
+              }else if(state==8){
+                val="<span class='text-danger text-bold'>需登录</span>"
+              }else if(state==9){
+                val="<span class='text-danger text-bold'>无法处理</span>"
+              }else if(state==10){
+                val="<span class='text-danger text-bold'>已删除</span>"
+              }
+              return val
+            }},
+          { "data": "platform"}
+        ]
+      });
+      common.setActive("index_site");
+    })
+  </script>
+</div>
+{{include "bottom.html"}}