Merge branch 'develop' of 192.168.3.17:zhanghongbo/qfw into develop

core/src/timetask.json

@@ -1 +1 @@
-{"comment":{"c_rate":720,"commentrate":900},"market":{"demand":{"attr":["i_hits","i_bids","i_status"],"timepoint":"2016-03-09 15:44:54"},"service":{"attr":["i_hits","i_sales","i_comments","i_score","i_appcounts"],"timepoint":"2016-03-09 15:44:54"}},"marketisstart":true,"marketrate":300}
+{"comment":{"c_rate":720,"commentrate":900},"market":{"demand":{"attr":["i_hits","i_bids","i_status"],"timepoint":"2016-03-10 10:08:18"},"service":{"attr":["i_hits","i_sales","i_comments","i_score","i_appcounts"],"timepoint":"2016-03-10 10:08:18"}},"marketisstart":true,"marketrate":300}

core/src/web/templates/common/memberleft.html

@@ -84,7 +84,7 @@
 	color:#0099FA;
 }
 .list-group .list-group-item {
-	border:0px;
+	border: 0px;
     font-size: 14px;
     color: #666666;
 	padding: 10px 0px 10px 70px;
@@ -131,15 +131,15 @@ a.list-group-item:hover{
 			</div>
 		</div>
 	</a>
-	<a class="list-group-item" style="padding:5px 2px;">
-	<div style="line-height: 40px;width:248px;height:50px;text-align:center;">
+	<a class="list-group-item" style="padding:5px 2px;border-top:1px solid #E5E6E9;">
+	<div style="line-height: 24px;width:248px;height:50px;text-align:center;">
 		<div onclick="creditModel(1)" style="float:left;width:120px;border-right:1px;">
-<!--			<div style="font-size: 12px;">积分</div>-->
-			<div><i class="glyphicon jinbi" style="width:20px;color:red;font-size:13px;margin-top: -7px;vertical-align: middle;"></i><span style="font-weight:700;font-size:16px;color:#FF5A5F;" class="gold">{{session "i_credit"}}</span><span class="font-size-12" style="color:#999999;"> 积分</span></div>
+			<div style="font-weight: 700;">积分</div>
+			<div><i class="glyphicon jinbi" style="width:20px;color:red;font-size:13px;margin-top: -7px;vertical-align: middle;"></i><span style="font-weight:normal;" class="gold">{{session "i_credit"}}</span><span class="font-size-12 hidden" style="color:#999999;"> 积分</span></div>
 		</div>
-		<div style="height:40px;float:left;width:1px;background-color:#e5e6e9"></div>
+		<div style="height:50px;float:left;width:1px;background-color:#e5e6e9"></div>
 		<div onclick="window.location.href='/member/accountset/index'" style="float:left;width:125px;">
-<!--			<div style="font-size: 12px;">认证</div>-->
+			<div style="font-weight: 700;">认证</div>
 			<div>
 			{{if session "identType"}}
 				{{$identType := session "identType"}}

core/src/web/templates/common/wsitehead.html

@@ -0,0 +1,73 @@
+<!---->
+<div class="index-new-head b-head">
+	{{include "/common/top.html"}}
+	<div class="b-nav index-new-nva">
+		<ul>
+			<li class="b-nav-logo"><a href="/"><img src="/images/logo.png" class="logo"></a></li>
+			<li class="b-nav-link b-nav-main"><a href="/front/content_qykb_list.html">微官网</a></li>
+			
+			<li class="b-h-search hidden">
+				<div class="input-group">
+			        <div class="input-group-btn hide">
+			        	<button type="button" id="header-searchBtn" class="btn btn-default dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false"><font>服务</font><span class="hide caret margin-l-10"></span></button>
+			        </div> <!--/btn-group -->
+					<form method="post" id="searchForm" action="/front/content_qykb_list.html">
+						<span id="searchFormBtn"><i class="glyphicon sousuo"></i></span>
+				        <input type="text" name="query" class="form-control" id="header-searchInput" placeholder="找新闻资讯" value="{{if .T.querymap}}{{index .T.querymap "query"}}{{end}}">
+						{{if .T.querymap}}
+						<input type="hidden" name="city" id="city" value="{{index .T.querymap "city"}}">
+						<input type="hidden" name="cityNo" id="cityNo" value="{{index .T.querymap "cityNo"}}">
+						<input type="hidden" name="c_author" id="hauthor" value="{{index .T.querymap "c_author"}}">
+						<input type="hidden" name="c_hfl" id="hfl" value="{{index .T.querymap "c_hfl"}}">
+						<input type="hidden" name="c_zb" id="zb" value="{{index .T.querymap "c_zb"}}">
+						{{end}}
+					</form>
+	       		</div><!-- /input-group -->
+			</li>
+			<li class="pull-right b-loginStatus" id="b-loginStatus"></li>
+		</ul>
+	</div>
+</div>
+<script type="text/javascript">
+//顶部是否加载完毕
+var topLoadOver = false;
+var keyCode=0;
+document.onkeydown=function(event){
+    var e = event || window.event || arguments.callee.caller.arguments[0];
+    if(e && e.keyCode==13){ // enter 键
+        keyCode=13
+    }
+};
+
+$(function (){
+	isMobile = !$("#b-loginStatus").is(":visible");
+	$("#searchFormBtn").click(function(){
+		keyCode = 13;
+		$("form#searchForm").submit();
+	});
+	$("form#searchForm").submit(function(){
+		var input_val = $(this).children("[type='text']").val().replace(/^\s+|\s+$/g,"");
+		if(input_val==""){
+			input_val = $("#header-searchInput").attr('placeholder');
+		}
+		if (keyCode==13){
+	 		if(input_val==""||input_val=="请输入关键字"){
+				keyCode=0;
+				return false;
+			}else {
+				$("input[name='words']").val(input_val);
+				keyCode=0;
+				return true;
+			}	
+		}
+	});
+	//
+	$("[id='releaseService']").click(function(){
+		if(isLogined){
+			window.location.href = "/front/reurl";
+		}else{
+			loginModalShow(6);
+		}
+	});
+});
+</script>

core/src/web/templates/member/microwebsite.html

@@ -44,11 +44,38 @@
 .selected{
 	border: 2px #dd4814 solid !important;
 }
+.b-nav {
+    border-top-color: #FFB200;
+}
+.b-head .b-com-four {
+    background-color: #FFB200;
+}
+body{
+	background-color:#F0F0F0;
+}
+.a-content{
+	margin-top:10px;
+}
+.b-round-dot {
+    background-color: #FFB200;
+}
+.b-head .head-hidemenu li:hover {
+    background-color: #FFB200;
+}
+.b-head .head-hidemenu li:hover .b-round-dot {
+    background-color: rgb(255, 255, 255);
+}
+.index-new-head .head-hidemenu i {
+    color: #FFB200;
+}
+.b-nav-main {
+    background-color: #FFB200;
+}
 </style>
 </head>
 <body>
 <!-- 头部 -->
-{{include "/common/head.html"}}
+{{include "/common/wsitehead.html"}}
 <!-- 中间 -->
 <div class="a-content">
 	<div class="panel panel-default">
@@ -99,7 +126,7 @@
 loginBackType = 1;
 var isPreview = "{{.T.isPreview}}" == "true";
 $(function(){
-	setNavbarActive(5);
+	//setNavbarActive(5);
 	if(!isPreview){
 		$(".step-1 span").each(function(){
 			if($(this).attr("value") == "{{.T.s_microwebsite}}"){

messageframe/src/util/client.go

@@ -22,11 +22,13 @@ func StartClient(parseEvent func([]byte), serveraddr string) (*Writer, net.Conn)
 
 	conn, err := net.DialTCP("tcp", nil, tcpAddr)
 	checkError(err)
-	w := NewWriter(conn)
 	//接受消息
 	messageQueue := make(chan RawData, 2000) //并发1000
 	go processMsg(messageQueue, parseEvent)
 	//从流中提取消息
 	go forwardMessage(conn, messageQueue)
+	//需要构造写缓冲区
+	w := NewWriter(conn, make(chan RawData, 20))
+	go w.processWrite()
 	return w, conn
 }

messageframe/src/util/event.go

@@ -1,15 +1,23 @@
 package util
 
+//只有事件，发送类型
 const (
-	EVENT_RETURN_MACHINE_ID              = iota
-	EVENT_REQUEST_HEARTBEAT              //心跳
-	EVENT_RETURN_HEARTBEAT               //
-	EVENT_PUBLISH_MYSERVICES             //发布我的服务
-	EVENT_BROADCAST_REQUEST_SPIDER_STATE //获取爬虫状态
+	EVENT_RETURN_MACHINE_ID    = iota
+	EVENT_REQUEST_HEARTBEAT    //心跳
+	EVENT_RETURN_HEARTBEAT     //
+	EVENT_PUBLISH_MYSERVICES   //发布我的服务
+	EVENT_REQUEST_SPIDER_STATE //获取爬虫状态
+	EVENT_RECIVE_SPIDER_STATE  //接受爬虫状态
+	EVENT_RECIVE_CALLBACK      //调用返回，用于调用服务需要同步返回值
 
 	//------------------------
-	SERVICE_RECIVE_SPIDER_STATE = 1000 //接受爬虫状态
-	SERVICE_ECPS_SPIDER_HN      = iota //河南爬虫
-	SERVICE_ECPS_SPIDER_HB             //河北爬虫
+	SERVICE_ECPS_SPIDER_HN //河南爬虫
+	SERVICE_ECPS_SPIDER_HB //河北爬虫
+
+	//-------发送方式----------------
+	SENDTO_TYPE_RAND_RECIVER = 0 //发送给任一服务接收者，默认是这种模式
+	SENDTO_TYPE_ALL          = 1 //发送给所有客户端
+	SENDTO_TYPE_ALL_RECIVER  = 2 //发送给所有指定服务接收者
+	SENDTO_TYPE_P2P          = 3 //发送给指定客户端
 
 )

messageframe/src/util/event_test.go

@@ -0,0 +1,18 @@
+package util
+
+import (
+	"log"
+	"regexp"
+	"testing"
+)
+
+func TestEvent(t *testing.T) {
+	log.Println(SENDTO_TYPE_RAND_RECIVER, SENDTO_TYPE_P2P)
+}
+
+func TestReg(t *testing.T) {
+	reg, _ := regexp.Compile("<[^/]*</strong>")
+	msg := `<strong..sldfkjsdlf>238947askdlfjasldfj</strong>abcdefg`
+	ret := reg.ReplaceAllString(msg, "")
+	log.Println(ret)
+}

messageframe/src/util/protocol.go

@@ -20,7 +20,8 @@ type Reader struct {
 
 //
 type Writer struct {
-	conn net.Conn
+	conn       net.Conn
+	writeQueue chan RawData
 }
 
 //
@@ -35,9 +36,10 @@ func NewReader(c net.Conn) *Reader {
 }
 
 //
-func NewWriter(c net.Conn) *Writer {
+func NewWriter(c net.Conn, wq chan RawData) *Writer {
 	return &Writer{
-		conn: c,
+		conn:       c,
+		writeQueue: wq,
 	}
 }
 
@@ -94,14 +96,14 @@ func processMsg(q <-chan RawData, parseEvent func([]byte)) {
 }
 
 //写数据
-func (w *Writer) Write(bs []byte) (int, error) {
+func (w *Writer) Write(bs []byte) {
 	data := Enpacket(bs)
-	return w.conn.Write(data)
+	w.writeQueue <- RawData(data)
 }
 
 //写入的是对象，一般用map[string]interface{}
-func (w *Writer) WriteObj(obj interface{}) (int, error) {
-	return w.conn.Write(EnpacketObj(obj))
+func (w *Writer) WriteObj(obj interface{}) {
+	w.writeQueue <- RawData(EnpacketObj(obj))
 }
 
 //封包
@@ -118,3 +120,13 @@ func EnpacketObj(obj interface{}) []byte {
 	bs, _ := json.Marshal(obj)
 	return Enpacket(bs)
 }
+
+//从缓冲区读取
+func (w *Writer) processWrite() {
+	for {
+		select {
+		case msg := <-w.writeQueue:
+			w.conn.Write(msg)
+		}
+	}
+}

messageframe/src/util/server.go

@@ -10,12 +10,12 @@ import (
 func StartServer(parseEvent func([]byte),
 	processconnection func(conn net.Conn),
 	addr string) {
-	netListen, err := net.Listen("tcp", ":6060")
+	netListen, err := net.Listen("tcp", addr)
 	if err != nil {
 		fmt.Println(err.Error())
 	}
 	defer netListen.Close()
-	fmt.Println("等待连接")
+	fmt.Println("等待连接，服务地址:", addr)
 	//接受消息
 	messageQueue := make(chan RawData, 2000) //并发1000
 	go processMsg(messageQueue, parseEvent)

messageframe/src/客户端/main.go

@@ -10,7 +10,7 @@ import (
 var w *util.Writer //通常只使用writer，不要使用conn
 var conn net.Conn
 var my_id string //所有通信都要使用的
-var my_services []int = []int{util.SERVICE_ECPS_SPIDER_HN}
+var my_services []int = []int{util.SERVICE_ECPS_SPIDER_HN, util.EVENT_REQUEST_SPIDER_STATE}
 
 func main() {
 	w, conn = util.StartClient(func(data []byte) {

messageframe/src/客户端/processbusinessmsg.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"log"
+	"util"
+)
+
+//业务消息处理
+func ProcessBusinessMsg(data map[string]interface{}) {
+	if v, ok := data["event"]; ok {
+		event := int(v.(float64))
+		switch event {
+		case util.EVENT_REQUEST_SPIDER_STATE: //回应，，请求爬虫状态
+			w.WriteObj(map[string]interface{}{
+				"myid":       my_id,
+				"event":      util.EVENT_RECIVE_SPIDER_STATE,
+				"sendtotype": util.SENDTO_TYPE_ALL_RECIVER,
+				"data":       "状态良好",
+			})
+		case util.SERVICE_ECPS_SPIDER_HN: //爬取河南工商公示数据
+			log.Println("爬河南公示数据")
+		default:
+			log.Println("服务器发过来的请求，我无法处理", data)
+		}
+	}
+}

messageframe/src/客户端/processmsg.go

@@ -7,11 +7,10 @@ import (
 	"util"
 )
 
-//通用时间处理
+//通用事件处理（一般不要改这个文件）
 func ProcessMsg(msg []byte) {
 	tmp := map[string]interface{}{}
 	json.Unmarshal(msg, &tmp)
-
 	if v, ok := tmp["event"]; ok {
 		event := int(v.(float64))
 		switch event {
@@ -30,16 +29,8 @@ func ProcessMsg(msg []byte) {
 				"data":  time.Now().Unix(),
 			})
 			log.Println("对服务器回应了本机心跳")
-		case util.EVENT_BROADCAST_REQUEST_SPIDER_STATE: //回应，，请求爬虫状态
-			w.WriteObj(map[string]interface{}{
-				"myid":  my_id,
-				"event": util.SERVICE_RECIVE_SPIDER_STATE,
-				"data":  "状态良好",
-			})
-		case util.SERVICE_ECPS_SPIDER_HN: //爬取河南工商公示数据
-			log.Println("爬河南公示数据")
-		default:
-			log.Println("服务器发过来的请求，我无法处理", string(msg))
+		default: //业务处理
+			ProcessBusinessMsg(tmp)
 		}
 	}
 }

messageframe/src/服务端/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"flag"
 	"log"
 	"math/rand"
 	"net"
@@ -29,8 +30,8 @@ var allservice map[int][]string = make(map[int][]string)
 func GC() {
 	now := time.Now().Unix()
 	for k, v := range allclient {
-		if now-v.timestamp > 12 {
-			//超过40秒未回应心跳
+		if now-v.timestamp > gcinterval*3 {
+			//3次GC未回应心跳
 			v.conn.Close()
 			removeClient(k)
 			continue
@@ -43,7 +44,7 @@ func GC() {
 			removeClient(k)
 		}
 	}
-	time.AfterFunc(4*time.Second, GC)
+	time.AfterFunc(time.Duration(gcinterval)*time.Second, GC)
 }
 
 //删除服务节点
@@ -81,44 +82,64 @@ func processmsg(msg []byte) {
 				allservice[service] = append(allservice[service], my_id)
 			}
 			log.Println("所有服务", allservice)
-		case util.EVENT_BROADCAST_REQUEST_SPIDER_STATE: //要个爬虫状态
-			data := util.Enpacket(msg)
-			for service_machine_id, v := range allclient { //发所有，不支持的不处理
-				if service_machine_id == my_id { //广播不用发给自己
-					continue
-				}
-				v.conn.Write(data)
+		default: //处理业务事件
+			//识别发送类型
+			var sttype int
+			if sendtotype, ok := tmp["sendtotype"]; ok {
+				sttype = int(sendtotype.(float64))
+			} else {
+				sttype = util.SENDTO_TYPE_RAND_RECIVER
 			}
-		case util.SERVICE_RECIVE_SPIDER_STATE: //爬虫的回应，发给所有监控端
-			bs := util.Enpacket(msg)
-			if v, ok := allservice[event]; ok {
-				for _, service_machine_id := range v {
+			bs := util.Enpacket(msg)            //待发送数据
+			if sttype == util.SENDTO_TYPE_ALL { //发送给所有节点
+				for service_machine_id, v := range allclient { //发所有，不支持的不处理
 					if service_machine_id == my_id { //广播不用发给自己
 						continue
 					}
-					allclient[service_machine_id].conn.Write(bs)
+					v.conn.Write(bs)
 				}
 			}
-		default: //处理业务事件
 			if v, ok := allservice[event]; ok {
-				//随机选择一个节点提供服务,允许出错尝试3次
-				bs := util.Enpacket(msg)
-				for i := 0; i < 3; i++ {
-					service_machine_id := v[rand.Intn(len(v))]
-					_, err := allclient[service_machine_id].conn.Write(bs)
-					if err == nil {
-						break
-					} else {
-						removeClient(service_machine_id)
+				switch sttype {
+				case util.SENDTO_TYPE_RAND_RECIVER: //随机选择一个节点提供服务,允许出错尝试3次
+					for i := 0; i < 3; i++ {
+						service_machine_id := v[rand.Intn(len(v))]
+						_, err := allclient[service_machine_id].conn.Write(bs)
+						if err == nil {
+							break
+						} else {
+							removeClient(service_machine_id)
+						}
+					}
+				case util.SENDTO_TYPE_ALL_RECIVER:
+					if v, ok := allservice[event]; ok {
+						for _, service_machine_id := range v {
+							if service_machine_id == my_id { //广播不用发给自己
+								continue
+							}
+							allclient[service_machine_id].conn.Write(bs)
+						}
+					}
+				case util.SENDTO_TYPE_P2P:
+					to := tmp["to"].(string)
+					if v2, ok2 := allclient[to]; ok2 {
+						v2.conn.Write(bs)
 					}
 				}
+
 			}
 		}
 	}
 }
 
+var port string
+var gcinterval int64
+
 //
 func main() {
+	flag.StringVar(&port, "p", "6060", "开放端口")
+	flag.Int64Var(&gcinterval, "g", 20, "GC间隔时间")
+	flag.Parse()
 	//心跳检测
 	go GC()
 	//启动服务
@@ -129,6 +150,6 @@ func main() {
 		uuid := util.UUID(32)
 		c.Write(util.EnpacketObj(map[string]interface{}{"event": util.EVENT_RETURN_MACHINE_ID, "data": uuid}))
 		allclient[uuid] = &Client{conn: c, timestamp: time.Now().Unix()}
-	}, ":6060")
+	}, ":"+port)
 
 }

messageframe/src/爬虫监控/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"log"
 	"net"
 	"time"
 	"util"
@@ -10,13 +11,14 @@ import (
 //
 var w *util.Writer //通常只使用writer，不要使用conn
 var conn net.Conn
-var my_id string                                                //所有通信都要使用的
-var my_services []int = []int{util.SERVICE_RECIVE_SPIDER_STATE} //接受爬虫状态
+var my_id string                                              //所有通信都要使用的
+var my_services []int = []int{util.EVENT_RECIVE_SPIDER_STATE} //接受爬虫状态
 
 func GC() {
 	w.WriteObj(map[string]interface{}{
-		"myid":  my_id,
-		"event": util.EVENT_BROADCAST_REQUEST_SPIDER_STATE,
+		"myid":       my_id,
+		"event":      util.EVENT_REQUEST_SPIDER_STATE,
+		"sendtotype": util.SENDTO_TYPE_ALL_RECIVER,
 	})
 	time.AfterFunc(20*time.Second, GC) //20秒查一次爬虫状态
 }
@@ -27,6 +29,7 @@ func main() {
 	flag.StringVar(&serveraddr, "s", "127.0.0.1:6060", "服务端地址")
 	flag.BoolVar(&dogc, "g", false, "是否运行GC，主监控控制gc就可以了，其他点，可以制作查看")
 	flag.Parse()
+	log.Println(serveraddr, dogc)
 	w, conn = util.StartClient(func(data []byte) {
 		ProcessMsg(data)
 	}, serveraddr)

messageframe/src/爬虫监控/processbusinessmsg.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"log"
+	"util"
+)
+
+//业务消息处理
+func ProcessBusinessMsg(data map[string]interface{}) {
+	if v, ok := data["event"]; ok {
+		event := int(v.(float64))
+		switch event {
+		case util.EVENT_RECIVE_SPIDER_STATE: //收到爬虫状态
+			log.Println(data)
+		default:
+			log.Println("服务器发过来的请求，我无法处理", data)
+		}
+	}
+}

messageframe/src/爬虫监控/processmsg.go

@@ -29,10 +29,8 @@ func ProcessMsg(msg []byte) {
 				"data":  time.Now().Unix(),
 			})
 			log.Println("对服务器回应了本机心跳")
-		case util.SERVICE_RECIVE_SPIDER_STATE: //获得各个爬虫状态
-			log.Println("各个爬虫状态，", string(msg))
 		default:
-			log.Println("服务器发过来的请求，我无法处理", string(msg))
+			ProcessBusinessMsg(tmp)
 		}
 	}
 }