新增工具类

common/src/qfw/util/mongodbutil/mongodbutil.go

@@ -0,0 +1,369 @@
+package mongoutil
+
+import (
+	"encoding/json"
+	"fmt"
+	mgo "gopkg.in/mgo.v2"
+	. "gopkg.in/mgo.v2/bson"
+	"log"
+	. "qfw/util"
+	"runtime"
+	"strings"
+	"time"
+)
+
+type PoolConfig struct {
+	Alias string
+	Addr  string
+	DB    string
+	Size  int
+}
+
+//连接池
+var pool map[string]chan *mgo.Session
+var Config []PoolConfig
+
+//[mongodb://][user:pass@]host1[:port1][,host2[:port2],...][/database][?options]
+//初始化mongodb连接池大小
+func InitMongodbPool() {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Println("[E]", r)
+			for skip := 1; ; skip++ {
+				_, file, line, ok := runtime.Caller(skip)
+				if !ok {
+					break
+				}
+				go log.Printf("%v,%v\n", file, line)
+			}
+		}
+	}()
+	pool = make(map[string]chan *mgo.Session)
+	for _, v := range Config {
+		pool[v.Alias] = make(chan *mgo.Session, v.Size)
+		for i := 0; i < v.Size; i++ {
+			sess, err := createConn(v.Alias)
+			if err == nil && sess.Ping() == nil {
+				pool[v.Alias] <- sess
+			}
+		}
+	}
+}
+
+//从连接池中取得mongodb连接,懒加载模式
+func GetMgoConn(alias string) (session *mgo.Session) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Println("[E]", r)
+			for skip := 1; ; skip++ {
+				_, file, line, ok := runtime.Caller(skip)
+				if !ok {
+					break
+				}
+				go log.Printf("%v,%v\n", file, line)
+			}
+		}
+	}()
+	if v, ok := pool[alias]; ok {
+		select {
+		case session = <-v:
+			err := session.Ping()
+			if err == nil {
+				return session
+			} else {
+				session.Close()
+				session, err = createConn(alias)
+				if err == nil && session.Ping() == nil {
+					return session
+				}
+				return nil
+			}
+		case <-time.After(time.Second * 2):
+			//超时
+			return nil
+		}
+	} else {
+		return nil
+	}
+
+}
+
+//取session链接
+func createConn(alias string) (sess *mgo.Session, err error) {
+	for _, v := range Config {
+		if v.Alias == alias {
+			sess, err = mgo.Dial(v.Addr)
+			break
+		}
+	}
+	return sess, err
+}
+
+//释放连接，在取得连接后要使用defer DestoryMongoConn()进行释放
+func DestoryMongoConn(alias string, session *mgo.Session) {
+	if v, ok := pool[alias]; ok {
+		v <- session
+	}
+}
+
+//查询单条对象
+func FindOne(c, alias, db string, query interface{}) *map[string]interface{} {
+	return FindOneByField(c, alias, db, query, nil)
+}
+
+//查询单条对象
+func FindOneByField(c, alias, db string, query interface{}, fields interface{}) *map[string]interface{} {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Println("[E]", r)
+			for skip := 1; ; skip++ {
+				_, file, line, ok := runtime.Caller(skip)
+				if !ok {
+					break
+				}
+				go log.Printf("%v,%v\n", file, line)
+			}
+		}
+	}()
+	res := Find(c, alias, db, query, nil, fields, true, -1, -1)
+	if nil != res && len(*res) > 0 {
+		return &((*res)[0])
+	}
+	return nil
+}
+
+//底层查询方法
+func Find(c, alias, db string, query interface{}, order interface{}, fields interface{}, single bool, start int, limit int) *[]map[string]interface{} {
+	sess := GetMgoConn(alias)
+	var res []map[string]interface{}
+	if sess != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		defer DestoryMongoConn(alias, sess)
+		res = make([]map[string]interface{}, 1)
+		coll := sess.DB(db).C(c)
+		var err error
+		if single {
+			err = coll.Find(ObjToM(query)).Select(ObjToOth(fields)).One(&res[0])
+		} else if start > -1 {
+			err = coll.Find(ObjToM(query)).Select(ObjToOth(fields)).Sort(ObjToArr(order)...).Skip(start).Limit(limit).All(&res)
+		} else {
+			err = coll.Find(ObjToM(query)).Select(ObjToOth(fields)).Sort(ObjToArr(order)...).All(&res)
+		}
+		if nil != err {
+			log.Println("FindError", err)
+		}
+	}
+	return &res
+}
+
+//修改对象
+func Update(c, alias, db string, query interface{}, set interface{}, upsert bool, multi bool) bool {
+	sess := GetMgoConn(alias)
+	b := true
+	if sess != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				b = false
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		defer DestoryMongoConn(alias, sess)
+		coll := sess.DB(db).C(c)
+		var err error
+		if upsert {
+			_, err = coll.Upsert(ObjToM(query), ObjToM(set))
+		} else {
+			if multi {
+				_, err = coll.UpdateAll(ObjToM(query), ObjToM(set))
+			} else {
+				err = coll.Update(ObjToM(query), ObjToM(set))
+			}
+		}
+		if nil != err {
+			log.Println("updateError", err)
+			b = false
+		}
+	}
+	return b
+}
+
+//删除对象
+func Del(c, alias, db string, query interface{}) bool {
+	sess := GetMgoConn(alias)
+	b := true
+	if sess != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		defer DestoryMongoConn(alias, sess)
+		coll := sess.DB(db).C(c)
+		_, err := coll.RemoveAll(ObjToM(query))
+		if nil != err {
+			log.Println("CountError", err)
+			b = false
+		}
+	}
+	return b
+}
+
+//保存对象
+func Save(c, alias, db string, doc interface{}) string {
+	sess := GetMgoConn(alias)
+	if sess != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		defer DestoryMongoConn(alias, sess)
+		coll := sess.DB(db).C(c)
+		obj := ObjToM(doc)
+		id := NewObjectId()
+		(*obj)["_id"] = id
+		err := coll.Insert(obj)
+		//log.Println(obj)
+		if nil != err {
+			log.Println("SaveError", err)
+			return ""
+		}
+		return (strings.Split(fmt.Sprintf("%s", id), `"`)[1])
+	}
+	return ""
+}
+
+//批量插入
+func SaveBulk(c, alias, db string, doc ...map[string]interface{}) bool {
+	sess := GetMgoConn(alias)
+	b := true
+	if sess != nil {
+		defer func() {
+			b = false
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		defer DestoryMongoConn(alias, sess)
+		coll := sess.DB(db).C(c)
+		bulk := coll.Bulk()
+		for _, v := range doc {
+			bulk.Insert(v)
+		}
+		_, err := bulk.Run()
+		if nil != err {
+			log.Println("BulkError", err)
+			b = false
+		}
+	} else {
+		b = false
+	}
+	return b
+}
+
+//对象转数组
+func ObjToArr(obj interface{}) []string {
+	if s, ok := obj.(string); ok {
+		if strings.ContainsAny(s, "{") {
+			//暂时简单支持此种写法
+			var temp = make(M)
+			var str = []string{}
+			json.Unmarshal([]byte(s), &temp)
+			for k, v := range temp {
+				m := IntAll(v)
+				if m > 0 {
+					str = append(str, k)
+				} else {
+					str = append(str, "-"+k)
+				}
+			}
+			return str
+		} else {
+			return strings.Split(s, ",")
+		}
+	} else if s1, ok1 := obj.([]string); ok1 {
+		return s1
+	} else {
+		return []string{}
+	}
+}
+
+func ObjToOth(query interface{}) *M {
+	return ObjToMQ(query, false)
+}
+func ObjToM(query interface{}) *M {
+	return ObjToMQ(query, true)
+}
+
+//obj(string,M)转M,查询用到
+func ObjToMQ(query interface{}, isQuery bool) *M {
+	data := make(M)
+	defer func() {
+		if r := recover(); r != nil {
+			log.Println("[E]", r)
+			for skip := 1; ; skip++ {
+				_, file, line, ok := runtime.Caller(skip)
+				if !ok {
+					break
+				}
+				go log.Printf("%v,%v\n", file, line)
+			}
+		}
+	}()
+	if s2, ok2 := query.(*map[string]interface{}); ok2 {
+		data = M(*s2)
+	} else if s3, ok3 := query.(*M); ok3 {
+		return s3
+	} else if s, ok := query.(string); ok {
+		json.Unmarshal([]byte(strings.Replace(s, "'", "\"", -1)), &data)
+		if ss, oks := data["_id"]; oks && isQuery {
+			data["_id"] = ObjectIdHex(ss.(string))
+		}
+	} else if s1, ok1 := query.(map[string]interface{}); ok1 {
+		data = s1
+	} else if s4, ok4 := query.(M); ok4 {
+		data = s4
+	} else {
+		data = nil
+	}
+	return &data
+}

common/src/qfw/util/mongodbutil/mongodbutil_test.go

@@ -0,0 +1,31 @@
+// mongodbutil_test
+package mongoutil
+
+import (
+	"log"
+	"testing"
+)
+
+func Test_pool(t *testing.T) {
+	conf := *new(PoolConfig)
+	conf.Addr = "192.168.3.18:27080"
+	conf.Alias = "A"
+	conf.DB = "qfw"
+	conf.Size = 5
+	config = append(config, conf)
+
+	conf = *new(PoolConfig)
+	conf.Addr = "192.168.3.18:27080"
+	conf.Alias = "B"
+	conf.DB = "spider"
+	conf.Size = 5
+	config = append(config, conf)
+
+	InitMongodbPool()
+
+	tmp := *FindOne("enterprise", "A", "qfw", `{"RegNo":"542125600001855"}`)
+	log.Println(tmp)
+
+	tmp = *FindOne("enterprise", "B", "spider", `{"REGNO":"410692000004499"}`)
+	log.Println(tmp)
+}

spider2/src/client/demo/Phone.go

@@ -28,13 +28,14 @@ func IMEI() string {
 	result := 0
 	for i := 0; i < len(ImeiStr); i++ {
 		intChar, _ := strconv.Atoi(string(ImeiStr[i]))
-		i++
-		tmp, _ := strconv.Atoi(string(ImeiStr[i]))
-		tmp = tmp * 2
-		if tmp < 10 {
-			tmp = tmp - 9
+		if i%2 == 0 {
+			tmp, _ := strconv.Atoi(string(ImeiStr[i:(i + 1)]))
+			tmp = tmp * 2
+			if tmp < 10 {
+				tmp = tmp - 9
+			}
+			result = intChar + tmp
 		}
-		result = intChar + tmp
 	}
 	result = result % 10
 	if result != 0 {

spider2/src/client/demo/config.json

@@ -0,0 +1 @@
+{"Area":"HA","AreaCode":"410000","currentpage":2,"maxpage":9000,"timesleep":5,"urlPrefix":"https://120.52.121.75:8443","z_mongodbaddr":"192.168.3.18:27080","z_mongodbname":"qfw"}

spider2/src/client/demo/downloadInfo.go

@@ -1,29 +0,0 @@
-// DownloadInfo
-package main
-
-import (
-	. "common/spiderutil"
-	"log"
-)
-
-var hd *HttpDownload
-
-func DownloadInfo() {
-
-	bs, _, _ := hd.Download(&RequestConfig{
-		Urlstr: "https://120.52.121.75:8443/QueryGSInfo/entPublicitySC/entPublicityDC/getJyycmlxx.action",
-		Method: "get",
-		Param: map[string]string{
-			"AreaCode": "410000",
-			"EntId":    "141000010452582520",
-			"EntNo":    "410104000038631",
-			"Info":     "All HTTP/1.1",
-		},
-		Head: map[string]string{
-			"Host":       "120.52.121.75:8443",
-			"Connection": "keep-alive",
-			"User-Agent": "Mozilla/5.0 (Android;4.4.4;iToolsAVM;iToolsAVM)",
-		},
-	})
-	log.Println(string(bs))
-}

spider2/src/client/demo/main.go

@@ -7,64 +7,156 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+
+	util "qfw/util"
+	"time"
 )
 
 var client *http.Client
 var urlPrefix string
+var headMap map[string]string
+var config map[string]interface{}              //配置文件
+var currentpage, maxpage, timesleep, count int //当前页、最大页、延时、计数器
 
-func main() {
+func init() {
 	tr := &http.Transport{
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 	}
 	client = &http.Client{Transport: tr}
+	headMap = make(map[string]string)
+	util.ReadConfig(&config)
+	urlPrefix = config["urlPrefix"].(string)
+	currentpage = int(config["currentpage"].(float64))
+	maxpage = int(config["maxpage"].(float64))
+	timesleep = int(config["timesleep"].(float64))
+	headMap["AreaCode"] = config["AreaCode"].(string)
+}
+func main() {
+	//异常名录
+	for i := currentpage; i < maxpage; i++ {
+		log.Println(headMap["AreaCode"], "正在下载第"+fmt.Sprint(i)+"页数据")
+		GetRegSession() //设备注册
 
-	//查询企业
-	urlPrefix = "https://120.52.121.75:8443"
-	headMap := make(map[string]string)
-	headMap["AreaCode"] = "410000"
-	headMap["Url"] = urlPrefix + "/QuerySummary?AreaCode=" + headMap["AreaCode"] + "&Page=1&Limit=50&Q=郑州赛驰计算机"
-	headMap["Cookie"] = "352284048787547"
-	headMap["User-Agent"] = "Mozilla/5.0 (Android;4.0.1;MANUFACTURER;Pro5);Version/ErrorVersion;ISN_GSXT"
-	body := SetHeader(headMap)
-
-	result1 := make(map[string]interface{})
-	json.Unmarshal(body, &result1)
-	//fmt.Println(result1["RESULT"])
-	infolist := result1["RESULT"].([]interface{})
+		headMap["Url"] = urlPrefix + "/QueryAbn?AreaCode=" + headMap["AreaCode"] + "&Q=&Page=" + fmt.Sprint(i) + "&Limit=20&Item=1"
+		body := DownContent(headMap)
+		result := make(map[string]interface{})
+		json.Unmarshal(body, &result)
+		log.Println("异常名录", result)
+		list := result["RESULT"].([]interface{})
+		for _, v := range list {
+			tmp := v.(map[string]interface{})
+			if tmp["REGNO"] != nil {
+				DownloadInfo(fmt.Sprint(tmp["REGNO"]))
+			}
+		}
+		currentpage = i
+		if i == maxpage {
+			i = 1
+			currentpage = 1
+		}
+		config["currentpage"] = currentpage
+		util.WriteSysConfig(&config)
+	}
+}
+
+func DownloadInfo(qstr string) {
+	//查询结果--名录
+	headMap["Url"] = urlPrefix + "/QuerySummary?AreaCode=" + headMap["AreaCode"] + "&Page=1&Limit=50&Q=" + qstr
+	body := DownContent(headMap)
+	//log.Println("body", headMap["Url"], string(body))
+	result := make(map[string]interface{})
+	json.Unmarshal(body, &result)
+	log.Println("查询结果--名录", result)
+
+	infolist := result["RESULT"].([]interface{})
 	var ID, REGNO string
-	for _, v := range infolist {
+	for _, v := range infolist { //取第一条有效数据
 		ID = fmt.Sprint((v.(map[string]interface{}))["ID"])
 		REGNO = fmt.Sprint((v.(map[string]interface{}))["REGNO"])
+		if ID != "" {
+			break
+		}
 	}
+	GSGSDownload(ID, REGNO) //工商公示
+	QYGSDownload(ID, REGNO) //企业公示
+}
 
-	//下载工商公示信息
+//下载工商公示信息
+func GSGSDownload(ID, REGNO string) {
 	headMap["Url"] = urlPrefix + "/QueryGSInfo?AreaCode=" + headMap["AreaCode"] + "&EntId=" + ID + "&EntNo=" + REGNO + "&Info=All"
-	body = SetHeader(headMap)
-
-	fmt.Println("工商公示", string(body))
-	tmp1 := make(map[string]interface{})
-	json.Unmarshal(body, &tmp1)
-	baseInfo := tmp1["RESULT"].(map[string]interface{})
-	REGINFO := baseInfo["REGINFO"].(map[string]interface{})
-	LEGINFO := REGINFO["LEGINFO"].([]interface{})
-	fmt.Println("股东列表", LEGINFO)
+	body := DownContent(headMap)
+	log.Println("查询结果--工商公示", string(body))
+
+	tmp := make(map[string]interface{})
+	json.Unmarshal(body, &tmp)
+	baseInfo, b := tmp["RESULT"].(map[string]interface{})
+	if !b {
+		return
+	}
+	/****************登记信息*************/
+	//登记信息(REGINFO) 基本信息、股东列表、变更信息
+	djxx := baseInfo["REGINFO"].(map[string]interface{})
+	//基本信息
+	BaseMap := djxx["BASEINFO"].(map[string]interface{})
+	//股东列表
+	LEGINFO := djxx["LEGINFO"].([]interface{})
+	gdList := []interface{}{}
 	//股东详情
 	for k, v := range LEGINFO {
 		gd := v.(map[string]interface{})
 		headMap["Url"] = urlPrefix + "/QueryGSDetail?Id=" + fmt.Sprint(gd["ID"]) + "&AreaCode=" + headMap["AreaCode"] + "&EntId=" + ID + "&EntNo=" + REGNO + "&Item=LegDetail"
-		body = SetHeader(headMap)
-		fmt.Println(k, string(body))
+		body = DownContent(headMap)
+		log.Println("查询结果--股东", k, string(body))
+		tmp := make(map[string]interface{})
+		json.Unmarshal(body, &tmp)
+		gdList = append(gdList, tmp)
 	}
+	BaseMap["investors"] = gdList
+	//变更信息
+	bgxxs := djxx["CHGINFO"].([]interface{})
+	BaseMap["alterInfos"] = bgxxs
+	/****************登记信息结束*************/
 
-	//下载企业公示信息
-	headMap["Url"] = urlPrefix + "/QueryQY?AreaCode=" + headMap["AreaCode"] + "&EntId=" + ID + "&EntNo=" + REGNO + "&Info=All"
-	body = SetHeader(headMap)
+	//经营异常
+	BaseMap["excDirects"] = baseInfo["EXCEINFO"].([]interface{})
+	//动产抵押
+	BaseMap["mortInfos"] = baseInfo["MORTINFO"].([]interface{})
+	//抽查检查
+	BaseMap["spotChecks"] = baseInfo["CHECKINFO"].([]interface{})
+	//行政处罚
+	BaseMap["punishInfos"] = baseInfo["PUNINFO"].([]interface{})
+
+	/****************备案信息*************/
+	//备案信息(RECINFO) 主要人员、分支机构、清算组
+	baxx := baseInfo["RECINFO"].(map[string]interface{})
+	//主要人员
+	BaseMap["staffinfos"] = baxx["PERINFO"].([]interface{})
+	//分支机构
+	BaseMap["childents"] = baxx["BRANINFO"].([]interface{})
+	//清算组
+	BaseMap["liquidationo"] = baxx["LIQINFO"].(map[string]interface{})
+	/****************备案信息结束*************/
+
+	//严重违法
+	BaseMap["serIllInfo"] = baseInfo["LAWINFO"].([]interface{})
+	//股权出质
+	BaseMap["pledgeInfos"] = baseInfo["EQUINFO"].([]interface{})
+
+	//log.Println("BaseMap", BaseMap)
+}
 
+//下载企业公示信息
+func QYGSDownload(ID, REGNO string) {
+	headMap["Url"] = urlPrefix + "/QueryQY?AreaCode=" + headMap["AreaCode"] + "&EntId=" + ID + "&EntNo=" + REGNO + "&Info=All"
+	body := DownContent(headMap)
 	//企业年报信息
-	result2 := make(map[string]interface{})
-	json.Unmarshal(body, &result2)
-	tmp2 := result2["RESULT"].(map[string]interface{})
-	nblist := tmp2["REPORTINFO"].([]interface{})
+	result := make(map[string]interface{})
+	json.Unmarshal(body, &result)
+	tmp, b := result["RESULT"].(map[string]interface{})
+	if !b {
+		return
+	}
+	nblist := tmp["REPORTINFO"].([]interface{})
 	nb := make(map[string]interface{})
 	for _, v := range nblist {
 		nb = v.(map[string]interface{})
@@ -72,13 +164,14 @@ func main() {
 			break
 		}
 	}
-	//fmt.Println(nb)
 	nbId := fmt.Sprint(nb["ID"])
 	headMap["Url"] = urlPrefix + "/QueryQYReport?Id=" + nbId + "&AreaCode=" + headMap["AreaCode"] + "&EntId=" + ID + "&EntNo=" + REGNO + "&Info=All"
-	body = SetHeader(headMap)
-	fmt.Println("年报", string(body))
+	body = DownContent(headMap)
+	fmt.Println("查询结果--年报", string(body))
 }
-func SetHeader(headMap map[string]string) []byte {
+
+//下载内容
+func DownContent(headMap map[string]string) []byte {
 	req, _ := http.NewRequest("GET", headMap["Url"], nil)
 	//req, _ := http.NewRequest("GET", "https://120.52.121.75:8443/QueryAutoName?AreaCode=410000&Q=计算机&Size=5", nil)
 	req.Header.Add("User-Agent", headMap["User-Agent"])
@@ -90,6 +183,7 @@ func SetHeader(headMap map[string]string) []byte {
 		log.Println(err.Error())
 		return nil
 	}
+	time.Sleep(time.Duration(timesleep) * time.Second)
 	defer resp.Body.Close()
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
@@ -98,3 +192,14 @@ func SetHeader(headMap map[string]string) []byte {
 	}
 	return body
 }
+
+//设备注册
+func GetRegSession() (imei, modal string) {
+	imei = IMEI()
+	modal = Modal()
+	headMap["Cookie"] = imei
+	headMap["User-Agent"] = "Mozilla/5.0 (Android;4.4.4;" + modal + ");Version/ErrorVersion;ISN_GSXT"
+	headMap["Url"] = urlPrefix + "/QuerySite?OS=Android;" + modal + "&IMEI=" + imei
+	DownContent(headMap)
+	return imei, modal
+}

spider2/src/client/demo/modal.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"strings"
+)
+
+//属性匹配
+func AttrPair(target, source map[string]interface{}) {
+	for k1, _ := range target {
+		tk1 := strings.ToUpper(k1)
+		for k2, _ := range source {
+			tk2 := strings.ToUpper(k2)
+			if tk1 == tk2 {
+				target[k1] = source[k2]
+				continue
+			}
+		}
+	}
+}
+
+func SetBaseMap() {
+	BaseMap := make(map[string]interface{})
+	BaseMap["Area"] = ""
+	BaseMap["ID"] = ""
+	BaseMap["RegNo"] = ""
+	BaseMap["EntName"] = ""
+	BaseMap["ParEntName"] = ""
+	BaseMap["EntType"] = ""
+	BaseMap["EntTypeName"] = ""
+	BaseMap["IndustryPhy"] = ""
+	BaseMap["IndustryPhyName"] = ""
+	BaseMap["OpLocDistrict"] = ""
+	BaseMap["LeRep"] = ""
+	BaseMap["LegCerNO"] = ""
+	BaseMap["Tel"] = ""
+	BaseMap["Dom"] = ""
+	BaseMap["OpLoc"] = ""
+	BaseMap["OpScope"] = ""
+	BaseMap["ABuItemCo"] = ""
+	BaseMap["CBuItem"] = ""
+	BaseMap["OpScoType"] = ""
+	BaseMap["OpScoTypeName"] = ""
+	BaseMap["OpFrom"] = ""
+	BaseMap["OpTo"] = ""
+	BaseMap["RegCap"] = ""
+	BaseMap["RecCap"] = ""
+	BaseMap["RegCapCur"] = ""
+	BaseMap["RegCapCurName"] = ""
+	BaseMap["RegCapUSD"] = ""
+	BaseMap["EstDate"] = ""
+	BaseMap["ConGro"] = ""
+	BaseMap["ConGroUSD"] = ""
+	BaseMap["CompForm"] = ""
+	BaseMap["CompFormName"] = ""
+	BaseMap["OpState"] = ""
+	BaseMap["OpStateName"] = ""
+	BaseMap["RegOrg"] = ""
+	BaseMap["RegOrgName"] = ""
+	BaseMap["IssBLicDate"] = ""
+	BaseMap["Timestamp"] = ""
+	BaseMap["SourceType"] = ""
+}
+
+//

spider2/src/client/demo/test.json

@@ -0,0 +1,138 @@
+{
+    "ERRCODE": "0",
+    "RESULT": {
+        "CHECKINFO": [],
+        "EQUINFO": [],
+        "EXCEINFO": [
+            {
+                "ABNTIME": "2015年09月22日",
+                "BUSINESSTYPE": "1",
+                "DECORG": "新乡市工商行政管理局专业分局",
+                "ENTNAME": "新乡市旭日通信工程有限公司",
+                "ID": "141000012983580899",
+                "MARPRID": "141000012077045121",
+                "REGNO": "410791000029758",
+                "REMDATE": "",
+                "REMEXCPRES": "",
+                "SPECAUSE": "公示企业信息隐瞒真实情况、弄虚作假的",
+                "STATE": "加入",
+                "TIMESTAMP": "2016年01月13日"
+            },
+            {
+                "ABNTIME": "2015年09月22日",
+                "BUSINESSTYPE": "1",
+                "DECORG": "新乡市工商行政管理局专业分局",
+                "ENTNAME": "新乡市旭日通信工程有限公司",
+                "ID": "141000012983579925",
+                "MARPRID": "141000012077045121",
+                "REGNO": "410791000029758",
+                "REMDATE": "",
+                "REMEXCPRES": "",
+                "SPECAUSE": "公示企业信息隐瞒真实情况、弄虚作假的",
+                "STATE": "加入",
+                "TIMESTAMP": "2016年01月13日"
+            },
+            {
+                "ABNTIME": "2015年09月22日",
+                "BUSINESSTYPE": "1",
+                "DECORG": "新乡市工商行政管理局专业分局",
+                "ENTNAME": "新乡市旭日通信工程有限公司",
+                "ID": "141000012983573204",
+                "MARPRID": "141000012077045121",
+                "REGNO": "410791000029758",
+                "REMDATE": "",
+                "REMEXCPRES": "",
+                "SPECAUSE": "公示企业信息隐瞒真实情况、弄虚作假的",
+                "STATE": "加入",
+                "TIMESTAMP": "2016年01月13日"
+            },
+            {
+                "ABNTIME": "2016年09月22日",
+                "BUSINESSTYPE": "1",
+                "DECORG": "新乡市工商行政管理局专业分局",
+                "ENTNAME": "新乡市旭日通信工程有限公司",
+                "ID": "141000012983572188",
+                "MARPRID": "141000012077045121",
+                "REGNO": "410791000029758",
+                "REMDATE": "",
+                "REMEXCPRES": "",
+                "SPECAUSE": "公示企业信息隐瞒真实情况、弄虚作假的",
+                "STATE": "加入",
+                "TIMESTAMP": "2016年01月13日"
+            }
+        ],
+        "LAWINFO": [],
+        "MORTINFO": [],
+        "PUNINFO": [],
+        "RECINFO": {
+            "BRANINFO": [],
+            "LIQINFO": {},
+            "PERINFO": [
+                {
+                    "CERNO": "*****",
+                    "CERTYPE": "中华人民共和国居民身份证",
+                    "FROMTYPE": "01",
+                    "ID": "5141000012077045132",
+                    "MARPRID": "141000012077045121",
+                    "NAME": "陈松江",
+                    "NAMEINFORG": "",
+                    "POSITION": "执行董事",
+                    "TIMESTAMP": "2014年11月07日"
+                },
+                {
+                    "CERNO": "*****",
+                    "CERTYPE": "中华人民共和国居民身份证",
+                    "FROMTYPE": "01",
+                    "ID": "141000012077045137",
+                    "MARPRID": "141000012077045121",
+                    "NAME": "李素兰",
+                    "NAMEINFORG": "",
+                    "POSITION": "监事",
+                    "TIMESTAMP": "2014年11月07日"
+                }
+            ]
+        },
+        "REGINFO": {
+            "BASEINFO": {
+                "ABUITEMCO": "",
+                "APPRDATE": "2014年11月07日",
+                "CBUITEM": "",
+                "COMPFORM": "",
+                "CONGRO": "",
+                "CONGROUSD": "",
+                "DOM": "新乡市花园新村西四排1＃",
+                "ENTNAME": "新乡市旭日通信工程有限公司",
+                "ENTTYPE": "有限责任公司（自然人独资）",
+                "ESTDATE": "2014年11月07日",
+                "ID": "141000012077045121",
+                "LEGCERNO": "419002196303034430",
+                "LEREP": "陈松江",
+                "OPFROM": "2014年11月07日",
+                "OPLOC": "",
+                "OPSCOPE": "通信工程设计与施工；电气安装*",
+                "OPSTATE": "存续",
+                "OPTO": "2024年11月06日",
+                "PARENTNAME": "",
+                "REGCAP": "800",
+                "REGCAPCUR": "",
+                "REGCAPUSD": "0",
+                "REGNO": "410791000029758",
+                "REGORG": "新乡市工商行政管理局专业分局",
+                "REGSTATE": "存续",
+                "TIMESTAMP": "2014年11月07日",
+                "UNISCID": ""
+            },
+            "CHGINFO": [],
+            "LEGINFO": [
+                {
+                    "BLICNO": "--",
+                    "ID": "141000012077045135",
+                    "INV": "陈松江",
+                    "INVTYPE": "自然人股东",
+                    "LIACCONAM": "",
+                    "LISUBCONAM": ""
+                }
+            ]
+        }
+    }
+}

spider2/src/client/ecps/demo/main.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	. "common/spiderutil"
+	"log"
+	"net"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var client *rpc.Client
+var err error
+var hd *HttpDownload
+
+func init() {
+	hd = &HttpDownload{Timeout: 30000, Sleep: int(1000), RetryTime: 5, RetrySleep: 10000}
+	hd.Init()
+}
+func main() {
+	client, err := jsonrpc.Dial("tcp", "127.0.0.1:3577")
+	defer client.Close()
+	if err != nil {
+		log.Println(err.Error())
+	}
+	addr := GetProxy(client)
+	log.Println(addr)
+	hd.UseProxy(addr)
+
+	bs, _, _ := hd.Download(&RequestConfig{
+		Urlstr: "http://gsxt.lngs.gov.cn/saicpub/entPublicitySC/entPublicityDC/getJyycmlxx.action",
+		Method: "get",
+		Param: map[string]string{
+			"code":      "",
+			"condition": "",
+			"city":      "",
+			"year":      "",
+			"pagesize":  "",
+			"num":       strconv.Itoa(1),
+		},
+		Head: map[string]string{
+			"Host":                      "gsxt.lngs.gov.cn",
+			"Connection":                "keep-alive",
+			"Content-Length":            "44",
+			"Cache-Control":             "max-age=0",
+			"Accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+			"Origin":                    "http://gsxt.lngs.gov.cn",
+			"Upgrade-Insecure-Requests": "1",
+			"User-Agent":                "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36",
+			"Content-Type":              "application/x-www-form-urlencoded",
+			"Referer":                   "http://gsxt.lngs.gov.cn/saicpub/entPublicitySC/entPublicityDC/getJyycmlxx.action",
+			"Accept-Encoding":           "gzip, deflate",
+			"Accept-Language":           "zh-CN,zh;q=0.8",
+		},
+	})
+	log.Println(string(bs))
+}
+
+//获取代理地址
+func GetProxy(client *rpc.Client) string {
+	var proxy, param string
+	err := client.Call("Proxy.GetProxy", &param, &proxy)
+	if err != nil {
+		log.Println(err.Error())
+	}
+	if !Check(proxy) {
+		GetProxy(client)
+	}
+	return proxy
+}
+
+//检测代理地址是否有效
+func Check(v string) bool {
+	var host string
+	if strings.HasPrefix(v, "https") {
+		host = v[8:]
+	} else {
+		host = v[7:]
+	}
+	conn, err := net.DialTimeout("tcp", host, 1*time.Second)
+	if err != nil {
+		return false
+	} else {
+		conn.Close()
+		return true
+	}
+}

spider2/src/client/ecps/ln/config.json

@@ -1,12 +1 @@
-{
-    "basedelay": 1000,
-    "baserand": 10000,
-    "currentpage": 1,
-    "maxpage": 9000,
-    "maxwaittime": 800,
-    "rate": 30,
-    "timeout": 60000,
-    "z_addr": "127.0.0.1:99",
-    "z_mongodbaddr": "192.168.3.18:27080",
-    "z_mongodbname": "qfw"
-}
+{"basedelay":1000,"baserand":10000,"currentpage":5,"maxpage":9000,"maxwaittime":800,"rate":30,"timeout":60000,"z_addr":"127.0.0.1:99","z_mongodbaddr":"192.168.3.18:27080","z_mongodbname":"qfw"}