增加jyclient

jyclient/src/code.go

@@ -0,0 +1,16 @@
+package main
+
+const (
+	CODE_TOKEN_EXPIRE = 1000 //token过期
+	MSG_TOKEN_EXPIRE  = "token已过期"
+	CODE_E1           = 1001
+	MSG_E1            = "token错误"
+	CODE_E2           = 1002
+	MSG_E2            = "参数错误"
+	CODE_E3           = 1003
+	MSG_E3            = "调用次数过超限制"
+	CODE_E4           = 1004
+	MSG_E4            = "服务过期"
+	CODE_E5           = 1005
+	MSG_E5            = "服务条数过超限制"
+)

jyclient/src/config.json

@@ -0,0 +1,16 @@
+{
+	"log-level":1,
+	"log-filename":"jyclient.log",
+	"mysql-url":"root:Topnet123@tcp(142.4.212.40:3306)/jyqyfw?loc=Local&timeout=30s&parseTime=true",
+	"mysql-pool":5,
+	"desc-1":"描述配置 loglevel:日志级别 0=trace 1=info 2=warn 3=error 4=off | mysql-url mysql连接地址 | mysql-pool mysql连接池大小",
+	"jyapi-token":"https://api.jianyu360.com/user/access_token",
+	"jyapi-getdata":"https://api.jianyu360.com/data/getdata",
+	"jyapi-getalldata":"https://api.jianyu360.com/data/getalldata",
+	"jyapi-appid":"jyPz5XQgMABgVeTkdOMTBx",
+	"jyapi-appkey":"349300qe",
+	"task-cron":"*/5 * * * * ?",
+	"day":0,
+	"runmode":1,
+	"desc-2":"描述配置 jy-apiurl/getdata/getalldata:剑鱼api地址 | jyapi-appid/appkey 用户身份 | task-cron cron表达式 | day 传递参数,0取今天数据,-1取昨天数据,-2取前天数据 | runmode 运行模式，1(全量)、2(增量)为生产运行模式程序按cron配置永久运行，0为初始模式只取一次便退出"
+}

jyclient/src/db.sql

@@ -0,0 +1,38 @@
+CREATE TABLE `jydata` (
+  `i_id` int(11) NOT NULL AUTO_INCREMENT,
+  `title` varchar(1024) NOT NULL COMMENT '标题',
+  `detail` text COMMENT '正文',
+  `publishtime` datetime NOT NULL COMMENT '发布时间',
+  `href` varchar(1024) NOT NULL COMMENT '链接',
+  `area` varchar(20) DEFAULT NULL COMMENT '省份',
+  `toptype` varchar(20) NOT NULL COMMENT '信息类型',
+  `budget` double DEFAULT NULL COMMENT '预算',
+  `bidamount` double DEFAULT NULL COMMENT '中标金额',
+  `bidopentime` datetime DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '开标时间',
+  `buyer` varchar(255) DEFAULT NULL COMMENT '采购单位',
+  `buyertel` varchar(50) DEFAULT NULL COMMENT '采购人电话',
+  `buyerperson` varchar(50) DEFAULT NULL COMMENT '采购联系人',
+  `city` varchar(20) DEFAULT NULL COMMENT '城市',
+  `id` char(24) NOT NULL COMMENT '唯一标识',
+  `matchscore` int(11) DEFAULT NULL COMMENT '匹配值',
+  `department` varchar(50) DEFAULT NULL COMMENT '部门',
+  `remark` varchar(50) DEFAULT NULL COMMENT '备注',
+  `priority` int(11) DEFAULT NULL COMMENT '预算级别',
+  `industry` varchar(50) DEFAULT NULL COMMENT '行业',
+  `matchtype` varchar(50) DEFAULT NULL COMMENT '匹配类型',
+  `matchkey` varchar(500) DEFAULT NULL COMMENT '匹配关键字',
+  PRIMARY KEY (`i_id`),
+  UNIQUE KEY `id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+CREATE TABLE `jylog` (
+  `id` int(11) NOT NULL,
+  `size` int(11) DEFAULT '0' COMMENT '数据条数',
+  `new_size` int(11) DEFAULT '0' COMMENT '获取新数据条数',
+  `available_times` int(11) DEFAULT '0' COMMENT '当天剩余调用次数',
+  `available_infos` int(11) DEFAULT '0' COMMENT '剩余信息量',
+  `status` tinyint(4) DEFAULT '0' COMMENT '请求状态1正常，0错误',
+  `next` int(11) DEFAULT '0' COMMENT '请求位置参数',
+  `createtime` datetime DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '请求时间',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

jyclient/src/github.com/aiwuTech/fileLogger

@@ -0,0 +1 @@
+Subproject commit fdc0c5b08dd6605e087f5b751e1d2d95e7f645ce

jyclient/src/github.com/go-sql-driver/mysql

@@ -0,0 +1 @@
+Subproject commit d523deb1b23d913de5bdada721a6071e71283618

jyclient/src/github.com/robfig/cron

@@ -0,0 +1 @@
+Subproject commit b41be1df696709bb6395fe435af20370037c0b4c

jyclient/src/jyclient.log.1

@@ -0,0 +1,13 @@
+2018/06/12 18:04:16.376732 [task.go:22][TRACE] task  %!(EXTRA string=*/20 * * * * *)
+2018/06/12 18:05:37.039503 [task.go:22][TRACE] task,*/20 * * * * *  
+2018/06/12 18:11:22.547512 [task.go:22][TRACE] task,*/20 * * * * *  
+2018/06/12 18:12:41.807282 [task.go:23][TRACE] task,*/20 * * * * *  
+2018/06/12 18:17:41.784017 ================ LOG SEQ SIZE:0 ==================
+2018/06/12 18:31:55.201337 [task.go:23][TRACE] task,*/20 * * * * ?  
+2018/06/12 18:32:37.831699 [task.go:23][TRACE] task,*/20 * * * * ?  
+2018/06/12 18:33:41.967524 [task.go:23][TRACE] task,*/20 * * * * ?  
+2018/06/12 18:34:00.733459 [task.go:23][TRACE] task,*/20 * * * * ?  
+2018/06/12 18:36:08.149772 [task.go:23][TRACE] task,*/20 * * * * ?  
+2018/06/12 18:37:13.244480 [task.go:23][TRACE] task,*/20 * * * * ?  
+2018/06/12 18:45:01.267783 [task.go:23][TRACE] task,*/20 * * * * ?  
+2018/06/12 18:50:01.243909 ================ LOG SEQ SIZE:0 ==================

jyclient/src/main.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"log"
+	"time"
+)
+
+func main() {
+	log.Println("启动剑鱼数据同步程序")
+	go RunTask()
+	time.Sleep(99999 * time.Hour)
+}

jyclient/src/task.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/url"
+	"qfw/util"
+	"time"
+
+	"github.com/robfig/cron"
+)
+
+var expires = int64(7100)
+
+func RunTask() {
+	taskcron, _ := config["task-cron"].(string)
+	jy := &jyapi{
+		appid: config["jyapi-appid"].(string),
+		key:   config["jyapi-appkey"].(string),
+		mode:  util.IntAll(config["runmode"]),
+		day:   fmt.Sprintf("%d", config["day"]),
+	}
+	logFile.T("task,%s", taskcron)
+	task := cron.New()
+	task.AddJob(taskcron, jy)
+	task.Start()
+}
+
+type jyapi struct {
+	token     string
+	tokenTime int64
+	appid     string
+	key       string
+	mode      int
+	day       string
+}
+
+func (j *jyapi) Run() {
+	log.Println("run")
+	b := false
+	if time.Now().Unix()-j.tokenTime > expires {
+		b = j.GetToken()
+		if !b {
+			time.Sleep(60 * time.Second)
+			b = j.GetToken()
+		}
+	} else {
+		b = true
+	}
+	if b {
+		if j.mode == 1 {
+			s_url, _ := config["jyapi-getalldata"].(string)
+			m, b := POST(s_url, url.Values{"day": []string{j.day}, "access_token": []string{j.token}})
+			if b && m != nil {
+				code := m["code"]
+				if code == nil {
+					data, _ := m["data"].([]interface{})
+					available_times, _ := m["available_times"].(int)
+					if data == nil || len(data) == 0 {
+						logFile.W("运行无结果:%v", b, available_times)
+					} else {
+						logFile.I("运行结果:%v", data)
+					}
+				} else {
+					msg := m["msg"]
+					logFile.W("运行无结果:%s", msg)
+				}
+			} else {
+				logFile.W("运行无结果:%v", b)
+			}
+		}
+	}
+}
+
+func (j *jyapi) GetToken() bool {
+	s_url := config["jyapi-token"].(string)
+	m, b := POST(s_url, url.Values{"appid": []string{j.appid}, "key": []string{j.key}})
+	if b && m != nil {
+		token := m["access_token"].(string)
+		times := util.IntAll(m["available_times"])
+		if token == "" && times == 0 {
+			logFile.W("获取token达到上限")
+		} else {
+			j.token, _ = url.QueryUnescape(token)
+			j.tokenTime = time.Now().Unix()
+			return true
+		}
+	} else {
+		logFile.E("获取token出错")
+	}
+	return false
+}

jyclient/src/util.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"qfw/util"
+
+	_ "github.com/go-sql-driver/mysql"
+
+	"github.com/aiwuTech/fileLogger"
+)
+
+var (
+	config   map[string]interface{}
+	DB       *sql.DB
+	storesmt *sql.Stmt
+	logFile  *fileLogger.FileLogger
+)
+
+func init() {
+	ReadConfig(&config)
+	logFile = fileLogger.NewDefaultLogger("./", config["log-filename"].(string))
+	level, _ := config["log-level"].(int)
+	var l fileLogger.LEVEL
+	switch level {
+	case 0:
+		l = fileLogger.TRACE
+	case 1:
+		l = fileLogger.INFO
+	case 2:
+		l = fileLogger.WARN
+	case 3:
+		l = fileLogger.ERROR
+	case 4:
+		l = fileLogger.OFF
+
+	}
+	logFile.SetLogLevel(l)
+	logFile.SetMaxFileCount(3)
+	logFile.SetMaxFileSize(1, fileLogger.KB)
+	dns, _ := config["mysql-url"].(string)
+	pool := util.IntAllDef(config["mysql-pool"], 5)
+	InitDb(dns, 3000, pool)
+}
+
+//数据库封装
+func InitDb(dns string, idle, maxpool int) {
+	var err error
+	DB, err = sql.Open("mysql", dns)
+	if err != nil {
+		logFile.E("数据库连接错误,程序退出", err)
+		os.Exit(0)
+	}
+	DB.SetMaxIdleConns(idle)
+	DB.SetMaxOpenConns(maxpool)
+}
+
+func ReadConfig(config ...interface{}) {
+	var r *os.File
+	if len(config) > 1 {
+		filepath, _ := config[0].(string)
+		r, _ = os.Open(filepath)
+		defer r.Close()
+		bs, _ := ioutil.ReadAll(r)
+		json.Unmarshal(bs, config[1])
+	} else {
+		r, _ = os.Open("./config.json")
+		defer r.Close()
+		bs, _ := ioutil.ReadAll(r)
+		json.Unmarshal(bs, config[0])
+	}
+}
+
+func POST(addr string, data url.Values) (m map[string]interface{}, res bool) {
+	defer util.Catch()
+	resp, err := http.PostForm(addr, data)
+	if err != nil {
+		logFile.E("post请求错误:%s,%s,%v", addr, data, err)
+		return
+	}
+	defer resp.Body.Close()
+	bs, err := ioutil.ReadAll(resp.Body)
+	if err == nil {
+		ej := json.Unmarshal(bs, &m)
+		if ej == nil {
+			res = true
+		}
+	} else {
+		logFile.E("post请求错误", addr, data, err)
+	}
+	return
+}

jyservice/src/usermanager/usermanager.go

@@ -22,9 +22,9 @@ var (
 
 func init() {
 	//用户使用地址
-	Route.HandleFunc("/user/access_token", GetAccessToken).Methods("POST")
-	Route.HandleFunc("/data/getdata", GetData).Methods("POST")
-	Route.HandleFunc("/data/getalldata", GetAllData).Methods("POST")
+	Route.HandleFunc("/user/access_token", GetAccessToken).Methods("GET", "POST")
+	Route.HandleFunc("/data/getdata", GetData).Methods("GET", "POST")
+	Route.HandleFunc("/data/getalldata", GetAllData).Methods("GET", "POST")
 	//管理地址
 	Route.HandleFunc("/api1/user/jy_newuser", NewUser).Methods("POST")
 	Route.HandleFunc("/api1/user/jy_newservice", NewService).Methods("POST")