main.go 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567
  1. package main
  2. import (
  3. "cluster"
  4. "config"
  5. "encoding/json"
  6. "fmt"
  7. "github.com/cron"
  8. "gopkg.in/mgo.v2/bson"
  9. "html/template"
  10. "log"
  11. "math"
  12. mu "mfw/util"
  13. "net"
  14. "net/http"
  15. qu "qfw/util"
  16. "qfw/util/mongodb"
  17. "regexp"
  18. "strings"
  19. "sync"
  20. "time"
  21. )
  22. var udpclient mu.UdpClient //udp对象
  23. var sys sync.RWMutex
  24. var DeleteNode = time.NewTimer(time.Minute * 50)
  25. func main() {
  26. udpclient = mu.UdpClient{Local: config.Sysconfig["udpip"].(string) + ":" + config.Sysconfig["udpport"].(string), BufSize: 1024}
  27. udpclient.Listen(processUdpMsg)
  28. log.Printf("Udp listening port: %s:%s\n", config.Sysconfig["udpip"], config.Sysconfig["udpport"])
  29. if config.Sysconfig["broadcast"].(bool) { //重启的话通知分布式节点
  30. ips := qu.ObjToString(config.Sysconfig["broadcast_ips"])
  31. ipsArr := strings.Split(ips, ";")
  32. for _, v := range ipsArr {
  33. udpclient.WriteUdp([]byte{}, mu.OP_TYPE_DATA, &net.UDPAddr{
  34. IP: net.ParseIP(v),
  35. Port: qu.IntAll(config.Sysconfig["broadcast_port"]),
  36. })
  37. }
  38. }
  39. mux := http.NewServeMux()
  40. mux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
  41. http.Redirect(writer, request, "/login", http.StatusFound)
  42. })
  43. mux.HandleFunc("/favicon.ico", func(writer http.ResponseWriter, request *http.Request) {
  44. writer.WriteHeader(http.StatusOK)
  45. })
  46. mux.HandleFunc("/login", func(writer http.ResponseWriter, request *http.Request) {
  47. tp, err := template.ParseFiles("src/web/login.html")
  48. if err != nil {
  49. log.Println(err)
  50. writer.Write([]byte( "页面找不到了~"))
  51. return
  52. }
  53. if request.Method == "GET" {
  54. tp.Execute(writer, "")
  55. return
  56. }
  57. if request.Method == "POST" {
  58. err := request.ParseForm()
  59. if err != nil {
  60. http.Error(writer, err.Error(), http.StatusBadRequest)
  61. return
  62. }
  63. email := request.Form.Get("username")
  64. pwd := request.Form.Get("pwd")
  65. if email == "ocr_task" && pwd == "ocr_task_pwd" {
  66. http.SetCookie(writer, &http.Cookie{Name: "username", Value: qu.GetMd5String("email")})
  67. http.Redirect(writer, request, "/query", http.StatusFound)
  68. } else {
  69. writer.WriteHeader(http.StatusUnauthorized)
  70. tp.Execute(writer, "密码错误")
  71. }
  72. return
  73. }
  74. })
  75. res := http.FileServer(http.Dir("src/res"))
  76. mux.Handle("/res/", http.StripPrefix("/res/", res))
  77. mux.HandleFunc("/query", func(writer http.ResponseWriter, request *http.Request) {
  78. tplogin, err := template.ParseFiles("src/web/login.html")
  79. if err != nil {
  80. log.Println(err)
  81. writer.Write([]byte( "页面找不到了~"))
  82. return
  83. }
  84. cookie, _ := request.Cookie("username")
  85. if cookie == nil {
  86. http.RedirectHandler("/login", http.StatusUnauthorized)
  87. tplogin.Execute(writer, "请先登录")
  88. return
  89. }
  90. if cookie.Value != "0c83f57c786a0b4a39efab23731c7ebc" {
  91. http.RedirectHandler("/login", http.StatusUnauthorized)
  92. tplogin.Execute(writer, "密钥错误")
  93. return
  94. }
  95. tp, err := template.ParseFiles("src/web/index.html", "src/public/header.html")
  96. if err != nil {
  97. log.Println(err)
  98. writer.Write([]byte( "页面找不到了~"))
  99. return
  100. }
  101. task := queryOcrTask()
  102. tp.Execute(writer, task)
  103. })
  104. mux.HandleFunc("/reload", func(writer http.ResponseWriter, request *http.Request) {
  105. tplogin, err := template.ParseFiles("src/web/login.html")
  106. if err != nil {
  107. log.Println(err)
  108. writer.Write([]byte( "页面找不到了~"))
  109. return
  110. }
  111. cookie, _ := request.Cookie("username")
  112. if cookie == nil {
  113. http.RedirectHandler("/login", http.StatusUnauthorized)
  114. tplogin.Execute(writer, "请先登录")
  115. return
  116. }
  117. tpReload, err := template.ParseFiles("src/web/reload.html")
  118. if err != nil {
  119. log.Println(err)
  120. writer.Write([]byte( "页面找不到了~"))
  121. return
  122. }
  123. if request.Method == "POST" {
  124. err = request.ParseForm()
  125. if err != nil {
  126. tpReload.Execute(writer, err)
  127. return
  128. }
  129. data := request.PostForm["ips"]
  130. if len(data) <= 0 {
  131. tpReload.Execute(writer, "参数无效")
  132. return
  133. }
  134. var tmpstr string
  135. for _, v := range data {
  136. if !regexp.MustCompile("((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})(\\.((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})){3}").MatchString(v) {
  137. tmpstr += v + "ip格式不正确</br>"
  138. continue
  139. }
  140. result := reload(v)
  141. tmpstr += v + "---->" + result + "</br>"
  142. log.Println(v, "重新部署完成", )
  143. }
  144. tpReload.Execute(writer, template.HTML(
  145. "</br><span style=\"color: red\">"+tmpstr+" 重新部署结束</span></br>"+
  146. "</br>"+
  147. "<form action=\"/reload\" method=\"post\">"+
  148. "<input id=\"ips1\" name=\"ips\" placeholder=\"请输入实例ip\" required/></br>"+
  149. "<input type=\"submit\" value=\"提交\" />"+
  150. "</form>"))
  151. return
  152. }
  153. tpReload.Execute(writer, template.HTML("<form action=\"/reload\" method=\"post\">"+
  154. "<input id=\"ips1\" name=\"ips\" placeholder=\"请输入实例ip\" required/></br>"+
  155. "<input type=\"submit\" value=\"提交\" />"+
  156. "</form>"))
  157. })
  158. c := cron.New()
  159. spec := qu.ObjToString(config.Sysconfig["cornstr"])
  160. c.AddFunc(spec, func() {
  161. d := time.Now()
  162. nowday := time.Date(d.Year(), d.Month(), d.Day(), 0, 0, 0, 0, d.Location())
  163. cluster.DescribeInstances() //查询多台实例的详细信息
  164. ocrescs := mongodb.Find("ocr_ecs", bson.M{}, nil, bson.M{"AutoReleaseTime": 1}, false, -1, -1)
  165. if ocrescs != nil || len(*ocrescs) > 0 {
  166. for _, v := range (*ocrescs) {
  167. if qu.ObjToString(v["AutoReleaseTime"]) != "" {
  168. utc, err := time.ParseInLocation("2006-01-02T15:04Z", qu.ObjToString(v["AutoReleaseTime"]), time.Local)
  169. if err != nil {
  170. log.Println("解析时间异常", err)
  171. continue
  172. }
  173. if utc.Before(nowday) {
  174. log.Println("删除过时实例", mongodb.Del("ocr_ecs", bson.M{"_id": v["_id"].(bson.ObjectId)}))
  175. }
  176. }
  177. }
  178. }
  179. taskArr := mongodb.Find("ocr_task", bson.M{}, `{_id:1}`, nil, false, -1, -1)
  180. taskNum := len(*taskArr)
  181. log.Println("当前任务数量:", taskNum)
  182. //if taskNum <= 0 {
  183. //计算释放,发送udp
  184. ccnum := compute()
  185. log.Println("ccnum:", ccnum, ", len(config.CID):", len(config.CID), config.CID)
  186. if ccnum <= 0 {
  187. if len(config.CID) == 0 {
  188. log.Println("当前实例为空,无需释放", config.CID, )
  189. return
  190. }
  191. if ccnum == 0 && len(config.CID) > 0 {
  192. log.Println("释放所有实例", config.CID)
  193. for tmpIid, _ := range config.CID {
  194. ttt := mongodb.FindOne("ocr_ecs", bson.M{"InstanceId": tmpIid})
  195. mongodb.Del("ocr_ecs", bson.M{"InstanceId": tmpIid})
  196. log.Println("5分钟后释放实例", tmpIid)
  197. udpclient.WriteUdp([]byte("5分钟后释放实例"), mu.OP_DELETE_DOWNLOADERCODES, &net.UDPAddr{
  198. IP: net.ParseIP(qu.ObjToString((*ttt)["ip_nw"])),
  199. Port: qu.IntAll(config.Sysconfig["broadcast_port"]),
  200. })
  201. go func(tmpIid string) {
  202. time.Sleep(time.Minute * 5)
  203. cluster.DeleteInstance(tmpIid)
  204. log.Println("5分钟后释放实例完成", tmpIid)
  205. }(tmpIid)
  206. }
  207. sys.Lock()
  208. config.CID = make(map[string]bool)
  209. sys.Unlock()
  210. } else {
  211. var tmpnum int
  212. for k, _ := range config.CID {
  213. if ccnum*(-1) >= tmpnum {
  214. return
  215. }
  216. tmpIid := k
  217. ttt := mongodb.FindOne("ocr_ecs", bson.M{"InstanceId": tmpIid})
  218. mongodb.Del("ocr_ecs", bson.M{"InstanceId": tmpIid})
  219. log.Println("5分钟后释放实例", tmpIid)
  220. udpclient.WriteUdp([]byte("5分钟后释放实例"), mu.OP_DELETE_DOWNLOADERCODES, &net.UDPAddr{
  221. IP: net.ParseIP(qu.ObjToString((*ttt)["ip_nw"])),
  222. Port: qu.IntAll(config.Sysconfig["broadcast_port"]),
  223. })
  224. go func(tmpIid string) {
  225. time.Sleep(time.Minute * 5)
  226. cluster.DeleteInstance(tmpIid)
  227. log.Println("5分钟后释放实例完成", tmpIid)
  228. }(tmpIid)
  229. sys.Lock()
  230. delete(config.CID, k)
  231. sys.Unlock()
  232. }
  233. }
  234. return
  235. }
  236. //}
  237. if len(config.CID) >= qu.IntAll(config.Sysconfig["pernum"]) {
  238. log.Println("实例申请上限,当前实例:", config.CID)
  239. escObject := mongodb.Find("ocr_ecs", bson.M{"TaskName": "ocr_task_arr", "OcrTaskStatus": "none"}, bson.M{"_id": -1}, nil, false, -1, -1)
  240. log.Println("实例未部署数量", len(*escObject))
  241. if escObject != nil || len(*escObject) > 0 {
  242. for i, v := range (*escObject) {
  243. if tmpip := qu.ObjToString(v["ip_nw"]); tmpip == "" {
  244. log.Println("没用获取到ip,实例ip异常", tmpip)
  245. } else {
  246. go func(tmpip, InstanceId string, i int) {
  247. if DoCMD(tmpip) {
  248. var tmpstr string
  249. isok2, udpstr := cluster.SshPgrep(tmpip, "pgrep udp2019")
  250. tmpstr += udpstr + ";&nbsp;&nbsp;"
  251. isok3, fil2textstr := cluster.SshPgrep(tmpip, "pgrep file2text")
  252. tmpstr += fil2textstr
  253. if isok2 && isok3 {
  254. (*escObject)[i]["OcrTaskStatus"] = "successful"
  255. mongodb.Update("ocr_ecs", bson.M{"_id": (*escObject)[i]["_id"]}, (*escObject)[i], true, false)
  256. sys.Lock()
  257. if config.CID["InstanceId"] == false {
  258. config.CID["InstanceId"] = true
  259. }
  260. sys.Unlock()
  261. log.Println((*escObject)[i]["_id"], tmpip, "部署成功")
  262. } else {
  263. log.Println(tmpip, "部署异常,"+tmpstr)
  264. }
  265. } else {
  266. log.Println(tmpip, "部署失败")
  267. }
  268. }(tmpip, qu.ObjToString(v["InstanceId"]), i)
  269. }
  270. }
  271. }
  272. return
  273. }
  274. //if taskNum > 1 {
  275. log.Println("申请实例")
  276. now := time.Now()
  277. hours := time.Date(now.Year(), now.Month(), now.Day(), 20, 0, 0, 0, now.Location()).Sub(now).Hours()
  278. cluster.RunInstances("ocr_task_arr", "8", "false", ccnum, int(math.Round(hours)))
  279. log.Println("实例申请成功")
  280. DynamicTask() //动态任务
  281. log.Println("申请实例结束")
  282. //}
  283. })
  284. c.Start()
  285. log.Println("Http listening port: ", qu.ObjToString(config.Sysconfig["http_port"]))
  286. if err := http.ListenAndServe(":"+qu.ObjToString(config.Sysconfig["http_port"]), mux); err != nil {
  287. fmt.Println("start http server faild:", err)
  288. }
  289. }
  290. func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
  291. defer qu.Catch()
  292. switch act {
  293. case mu.OP_TYPE_DATA: //保存服务
  294. go udpclient.WriteUdp([]byte("数据接收成功:"+string(data)), mu.OP_NOOP, ra)
  295. tmp := make(map[string]interface{})
  296. err := json.Unmarshal(data, &tmp)
  297. if err != nil {
  298. go udpclient.WriteUdp([]byte("数据接收成功:"+string(data)+",data解析失败,"+err.Error()), mu.OP_NOOP, ra)
  299. return
  300. }
  301. tmp["start"] = tmp[qu.ObjToString(config.SidField)]
  302. tmp["import_time"] = time.Now().Unix()
  303. bytes, _ := json.Marshal(tmp)
  304. b := mongodb.Save("ocr_task", string(bytes))
  305. mongodb.Save("ocr_task_bak", string(bytes))
  306. log.Println("保存id:", b)
  307. case mu.OP_NOOP: //其他节点回应消息打印
  308. log.Println("节点接收成功", string(data), ra.String())
  309. case mu.OP_GET_DOWNLOADERCODE: //分发任务
  310. if `{"permission":"get_ocr_task"}` != string(data) {
  311. log.Println("没有权限:", string(data), ra)
  312. go udpclient.WriteUdp([]byte("没有权限"), mu.OP_NOOP, ra)
  313. return
  314. }
  315. sys.Lock()
  316. datas := mongodb.Find("ocr_task", nil, `{"_id":1}`, nil, false, -1, -1)
  317. if len(*datas) == 0 {
  318. go udpclient.WriteUdp([]byte("没有新数据"), mu.OP_TYPE_DATA, ra)
  319. sys.Unlock()
  320. return
  321. }
  322. tmp := (*datas)[0]
  323. ObjectId := tmp["_id"]
  324. sid := qu.ObjToString(tmp[qu.ObjToString(config.SidField)])
  325. eid := qu.ObjToString(tmp[qu.ObjToString(config.EidField)])
  326. rdata := mongodb.FindOneByField(config.MgoC, bson.M{"_id": bson.M{
  327. "$gt": bson.ObjectIdHex(sid),
  328. }}, `{"_id":1,"`+config.MgoFileFiled+`":1}`)
  329. //log.Println(rdata)
  330. if len((*rdata)) == 0 {
  331. go udpclient.WriteUdp([]byte("没有获取到最新数据"), mu.OP_TYPE_DATA, ra)
  332. sys.Unlock()
  333. return
  334. }
  335. newId := (*rdata)["_id"]
  336. if newId.(bson.ObjectId).Hex() >= eid {
  337. go udpclient.WriteUdp([]byte(`{"id":"`+qu.ObjToString(tmp["start"])+`","permission":"ocr_task","is_start":"true"}`), mu.OP_TYPE_DATA, ra) //起始位置
  338. go udpclient.WriteUdp([]byte(`{"id":"`+newId.(bson.ObjectId).Hex()+`","permission":"ocr_task"}`), mu.OP_TYPE_DATA, ra) //分发任务
  339. totmp := make(map[string]string)
  340. totmp["sid"] = qu.ObjToString(tmp[qu.ObjToString("start")])
  341. totmp["eid"] = qu.ObjToString(tmp[qu.ObjToString(config.EidField)])
  342. tobyte, _ := json.Marshal(totmp)
  343. go udpclient.WriteUdp(tobyte, mu.OP_TYPE_DATA, &net.UDPAddr{
  344. IP: net.ParseIP(qu.ObjToString(config.Sysconfig["toudpip"])),
  345. Port: qu.IntAll(config.Sysconfig["toudpport"]),
  346. })
  347. log.Println("ocr_task处理完成,发送下个节点", string(tobyte))
  348. mongodb.Del("ocr_task", bson.M{"_id": ObjectId.(bson.ObjectId)})
  349. tmp["end_time"] = time.Now().Unix()
  350. mongodb.Update("ocr_task_bak", bson.M{"_id": ObjectId.(bson.ObjectId)}, map[string]interface{}{"$set": tmp}, true, false)
  351. sys.Unlock()
  352. return
  353. }
  354. go udpclient.WriteUdp([]byte(`{"id":"`+newId.(bson.ObjectId).Hex()+`","permission":"ocr_task"}`), mu.OP_TYPE_DATA, ra) //分发任务
  355. //log.Println(newId.(bson.ObjectId).Hex())
  356. tmp[config.SidField] = newId.(bson.ObjectId).Hex()
  357. mongodb.Update("ocr_task",
  358. bson.M{"_id": ObjectId.(bson.ObjectId)}, tmp, false, false)
  359. sys.Unlock()
  360. }
  361. }
  362. func queryOcrTask() map[string]int {
  363. data := make(map[string]int)
  364. taskArr := mongodb.Find("ocr_task", bson.M{}, `{_id:1}`, nil, false, -1, -1)
  365. taskNum := len(*taskArr)
  366. if taskNum == 0 {
  367. return data
  368. }
  369. data["taskNum"] = taskNum
  370. sumNum := 0
  371. nowSumNum := 0
  372. for i, v := range *taskArr {
  373. sid := bson.ObjectIdHex(qu.ObjToString(v["start"]))
  374. eid := bson.ObjectIdHex(qu.ObjToString(v[qu.ObjToString(config.EidField)]))
  375. sumNum += mongodb.Count("bidding", bson.M{"_id": bson.M{
  376. "$gte": sid,
  377. "$lte": eid,
  378. }})
  379. if i == 0 {
  380. nowSumNum = sumNum
  381. }
  382. }
  383. data["sumNum"] = sumNum
  384. data["nowSumNum"] = nowSumNum
  385. tmpsid := bson.ObjectIdHex(qu.ObjToString((*taskArr)[0]["start"]))
  386. over := (*taskArr)[0][qu.ObjToString(config.SidField)]
  387. overNum := mongodb.Count("bidding", bson.M{"_id": bson.M{
  388. "$gte": tmpsid,
  389. "$lt": bson.ObjectIdHex(qu.ObjToString(over)),
  390. }})
  391. data["overNum"] = overNum
  392. undoneNum := sumNum - overNum
  393. data["undoneNum"] = undoneNum
  394. nowUnDoneNum := nowSumNum - overNum
  395. data["nowUnDoneNum"] = nowUnDoneNum
  396. return data
  397. }
  398. func DynamicTask() {
  399. time.Sleep(time.Second * 30)
  400. cluster.DescribeInstances() //查询多台实例的详细信息
  401. escObject := mongodb.Find("ocr_ecs", bson.M{"TaskName": "ocr_task_arr", "OcrTaskStatus": "none"}, bson.M{"_id": -1}, nil, false, -1, -1)
  402. log.Println("实例未部署数量", len(*escObject))
  403. if escObject != nil || len(*escObject) > 0 {
  404. for i, v := range (*escObject) {
  405. if tmpip := qu.ObjToString(v["ip_nw"]); tmpip == "" {
  406. log.Println("没用获取到ip,实例ip异常", tmpip)
  407. DynamicTask()
  408. return
  409. } else {
  410. go func(tmpip,InstanceId string,i int) {
  411. if DoCMD(tmpip) {
  412. var tmpstr string
  413. isok2, udpstr := cluster.SshPgrep(tmpip, "pgrep udp2019")
  414. tmpstr += udpstr + ";&nbsp;&nbsp;"
  415. isok3, fil2textstr := cluster.SshPgrep(tmpip, "pgrep file2text")
  416. tmpstr += fil2textstr
  417. if isok2 && isok3 {
  418. (*escObject)[i]["OcrTaskStatus"] = "successful"
  419. mongodb.Update("ocr_ecs", bson.M{"_id": (*escObject)[i]["_id"]}, (*escObject)[i], true, false)
  420. sys.Lock()
  421. if config.CID[InstanceId] == false {
  422. config.CID[InstanceId] = true
  423. }
  424. sys.Unlock()
  425. log.Println((*escObject)[i]["_id"], tmpip, "部署成功")
  426. } else {
  427. log.Println(tmpip, "部署异常,"+tmpstr)
  428. }
  429. } else {
  430. log.Println(tmpip, "部署失败")
  431. }
  432. }(tmpip,qu.ObjToString(v["InstanceId"]),i)
  433. }
  434. }
  435. } else {
  436. log.Println("动态任务创建失败")
  437. }
  438. }
  439. func DoCMD(ip string) bool {
  440. if ip == "" {
  441. return false
  442. }
  443. return cluster.RunSsh(ip)
  444. }
  445. func reload(ip string) string {
  446. tmp := mongodb.FindOne("ocr_ecs", bson.M{"ip_nw": ip})
  447. if tmp == nil || len(*tmp) <= 0 {
  448. return "ip不存在"
  449. }
  450. sys.Lock()
  451. if config.CID[qu.ObjToString((*tmp)["InstanceId"])] == true{
  452. delete(config.CID,qu.ObjToString((*tmp)["InstanceId"]))
  453. }
  454. sys.Unlock()
  455. cluster.DeleteInstance(qu.ObjToString((*tmp)["InstanceId"])) //删除要重新部署的实例
  456. now := time.Now()
  457. hours := time.Date(now.Year(), now.Month(), now.Day(), 20, 0, 0, 0, now.Location()).Sub(now).Hours()
  458. cluster.RunInstances("ocr_task_arr", "8", "false", 1, int(math.Round(hours))) //创建新实例
  459. time.Sleep(time.Second * 30)
  460. cluster.DescribeInstances() //查询多台实例的详细信息
  461. escObject := mongodb.Find("ocr_ecs", bson.M{"TaskName": "ocr_task_arr", "OcrTaskStatus": "none"}, bson.M{"_id": -1}, nil, true, -1, -1)
  462. if escObject != nil || len(*escObject) > 0 {
  463. var tmpip string
  464. if tmpip = qu.ObjToString((*escObject)[0]["ip_nw"]); tmpip == "" {
  465. log.Println("没用获取到ip,实例ip异常", tmpip)
  466. time.Sleep(time.Minute)
  467. cluster.DescribeInstances() //查询多台实例的详细信息
  468. escObject = mongodb.Find("ocr_ecs", bson.M{"TaskName": "ocr_task_arr", "OcrTaskStatus": "none"}, bson.M{"_id": -1}, nil, true, -1, -1)
  469. tmpip = qu.ObjToString((*escObject)[0]["ip_nw"])
  470. }
  471. if DoCMD(tmpip) {
  472. var tmpstr string
  473. isok2, udpstr := cluster.SshPgrep(tmpip, "pgrep udp2019")
  474. tmpstr += udpstr + ";&nbsp;&nbsp;"
  475. isok3, fil2textstr := cluster.SshPgrep(tmpip, "pgrep file2text")
  476. tmpstr += fil2textstr
  477. if isok2 && isok3 {
  478. (*escObject)[0]["OcrTaskStatus"] = "successful"
  479. mongodb.Update("ocr_ecs", bson.M{"_id": (*escObject)[0]["_id"]}, (*escObject)[0], true, false)
  480. sys.Lock()
  481. if config.CID[qu.ObjToString((*escObject)[0]["InstanceId"])] == false {
  482. config.CID[qu.ObjToString((*escObject)[0]["InstanceId"])] = true
  483. }
  484. sys.Unlock()
  485. log.Println((*escObject)[0]["_id"], tmpip, "部署成功")
  486. return tmpip + "部署成功," + tmpstr
  487. } else {
  488. return tmpip + "部署异常," + tmpstr
  489. }
  490. } else {
  491. return tmpip + "部署失败"
  492. }
  493. }
  494. return "重新部署" + ip + "未知错误,查询多台实例的详细信息,没有查询到新创建实例"
  495. }
  496. func compute() int {
  497. nowtime := time.Now().Unix()
  498. taskArrase := mongodb.Find("ocr_task", bson.M{}, `{_id:1}`, nil, false, -1, -1)
  499. if taskArrase == nil || len(*taskArrase) == 0 {
  500. log.Println(464, "nil ro len(*taskArrase) == 0")
  501. return 0
  502. }
  503. stmp := (*taskArrase)[0]
  504. etmp := (*taskArrase)[len(*taskArrase)-1]
  505. if stmp != nil && etmp != nil {
  506. stime := qu.Int64All(stmp["import_time"])
  507. if nowtime-stime <= qu.Int64All(config.Sysconfig["time_consuming_limit"]) {
  508. log.Println(472, nowtime-stime, "<=", qu.Int64All(config.Sysconfig["time_consuming_limit"]))
  509. return 0
  510. }
  511. sid := bson.ObjectIdHex(qu.ObjToString(stmp["start"]))
  512. eid := bson.ObjectIdHex(qu.ObjToString(etmp[qu.ObjToString(config.EidField)]))
  513. sum := mongodb.Count("bidding", bson.M{"_id": bson.M{
  514. "$gte": sid,
  515. "$lt": eid,
  516. }})
  517. if sum <= qu.IntAll(config.Sysconfig["accumulated_task_limit"]) {
  518. log.Println(483, sum, sid, eid)
  519. return 0
  520. }
  521. gteid := bson.ObjectIdHex(qu.ObjToString(stmp[qu.ObjToString(config.SidField)]))
  522. overNum := mongodb.Count("bidding", bson.M{"_id": bson.M{
  523. "$gte": sid,
  524. "$lt": gteid,
  525. }})
  526. if overNum == 0 {
  527. log.Println(492, overNum, sid, gteid)
  528. return 0
  529. }
  530. if sum <= qu.IntAll(config.Sysconfig["accumulated_task_lowlimit"]) {
  531. log.Println(496, sum, "<=", qu.IntAll(config.Sysconfig["accumulated_task_lowlimit"]))
  532. return 0
  533. }
  534. mtmm := float64(overNum) / float64(nowtime-stime) / float64(len(config.CID)+3) //每台每秒
  535. if mtmm <= 0 {
  536. log.Println(501, overNum, int(nowtime-stime), (len(config.CID) + 3))
  537. return 0
  538. }
  539. cc := float64(sum)/float64(qu.IntAll(config.Sysconfig["corntime_consuming"]))/mtmm - float64(len(config.CID)) - 3
  540. log.Println("overNum:", overNum, ",hs:", int(nowtime-stime), ",mtms:", mtmm, ",sum:", sum, cc)
  541. if cc > qu.Float64All(config.Sysconfig["pernum"]) {
  542. return qu.IntAll(config.Sysconfig["pernum"])
  543. } else {
  544. return int(cc)
  545. }
  546. }
  547. return 0
  548. }