spider_editor/main_test.go

// main_test
package main

import (
	"log"
	"luaweb/spider"
	"luaweb/util"
	mgdb "qfw/util/mongodb"
	"testing"
	"time"

	"github.com/lauyoume/gopinyin"
)

//模板测试
func Test_Tmp(t *testing.T) {
	proficient := ""
	//通用变量配置
	common := []interface{}{
		"upload_test", "中央采购网-上传测试", "测试脚本栏目", true, 1, 10, 30, "bidding", "utf8", 1, 4002,
		"http://www.ccgp.gov.cn/zycg/zycgdt/index.htm",
	}
	//最新时间配置
	ptime := []interface{}{
		"yyyyMMddHHmm",
		"http://www.ccgp.gov.cn/zycg/zycgdt/index.htm",
		"ul li em:eq(0)",
	}
	//	proficient = `function getLastPublishTime()
	//					local content = download("",{})
	//					return lastpushtime
	//				end`
	log.Println(spider.GetLastPublishTime(common, ptime, proficient, 0))
	//列表页配置
	list := []interface{}{
		"http://www.ccgp.gov.cn/zycg/zycgdt/index#pageno#.htm",
		"'http://www.ccgp.gov.cn/zycg/zycgdt/index.htm','http://www.ccgp.gov.cn/zycg/zycgdt/index_1.htm'",
		"ul#main_list_lt_list>li",
		"a:eq(1):attr(href)",
		"a:eq(1):attr(title)",
		"em:eq(0)",
		"yyyyMMddHHmm",
	}
	model := map[string]interface{}{}
	model["type"] = "other"
	model["area"] = "HA"
	model["city"] = "郑州"
	model["publishdept"] = "郑州市财政厅"
	//	proficient = `function downloadAndParseListPage(pageno)
	//				end`
	log.Println(spider.GetPageList(common, list, model, proficient, 0))
	content := []interface{}{
		"div.TRS_Editor",
		"div.TRS_Editor",
	}
	data := map[string]interface{}{}
	data["title"] = "苏州市召开市级政府采购信用融资工作座谈会"
	data["href"] = "http://www.zfcg.suzhou.gov.cn/html/content/20160729165741501.shtml"
	data["publishtime"] = "2016-08-17 12:12:12"
	//	proficient = `function downloadDetailPage(data)
	//			end`
	log.Println(spider.GetContentInfo(common, content, data, proficient, 0))
}

//保存更新爬虫
func Test_saveSpider(t *testing.T) {
	mgdb.InitMongodbPool(2, "192.168.3.18:27080", "luaweb")
	param := map[string]interface{}{}
	//通用变量配置
	common := []interface{}{
		"upload_test", "中央采购网-上传测试", "测试脚本栏目", true, 1, 10, 30, "bidding", "utf8", 1, 4002,
		"http://www.ccgp.gov.cn/zycg/zycgdt/index.htm",
	}
	//最新时间配置
	ptime := []interface{}{
		"yyyyMMddHHmm",
		"http://www.ccgp.gov.cn/zycg/zycgdt/",
		"ul li em:eq(0)",
	}
	param["type_time"] = 0 //0向导模式 1专家模式

	//列表页配置
	list := []interface{}{
		"http://www.ccgp.gov.cn/zycg/zycgdt/index#pageno#.htm",
		"'http://www.ccgp.gov.cn/zycg/zycgdt/index.htm','http://www.ccgp.gov.cn/zycg/zycgdt/index_1.htm'",
		"ul#main_list_lt_list li",
		"a:eq(1):attr(href)",
		"a:eq(1):attr(title)",
		"em:eq(0)",
		"yyyyMMddHHmm",
	}
	param["type_list"] = 0 //0向导模式 1专家模式

	//三级页配置
	content := []interface{}{
		"div.TRS_Editor",
		"div.TRS_Editor",
	}
	param["type_content"] = 0 //0向导模式 1专家模式

	param["param_common"] = common
	//向导模式
	param["param_time"] = ptime
	param["param_list"] = list
	param["param_content"] = content
	//专家模式
	param["str_time"] = `function getLastPublishTime()
							local content = download("href",{})
							return lastpushtime
						end`
	param["str_list"] = `function downloadAndParseListPage(pageno)
	 					end`
	param["str_content"] = `function downloadDetailPage(data)
 							end`

	param["comeintime"] = time.Now().Unix()
	param["model"] = map[string]interface{}{
		"type":        "tender",
		"area":        "HA",
		"city":        "郑州",
		"publishdept": "郑州市财政厅",
	} //补充数据模型
	param["createuser"] = "zjk" //姓名
	param["upload"] = false     //是否上传
	spider.SaveSpider("upload_test", param)
}

//生成lua文件测试
func Test_createFile(t *testing.T) {
	mgdb.InitMongodbPool(2, "192.168.3.18:27080", "luaweb")
	scritp := spider.GetScript("upload_test")
	if scritp != "" {
		_, err := spider.CreateFile("upload_test", scritp)
		log.Println(err)
	}
}

//上传脚本
func Test_uploadFile(t *testing.T) {
	util.InitMsgClient(util.Config.Msgserveraddr, util.Config.Msgname)
	mgdb.InitMongodbPool(2, "192.168.3.18:27080", "luaweb")
	ret, err := spider.UpLoadScript("upload_test", 7001)
	if err != nil {
		log.Println("err", err)
	} else {
		log.Println("ret", ret)
	}
}

//获取拼音首子母
func Test_getFirstName(t *testing.T) {
	str := gopinyin.Convert("HA_河南省政府采购网_招标公告", true)
	log.Println(str)
}

func TestCheck(t *testing.T) {
	steps := make([]bool, 3)
	s := spider.CreateSpider(spider.GetScript("zgwlzbw_zbgg"))
	s.SpiderMaxPage = 1
	time, timeerr := s.GetLastPublishTime()
	if timeerr != nil || len(time) < 5 {
		steps[0] = false
		return
	} else {
		list, listerr := s.DownListPageItem()
		if listerr != nil || len(list) == 0 {
			steps[0] = false
		} else {
			steps[0] = true
			listone := list[0]
			if len(listone["href"].(string)) < 7 || len(listone["publishtime"].(string)) < 5 || len(listone["title"].(string)) < 10 {
				steps[1] = false
			} else {
				steps[1] = true
				param := map[string]string{}
				param["title"] = list[0]["title"].(string)
				param["href"] = list[0]["href"].(string)
				param["publishtime"] = list[0]["publishtime"].(string)
				data := map[string]interface{}{}
				s.DownloadDetailPage(param, data)
				if len(data) == 0 || len(data["detail"].(string)) < 50 {
					steps[2] = false
				} else {
					steps[2] = true
				}
			}
		}
	}

}