// main_test package main import ( "log" "luaweb/spider" "luaweb/util" mgdb "qfw/util/mongodb" "testing" "time" "github.com/lauyoume/gopinyin" ) //模板测试 func Test_Tmp(t *testing.T) { proficient := "" //通用变量配置 common := []interface{}{ "upload_test", "中央采购网-上传测试", "测试脚本栏目", true, 1, 10, 30, "bidding", "utf8", 1, 4002, "http://www.ccgp.gov.cn/zycg/zycgdt/index.htm", } //最新时间配置 ptime := []interface{}{ "yyyyMMddHHmm", "http://www.ccgp.gov.cn/zycg/zycgdt/index.htm", "ul li em:eq(0)", } // proficient = `function getLastPublishTime() // local content = download("",{}) // return lastpushtime // end` log.Println(spider.GetLastPublishTime(common, ptime, proficient, 0)) //列表页配置 list := []interface{}{ "http://www.ccgp.gov.cn/zycg/zycgdt/index#pageno#.htm", "'http://www.ccgp.gov.cn/zycg/zycgdt/index.htm','http://www.ccgp.gov.cn/zycg/zycgdt/index_1.htm'", "ul#main_list_lt_list>li", "a:eq(1):attr(href)", "a:eq(1):attr(title)", "em:eq(0)", "yyyyMMddHHmm", } model := map[string]interface{}{} model["type"] = "other" model["area"] = "HA" model["city"] = "郑州" model["publishdept"] = "郑州市财政厅" // proficient = `function downloadAndParseListPage(pageno) // end` log.Println(spider.GetPageList(common, list, model, proficient, 0)) content := []interface{}{ "div.TRS_Editor", "div.TRS_Editor", } data := map[string]interface{}{} data["title"] = "苏州市召开市级政府采购信用融资工作座谈会" data["href"] = "http://www.zfcg.suzhou.gov.cn/html/content/20160729165741501.shtml" data["publishtime"] = "2016-08-17 12:12:12" // proficient = `function downloadDetailPage(data) // end` log.Println(spider.GetContentInfo(common, content, data, proficient, 0)) } //保存更新爬虫 func Test_saveSpider(t *testing.T) { mgdb.InitMongodbPool(2, "192.168.3.18:27080", "luaweb") param := map[string]interface{}{} //通用变量配置 common := []interface{}{ "upload_test", "中央采购网-上传测试", "测试脚本栏目", true, 1, 10, 30, "bidding", "utf8", 1, 4002, "http://www.ccgp.gov.cn/zycg/zycgdt/index.htm", } //最新时间配置 ptime := []interface{}{ "yyyyMMddHHmm", "http://www.ccgp.gov.cn/zycg/zycgdt/", "ul li em:eq(0)", } param["type_time"] = 0 //0向导模式 1专家模式 //列表页配置 list := []interface{}{ "http://www.ccgp.gov.cn/zycg/zycgdt/index#pageno#.htm", "'http://www.ccgp.gov.cn/zycg/zycgdt/index.htm','http://www.ccgp.gov.cn/zycg/zycgdt/index_1.htm'", "ul#main_list_lt_list li", "a:eq(1):attr(href)", "a:eq(1):attr(title)", "em:eq(0)", "yyyyMMddHHmm", } param["type_list"] = 0 //0向导模式 1专家模式 //三级页配置 content := []interface{}{ "div.TRS_Editor", "div.TRS_Editor", } param["type_content"] = 0 //0向导模式 1专家模式 param["param_common"] = common //向导模式 param["param_time"] = ptime param["param_list"] = list param["param_content"] = content //专家模式 param["str_time"] = `function getLastPublishTime() local content = download("href",{}) return lastpushtime end` param["str_list"] = `function downloadAndParseListPage(pageno) end` param["str_content"] = `function downloadDetailPage(data) end` param["comeintime"] = time.Now().Unix() param["model"] = map[string]interface{}{ "type": "tender", "area": "HA", "city": "郑州", "publishdept": "郑州市财政厅", } //补充数据模型 param["createuser"] = "zjk" //姓名 param["upload"] = false //是否上传 spider.SaveSpider("upload_test", param) } //生成lua文件测试 func Test_createFile(t *testing.T) { mgdb.InitMongodbPool(2, "192.168.3.18:27080", "luaweb") scritp := spider.GetScript("upload_test") if scritp != "" { _, err := spider.CreateFile("upload_test", scritp) log.Println(err) } } //上传脚本 func Test_uploadFile(t *testing.T) { util.InitMsgClient(util.Config.Msgserveraddr, util.Config.Msgname) mgdb.InitMongodbPool(2, "192.168.3.18:27080", "luaweb") ret, err := spider.UpLoadScript("upload_test", 7001) if err != nil { log.Println("err", err) } else { log.Println("ret", ret) } } //获取拼音首子母 func Test_getFirstName(t *testing.T) { str := gopinyin.Convert("HA_河南省政府采购网_招标公告", true) log.Println(str) } func TestCheck(t *testing.T) { steps := make([]bool, 3) s := spider.CreateSpider(spider.GetScript("zgwlzbw_zbgg")) s.SpiderMaxPage = 1 time, timeerr := s.GetLastPublishTime() if timeerr != nil || len(time) < 5 { steps[0] = false return } else { list, listerr := s.DownListPageItem() if listerr != nil || len(list) == 0 { steps[0] = false } else { steps[0] = true listone := list[0] if len(listone["href"].(string)) < 7 || len(listone["publishtime"].(string)) < 5 || len(listone["title"].(string)) < 10 { steps[1] = false } else { steps[1] = true param := map[string]string{} param["title"] = list[0]["title"].(string) param["href"] = list[0]["href"].(string) param["publishtime"] = list[0]["publishtime"].(string) data := map[string]interface{}{} s.DownloadDetailPage(param, data) if len(data) == 0 || len(data["detail"].(string)) < 50 { steps[2] = false } else { steps[2] = true } } } } }