|
@@ -198,6 +198,7 @@ func FilterAndDownload(hrefMap map[string]string) (result []*Data) {
|
|
|
FileType: strings.ReplaceAll(fileType, ".", ""),
|
|
FileType: strings.ReplaceAll(fileType, ".", ""),
|
|
|
})
|
|
})
|
|
|
} else {
|
|
} else {
|
|
|
|
|
+ continue
|
|
|
//textStr = reg_fileter_text.ReplaceAllString(textStr, "") //过滤无效字符
|
|
//textStr = reg_fileter_text.ReplaceAllString(textStr, "") //过滤无效字符
|
|
|
if reg_invalid_text.ReplaceAllString(tmpText, "") == "" { //无效，全文本匹配，舍弃
|
|
if reg_invalid_text.ReplaceAllString(tmpText, "") == "" { //无效，全文本匹配，舍弃
|
|
|
continue
|
|
continue
|
|
@@ -344,7 +345,7 @@ func DealAndDownload(tmp []*Data, href string) (result []*Data, attachments, att
|
|
|
return
|
|
return
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-//下载
|
|
|
|
|
|
|
+// 下载
|
|
|
func Download(url string) (string, []byte) {
|
|
func Download(url string) (string, []byte) {
|
|
|
defer qu.Catch()
|
|
defer qu.Catch()
|
|
|
client := &http.Client{
|
|
client := &http.Client{
|
|
@@ -353,6 +354,7 @@ func Download(url string) (string, []byte) {
|
|
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
|
|
},
|
|
},
|
|
|
}
|
|
}
|
|
|
|
|
+ defer client.CloseIdleConnections()
|
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
|
if err != nil {
|
|
if err != nil {
|
|
|
//fmt.Println("Error creating request:", err)
|
|
//fmt.Println("Error creating request:", err)
|
