通用平台爬虫发布work

src/commutil/codesync.go

@@ -0,0 +1,243 @@
+package commutil
+
+import (
+	"archive/zip"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"path/filepath"
+	sp "spiderutil"
+	"strconv"
+	"strings"
+)
+
+// postModule 推送模块
+func postModule(path, module string) error {
+	name := module
+	bs, err := dir2Zip(path)
+	if err != nil {
+		return err
+	}
+	uploadUrl := fmt.Sprintf("%s/push", sp.Config.CodeDbSerever)
+
+	// 创建一个 multipart.FormDataWriter
+	body := bytes.NewBuffer(nil)
+	writer := multipart.NewWriter(body)
+	// 添加文件
+	part, err := writer.CreateFormFile("file", "1.zip")
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(part, bytes.NewReader(bs))
+	if err != nil {
+		return err
+	}
+	// 添加表单数据
+	err = writer.WriteField("name", name)
+	if err != nil {
+		return err
+	}
+	// 完成 multipart 插头
+	err = writer.Close()
+	if err != nil {
+		return err
+	}
+
+	// 创建一个表示 multipart/form-data 的 http.Client
+	client := httpClient()
+	// 构建 POST 请求
+	req, err := http.NewRequest("POST", uploadUrl, nil)
+	if err != nil {
+		return err
+	}
+	// 设置 Content-Type 为 multipart/form-data
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	// 设置请求体
+	req.Body = ioutil.NopCloser(body)
+	// 发送请求
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	// 读取响应
+	_, err = ioutil.ReadAll(resp.Body)
+	return err
+}
+
+// 任务发布
+func publishModule(taskCode, scriptCode, params string, workStart, workEnd, workTimeout, interval int) (err error) {
+	if workStart < 0 || workEnd < 0 || workStart > workEnd || workEnd > 23 {
+		return errors.New("起始或结束时间异常！")
+	} else {
+		req, err := http.NewRequest("GET", fmt.Sprintf("%s/publish", sp.Config.TaskMgrSerever), nil)
+		if err != nil {
+			return errors.New("服务端链接创建失败！")
+		}
+		req.Header.Set("task_code", taskCode)
+		req.Header.Set("script_code", scriptCode)
+		req.Header.Set("work_start", strconv.Itoa(workStart))
+		req.Header.Set("work_end", strconv.Itoa(workEnd))
+		req.Header.Set("worktimeout", strconv.Itoa(workTimeout))
+		req.Header.Set("interval", strconv.Itoa(interval))
+		req.Header.Set("params", params)
+		client := httpClient()
+		resp, err := client.Do(req)
+		if err != nil {
+			return errors.New("服务端发布请求失败！")
+		}
+		defer resp.Body.Close()
+	}
+	return
+}
+
+// unPubModule 撤销任务发布
+func unPubModule(taskCode string) error {
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s/unpublish", sp.Config.CodeDbSerever), nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Add("task_code", taskCode)
+	client := httpClient()
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return nil
+}
+
+// 压缩目录为zip文件
+func dir2Zip(srcPath string) (bs []byte, err error) {
+	buf := bytes.NewBuffer(nil)
+	zipWriter := zip.NewWriter(buf)
+	// 遍历目录并添加文件到 ZIP 归档
+	err = filepath.Walk(srcPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		// 跳过根目录
+		if path == srcPath {
+			return nil
+		}
+		// 计算相对于根目录的路径
+		relPath, err := filepath.Rel(srcPath, path)
+		if err != nil {
+			return err
+		}
+		// 创建 ZIP 文件条目
+		header, err := zip.FileInfoHeader(info)
+		if err != nil {
+			return err
+		}
+		// 设置文件条目的名称
+		header.Name = relPath
+		// 处理目录
+		if info.IsDir() {
+			header.Name += "/"
+		} else {
+			header.Method = zip.Deflate
+		}
+		// 写入 ZIP 文件条目
+		writer, err := zipWriter.CreateHeader(header)
+		if err != nil {
+			return err
+		}
+		// 如果是文件，写入文件内容
+		if !info.IsDir() {
+			file, err := os.Open(path)
+			if err != nil {
+				return err
+			}
+			defer file.Close()
+			_, err = io.Copy(writer, file)
+			if err != nil {
+				return err
+			}
+			zipWriter.Flush()
+		}
+		return nil
+	})
+	zipWriter.Close()
+	bs = buf.Bytes()
+	return
+}
+
+// pullModule 拉取模块
+func pullModule(path, module string) error {
+	pullUrl := fmt.Sprintf("%s/pull", sp.Config.CodeDbSerever)
+	client := httpClient()
+	req, err := http.NewRequest("POST", pullUrl, strings.NewReader(fmt.Sprintf("%s=%s", "name", module)))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	version := resp.Header.Get("code-version")
+	fmt.Println("version", version)
+	bs, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	//保存临时文件
+	// os.WriteFile(fmt.Sprintf("%s_%s_%04d.zip", path,
+	// 	time.Now().Format("20060102150405"),
+	// 	rand.Intn(10000)), bs, 0777)
+	//zip解压
+	return unzip(bs, path)
+}
+
+// 解压缩zip文件到指定目录
+func unzip(zipData []byte, destDir string) error {
+	// 打开 zip 文件
+	r, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData)))
+	if err != nil {
+		return errors.New("open zipfile " + err.Error())
+	}
+
+	// 确保目标目录存在
+	err = os.MkdirAll(destDir, 0755)
+	if err != nil {
+		return errors.New("open zipfile " + err.Error())
+	}
+
+	// 遍历 zip 文件中的所有条目
+	for _, f := range r.File {
+		// 打开文件
+		func() error {
+			rc, err := f.Open()
+			if err != nil {
+				return err
+			}
+			defer rc.Close()
+			// 处理文件
+			path := f.Name
+			if path == "/" {
+				return errors.New("skip root dir") // 跳过根目录
+			}
+
+			// 创建目标文件
+			w, err := os.Create(destDir + "/" + path)
+			if err != nil {
+				return err
+			}
+			defer w.Close()
+			// 复制文件内容
+			_, err = io.Copy(w, rc)
+			return err
+		}()
+
+	}
+	return nil
+}

src/util/commcodecheck.go → src/commutil/commcodecheck.go

@@ -1,4 +1,4 @@
-package util
+package commutil
 
 import (
 	"encoding/json"
@@ -10,10 +10,11 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"util"
 )
 
 var (
-	CommCodesCache = make(chan map[string]string, 5000)
+	CommCodesCache = make(chan map[string]interface{}, 5000)
 )
 
 const (
@@ -71,16 +72,16 @@ func CommCodeCheck() {
 	wg := &sync.WaitGroup{}
 	ch := make(chan bool, 5)
 	for {
-		code := <-CommCodesCache //取数据
+		r := <-CommCodesCache //取数据
 		ch <- true
 		wg.Add(1)
-		go func(tmp map[string]string) {
+		go func(tmp map[string]interface{}) {
 			defer func() {
 				<-ch
 				wg.Done()
 			}()
-			href := tmp["href"]
-			stype := tmp["stype"]
+			href := qu.ObjToString(tmp["href"])
+			stype := qu.ObjToString(tmp["stype"])
 			r, err := OmnipotentCodeCheck(href, stype)
 			omnipotent, _ := r["omnipotent"].(bool)
 			if err == nil {
@@ -89,7 +90,7 @@ func CommCodeCheck() {
 				r["channel"] = tmp["channel"]
 				r["comeintime"] = time.Now().Unix()
 				r["href"] = href
-				MgoEB.Save("luaconfig_omnipotent_log", r)
+				util.MgoEB.Save("luaconfig_omnipotent_log", r)
 				set := map[string]interface{}{}
 				if omnipotent {
 					set["cssmark"] = r["cssmark"]
@@ -109,7 +110,7 @@ func CommCodeCheck() {
 					set["incrementevent"] = 7200
 				}
 				if len(set) > 0 {
-					MgoEB.Update("luaconfig",
+					util.MgoEB.Update("luaconfig",
 						map[string]interface{}{
 							"code":      tmp["code"],
 							"claimtype": 0,
@@ -120,16 +121,25 @@ func CommCodeCheck() {
 						false, false)
 					if omnipotent {
 						time.Sleep(5 * time.Second)
-						one, _ := MgoEB.FindOne("luaconfig", map[string]interface{}{"code": tmp["code"]})
-						if len(*one) > 0 {
-							MgoEB.SaveByOriID("luaconfig_online", *one)
+						one, _ := util.MgoEB.FindOneByField("luaconfig",
+							map[string]interface{}{"code": tmp["code"]},
+							map[string]interface{}{"str_list": 0, "str_content": 0},
+						)
+						if len(*one) > 0 { //保存线上表
+							util.MgoEB.SaveByOriID("luaconfig_online", *one)
+							err = CommCodeMgr(*one)
+							if err != nil {
+								qu.Debug("爬虫发布失败：", err.Error())
+							}
+						} else {
+							qu.Debug("爬虫查询异常，发布失败：", tmp["code"])
 						}
 					}
 				}
 			} else {
 				qu.Debug("万能爬虫验证异常：", err)
 			}
-		}(code)
+		}(r)
 	}
 	wg.Wait()
 }

src/commutil/httpclient.go

@@ -0,0 +1,71 @@
+package commutil
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"embed"
+	"errors"
+	"github.com/go-xweb/log"
+	"net/http"
+)
+
+var (
+	//go:embed keys
+	keys embed.FS
+	//
+	tlsConfig *tls.Config
+	//是否使用HTTPS
+	UseHttps = true
+)
+
+// inic
+func init() {
+	//tls
+	clientCrt, err := keys.ReadFile("keys/client.crt")
+	clientKey, err := keys.ReadFile("keys/client.key")
+	// 加载客户端证书和密钥
+	clientCertPair, err := tls.X509KeyPair(clientCrt, clientKey)
+	if err != nil {
+		log.Fatalf("加载客户端证书和密钥失败: %v", err)
+	}
+	// 加载CA证书
+	caCertPool, err := loadCA()
+	if err != nil {
+		log.Fatalf("加载CA证书失败: %v", err)
+	}
+	// 配置TLS配置
+	tlsConfig = &tls.Config{
+		Certificates: []tls.Certificate{clientCertPair},
+		RootCAs:      caCertPool,
+	}
+}
+
+// 加载CA证书
+func loadCA() (*x509.CertPool, error) {
+	caCert, err := keys.ReadFile("keys/ca.crt")
+	if err != nil {
+		return nil, err
+	}
+
+	caCertPool := x509.NewCertPool()
+	if !caCertPool.AppendCertsFromPEM(caCert) {
+		return nil, errors.New("无法添加CA证书到证书池")
+	}
+	return caCertPool, nil
+}
+
+// httpClient
+func httpClient() *http.Client {
+	// 创建HTTP客户端
+	var transport *http.Transport
+	if UseHttps {
+		transport = &http.Transport{
+			TLSClientConfig: tlsConfig,
+		}
+	} else {
+		transport = &http.Transport{}
+	}
+	return &http.Client{
+		Transport: transport,
+	}
+}

src/commutil/keys/ca.crt

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFdTCCA10CFHnlEWRdBDVrhF4ldCFTpdg4gPchMA0GCSqGSIb3DQEBCwUAMHcx
+CzAJBgNVBAYTAkNOMQ4wDAYDVQQIDAVIZU5hbjESMBAGA1UEBwwJWmhlbmdaaG91
+MQ8wDQYDVQQKDAZqaWFueXUxDDAKBgNVBAsMA2RldjElMCMGA1UEAwwcanNzcGlk
+ZXIuanlkZXYuamlhbnl1MzYwLmNvbTAeFw0yNTA0MDcwODM4NDVaFw0zNTA0MDUw
+ODM4NDVaMHcxCzAJBgNVBAYTAkNOMQ4wDAYDVQQIDAVIZU5hbjESMBAGA1UEBwwJ
+WmhlbmdaaG91MQ8wDQYDVQQKDAZqaWFueXUxDDAKBgNVBAsMA2RldjElMCMGA1UE
+AwwcanNzcGlkZXIuanlkZXYuamlhbnl1MzYwLmNvbTCCAiIwDQYJKoZIhvcNAQEB
+BQADggIPADCCAgoCggIBAKvBuUsG3Fxqwk285ijOwGnFXYZkI8XPJdTqdVNcpZIC
+5S9FjMZCOqP3Bk87EXyB7HcIkRdFLExhx9cji6dm4AIFXSvb/TNMJHnTOQAPbN0O
+Wwt36gH9UMfpbhUKF36CZesq4zAOpCbyaH/7/7Kpe40UGqOaWCn5PcFX/50Q9qN3
+CJzJw0CNWN5OFE0GmC/OLDe6/uQhgfLCel5NcsL/8Zk7WeU2HIWpkGCVAH+oW/HW
+GXQmE5CRB1ZlLSMs7+muBSIC6uLOmg0g3hAurirzkRks2OTYMj4mfvm/IGg4HCaW
+OJXg4N7yDH7G7CwX0Sc8Wmcwiv/anjmamAiDdSIjxWQbppxSaGwj73jTRMy1Ouby
+VBby+fcTdU3XRRl/JSPphr86AyFvJV3KDAUIvAvgnm4mhClRBvLaOplb0jZSoRJO
+9bj0T1Y1YIu4fkLNY1rIaZ2bHytQvIy/KhpiMeXpWK2nUnu3Rf2vaRXpJQNCKIzP
+aXudRnWZwsu/XkuKqdB71JHV7AcTXEJqVwSMlUzVRZABgt2uKjE/wkTLptMR286U
+oqMKttKGxYehOu2rg5xq/0e3ZuMuhwuuUpWq27lBYqvZ46N8R5fLp59rqJesswjg
+IgPXOJKGuBofuUrZIX0TsYLrl6Ph6doBe9m9TOqK8IxyyfRBvkMv1vxTiO/OxvLn
+AgMBAAEwDQYJKoZIhvcNAQELBQADggIBAI2VEUnnAEJKoCC+BW1MLIEbC6DBdHY8
+ms6dB0RU93hne5SS1cirl0LX5JYsRcyWLFGOc/mwecgPwg28FG7m2SOZD3HZYTT9
+fGXamYaiTbmaLJWu7p1XPGgAsAfkHkEcxDi6KpqaAQE4mgUlEX4mtf9Hx83zA+2v
+1JK5PciUNbKc3/Kn+JRfyIXKk36DHrmnWQiqJdZ2eqtlnzJhM6ZuYGAJvFfqkA0r
+psRpxDfmua6QcIoLzOxvrfwbvCF6qKJjC/QcCB5PmTFCo3foUFcNmXe7Gy/Ix+YY
+ApM8XbXZgsjkgKx2OGbQXBJZbYz0ox8hf2Kw8pT13wV4cLNEZCw0ZvInWLKPFl/9
+Ag+041vIYbso6PJZu5vOOfb/8SlEmuev4YgBL16deRcO7OPVIhM/yEERcxIdrHMX
+V8fIwCsPYVfGYTdH433f17F4bkA+/pccg60IkAlJwdOJhOt6EXT1wVVvrWf6N18b
+SOmimIJfk1Ia+0gAgp7owxEAvk8skk9xgATK0XkUGl6fkicUlAnTYqWhUXXC91X2
+pXOhPb75IHQoqwV50Gv09+TxdmXYmGrwdMPCf7kT+SYUjb8HKVHk35NboA/hERdK
+YOh4U5fqaNRh5XfnY1cKzl+d9EGbMhv+YXElIPclw7JM+YC7wtPZ+Eev6HnOrnQh
+hf3yPe7XQe0U
+-----END CERTIFICATE-----

src/commutil/keys/client.crt

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFdTCCA10CFBnU5S43k57625IAhGd+Lvdn1w5xMA0GCSqGSIb3DQEBCwUAMHcx
+CzAJBgNVBAYTAkNOMQ4wDAYDVQQIDAVIZU5hbjESMBAGA1UEBwwJWmhlbmdaaG91
+MQ8wDQYDVQQKDAZqaWFueXUxDDAKBgNVBAsMA2RldjElMCMGA1UEAwwcanNzcGlk
+ZXIuanlkZXYuamlhbnl1MzYwLmNvbTAeFw0yNTA0MDcwODM5MTZaFw0zNTA0MDUw
+ODM5MTZaMHcxCzAJBgNVBAYTAkNOMQ4wDAYDVQQIDAVIZU5hbjESMBAGA1UEBwwJ
+WmhlbmdaaG91MQ8wDQYDVQQKDAZqaWFueXUxDDAKBgNVBAsMA2RldjElMCMGA1UE
+AwwcanNzcGlkZXIuanlkZXYuamlhbnl1MzYwLmNvbTCCAiIwDQYJKoZIhvcNAQEB
+BQADggIPADCCAgoCggIBAK0AY9W2U9h0S382/G+psKTu7olN5rOhNKDS4VAlTGsk
+z+/pXuBHvQ7GAiuDcO3bw+1RMA2x4Lw/oTkThF1FT4KUMU9DPmymlJ/4rRf9XsQ2
+5/ErS4b148F91R/7/LD2pa3vh5kznbfxesZ9azTTuuGxcwe4KIze619eBbgP/bj6
+FdAdKRCz/ExsHJypgqC3Wi/5gWIyFDTlZDFKcyc9xBN0E3aa0pxSMSpwsf0Ea92E
+JCipDiIzd2G0wjfnTCDPO2norCnM6JNhWXaSfKfCrDqLKgJYYNFcrsrJYOLNufvz
+5wl2SoYCj+TlPtXigGPx4we2ZMBsd5SEOhkjYn7tqLsKtVNMQMbCzI43Xrdeu5Gn
+oi/4u8AYDGOn8vw1Ss0Lu++oFx3jzqkGodCj7ojf764erj5S11pKRgVVsRmv955r
+OFPtLOlIQ+209DpG9vNFhJLJ6WoxsmRUxViP/sb4sCSH6JzLt/ZungR/ZQ3Ffr0Z
+FPKkQgh9v7qBh6cFZ9weKse1mYixVFT3xro9p01buv79uE7nbS2Mvl2XYccnJPOi
+XlGE33RHZYuaMB65EWZ+/4OfSFupaZOBTFeTIhHsEKGQF7Lthx/IVAjWZYPj/LsW
+IX+MQMoMDc/6mB5kZWAQy1VSFtDOa5bosyHzGSDiFVhimNYsofaRKeHia8MasDNf
+AgMBAAEwDQYJKoZIhvcNAQELBQADggIBADzGfnjyhAZT0TobX4hJjAJBKmWsRVnP
+sxCUG4oKd/eOAJvjHBEXQj28KaPTcPs39AZ5J7+8ZUb76/6dIsw+AYhtuV0vO+8i
+NqCoE/zOgGfVWVTYbBPzc8UWl/mZzamI1aSmvXw/LoPOc9AkRynEG8Elby0j3AwW
+R8iqQKqE0t/+km1aTBvg+WRKCuAp2WUfIP6xivvY/95f82hLroDS7ULOrHsObMqm
+B+1DtSe8UMfqSDrVRlzCA/CQ2d+pCoMWsv6msoakBO1GE2zl3Wx57xhV/SsZL0IK
+XYV1EAWJviho6rjP+Rd02iX61TUDC/C2h7v//KJCtqT9uBKzAYlNAubI8+TazAFb
+wpK42hJG6WkgtkMgQTuQ422jwNxLgLFgrGqV34KNqrrqkPoO5wZ7dwYsZ2BXXMse
+LlbRvaEvyLoxJpa+GX6qCtHGbdXRpRD5McoYYg1778dBPonyD/9X8icdSE7OgqVE
+xrZ197XdcRreGBzBePqzCnKMer4M38dO/+hia6D6EJd3/ijfZyWU8eEwR5WuVUTf
+spMyILl1pd4gS0bXMJMfeQlcc6XORPrLNWIu04eObZhzdYyZqU3bQkDbh6JeWCE+
+hHytIBBcGIo2gJxwSrdaMjXxDWw781KNgS/IpsX9HxolN4CI+FKtP4BKCuNHrOog
+2RaQyiGc+rzN
+-----END CERTIFICATE-----

src/commutil/keys/client.key

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCtAGPVtlPYdEt/
+NvxvqbCk7u6JTeazoTSg0uFQJUxrJM/v6V7gR70OxgIrg3Dt28PtUTANseC8P6E5
+E4RdRU+ClDFPQz5sppSf+K0X/V7ENufxK0uG9ePBfdUf+/yw9qWt74eZM5238XrG
+fWs007rhsXMHuCiM3utfXgW4D/24+hXQHSkQs/xMbBycqYKgt1ov+YFiMhQ05WQx
+SnMnPcQTdBN2mtKcUjEqcLH9BGvdhCQoqQ4iM3dhtMI350wgzztp6KwpzOiTYVl2
+knynwqw6iyoCWGDRXK7KyWDizbn78+cJdkqGAo/k5T7V4oBj8eMHtmTAbHeUhDoZ
+I2J+7ai7CrVTTEDGwsyON163XruRp6Iv+LvAGAxjp/L8NUrNC7vvqBcd486pBqHQ
+o+6I3++uHq4+UtdaSkYFVbEZr/eeazhT7SzpSEPttPQ6RvbzRYSSyelqMbJkVMVY
+j/7G+LAkh+icy7f2bp4Ef2UNxX69GRTypEIIfb+6gYenBWfcHirHtZmIsVRU98a6
+PadNW7r+/bhO520tjL5dl2HHJyTzol5RhN90R2WLmjAeuRFmfv+Dn0hbqWmTgUxX
+kyIR7BChkBey7YcfyFQI1mWD4/y7FiF/jEDKDA3P+pgeZGVgEMtVUhbQzmuW6LMh
+8xkg4hVYYpjWLKH2kSnh4mvDGrAzXwIDAQABAoICACl6e3VE9sth7qv8of+fvsN8
+qeJvtA2DbBvXE8BkgPL7a4OI/39kk9aT8Pc/13agB1TB73SJHmnkE6KcpFK8iHi1
+UTFA3RmtwtqsA8xJqkv/IhM3ny8ni0jcA5pcj1RafpS6Zld8AaeVlSs77NVIonj0
+U7EZHVfRJrvBzjs4UkvkYhB4vVzM2oP0M9P37GLeMxhniPbjLOcVzLTotb1F2NoV
+LYl1S2PBbRpQ1MvmIHqG8Shyh/n5PvlFHn/Ky/zd0s/swkJvZ5/Wal28d3i9ew+6
+SAMx71yjDSF/sOQQETRwYQdqviltOdTWhf9ABCpyV02FFJjyKKWKImfejQIDtz1j
+VK8SzktfSdLk5ixkMrx8Leuq3fvTzP9GTTjpVvSUSrD5DpvyCuM2GK83znm7VAWL
+cg8AYCTcZdiLEcV62ge1Im4RaRS22LfGq1GlDerCnxdU5ZsBDawCG1h5lzqzWAx7
+q+lCeHZ+/De8UWcZkXuo8yxjcPMdluMcBqX3UnaMZjWYw4cyCazxZoaYlNVfO+13
+OIV9m6nVbj4MM+kfbLO3kBuld70/tb9kyztoLl8tezwgAV2txartD6FigoN6mgU/
+gkzHEQeWojqBFbru9H3Bn11YuiPI6CppRYsdpRhh/p//Mmr6+Wi0HeszqA4dAtNp
+5QqePcBXT9A3ZePuld75AoIBAQDxKKh6LqXkKqiobE+VYS/Bm88WTNhHc9zCHuRX
+Jz4ZvxbuxMem+swbD5/fmP4x1bfv4hsMhXP027oK9c61jgq9iHT9/jejxNsOWh6g
+od5/Qo3ddvboOV1+8iX+3xk6tq3bT+Tt68n8Jf6Tzo7KvnIZZdw4gXMsFf/UD0iM
+ksjKUudeHxaUy+7ad4LoP4d4MLWwSf57FwNYUPb/M8H5k7zgT+QMPipmQ0bfcRId
+QmxVdTZzTwYjMcazBS89QNxoEEbmetN9fiGg5HJGxtX9et2bEP6Ad24xRsJyfbMP
+48HmHW5VW+p1PPhQ/mDCXd0kkBUvM7th1HtazDv8YC8hgnM7AoIBAQC3pfI84HNQ
+sPKhGGkVd+SQTgjDYsoA57jo+yiXOI92bEPRIaOp/QNWatLE3Ek8wqMmYXtDE00U
+eqcI8DvVIKCcoHR/65WKJRB6RO1TJruU+e1EeZ7bpuvC2uTvdezQ+a2dv3ZZXMi7
+N+bCwdvxiVoP4ZJcvdp4hRvxl41xaZCqfUZ2QJe4lP474eED/GyXBuTwmRHXsy+1
+xSzx+7iEfTy/z5XE0pPGEo+h4/KqJjdf5pZNwY35gzSXfo9qCO4B0Xevy38cp5Ig
+LCsoTpHuiE4n8IGyPIlf1kwWNHKswglzHU8Mdi0dj/y4edYyrrDVA9R7tXMbmy17
+Ji4+tLPRIrYtAoIBAQCZHSFrAfcd2N/gn3Qf+eaiXFStafjf1PZS4d5EX4wUOLlM
+u++vWwkAHXl6M98gsPrp+XsHBixLSL7kepDVwOuczujI4wFcn34drKZJgryLvCvq
+cr/tTY7JLOFm4ytXlejol99YGB7UUyNBoSIUI7q52WxrC/WeBhG6CEIdzoxhuSVh
+IwfpTVnHp52b68oNS4/v7+r9J+1dw3ovKiN8XIrNOKwoAchVyV1UPKwtm6DA2XzK
+1V+mNNJdk2sczvOhtsPtTgq9YLnlN2Y+/bucBshwzedolWuHKdAHdTudWRQhEIqe
+8473BND8S6YPgh0IO9rXb1cHlCsFySpE4KTL99FFAoIBAGc6Si1Z5LfInrlapUbB
++6H5mwdosi9MES40bx6BVnK0bHJTPFHCvGuI5CxvvmL+7QIX7VdItBz8zINDtf4Y
+tu8Taii1XXsnVlnEALqqnOoiifHhgx/odcEzlVfcq17T2XlAM+ALPXGu/HEg0uaz
+1AH1Q3YJ6ZGxEIH+NC05r22FuWOi7yV76mPyla7N5qQF5g4vnE3olLrRuG/UlGU0
+mD8hCgh2bUX7DO1s2ew2Sw4G6RmP4KQN0jQNmcN6fX5cJbCU7EBJGOQor6GaeIDh
+8TyV/fEYAgfyxxX8KTFeaD1ijhptGvLxCpcPNQ2MEC3MV1WqKqer1CbS0U/B9wlp
+P4kCggEAC84jD0d/60AU2I//I9rk7c+NxxS3b2I7qpWio5S68Zbw5cvi+zBecTfT
+eadUkHTmYE6ksbTB/dsoL5zwqpB2ZUIPlYk7IPE7L4ZFsMszuEr4PcvE9IopZ/G+
+QVjj5955a4EIFVvJP0nKQc/V0fFp2E3sBCK/a4uDyt0oTqkY3r9ftJsP/yT/tusx
+B/vnsk3Y0oo1A9dcxrlwjAlCCOZGbmkvvInSMxTgMEMcUdbOXy8wxYNngulAORez
+wEo9hUw+ZeKuCKLtQ7CaDBy8Stb/97bJK1oOsxieKVbUGEFc3kCXdfClDeHwTzAO
+4ApoM/HSTDkFfpTSGCZwpB7gjynASw==
+-----END PRIVATE KEY-----

src/config.json

@@ -106,6 +106,8 @@
     "proxyaddr": "http://cc.spdata.jianyu360.com/crawl/proxy/socks5/fetch",
     "proxyauthor": "Basic amlhbnl1MDAxOjEyM3F3ZSFB",
     "optcodeserver": "http://127.0.0.1:7081/check",
+    "codedbserever": "https://jsspider.jydev.jianyu360.com:18080",
+    "taskmgrserever": "https://jsspider.jydev.jianyu360.com:18082",
     "luadisablelib": {
         "baselib": {
             "print": false

src/front/front.go

@@ -2,6 +2,7 @@
 package front
 
 import (
+	"commutil"
 	"fmt"
 	"github.com/go-xweb/httpsession"
 	"github.com/go-xweb/xweb"
@@ -25,23 +26,24 @@ import (
 
 type Front struct {
 	*xweb.Action
-	login        xweb.Mapper `xweb:"/"`
-	logout       xweb.Mapper `xweb:"/center/logout"`               //退出
-	loadIndex    xweb.Mapper `xweb:"/center"`                      //控制中心
-	spidernew    xweb.Mapper `xweb:"/center/spider"`               //爬虫新建
-	reg          xweb.Mapper `xweb:"/center/reg"`                  //爬虫注册
-	assign       xweb.Mapper `xweb:"/center/user/assign"`          //分配爬虫
-	auditExport  xweb.Mapper `xweb:"/center/user/auditexport"`     //导出审核日志
-	loadSpider   xweb.Mapper `xweb:"/center/spider/edit/(.*)"`     //爬虫加载
-	viewSpider   xweb.Mapper `xweb:"/center/spider/view/(.*)"`     //爬虫查看
-	downSpider   xweb.Mapper `xweb:"/center/spider/download/(.*)"` //爬虫下载
-	upState      xweb.Mapper `xweb:"/center/spider/upstate"`       //爬虫状态更新
-	assort       xweb.Mapper `xweb:"/center/spider/assort"`        //审核人员分类(无发布、需登录、无法处理、需删除)
-	batchShelves xweb.Mapper `xweb:"/center/spider/batchShelves"`  //批量上下架
-	checktime    xweb.Mapper `xweb:"/center/spider/checktime"`     //爬虫核对
-	disables     xweb.Mapper `xweb:"/center/spider/disable"`       //批量作废
-	changeEvent  xweb.Mapper `xweb:"/center/changeEvent"`          //节点更新
-	getJson      xweb.Mapper `xweb:"/center/spider/json"`          //
+	login         xweb.Mapper `xweb:"/"`
+	logout        xweb.Mapper `xweb:"/center/logout"`               //退出
+	loadIndex     xweb.Mapper `xweb:"/center"`                      //脚本列表
+	monitorCenter xweb.Mapper `xweb:"/center/luaMonitor"`           //lua脚本管理中心
+	spidernew     xweb.Mapper `xweb:"/center/spider"`               //爬虫新建
+	reg           xweb.Mapper `xweb:"/center/reg"`                  //爬虫注册
+	assign        xweb.Mapper `xweb:"/center/user/assign"`          //分配爬虫
+	auditExport   xweb.Mapper `xweb:"/center/user/auditexport"`     //导出审核日志
+	loadSpider    xweb.Mapper `xweb:"/center/spider/edit/(.*)"`     //爬虫加载
+	viewSpider    xweb.Mapper `xweb:"/center/spider/view/(.*)"`     //爬虫查看
+	downSpider    xweb.Mapper `xweb:"/center/spider/download/(.*)"` //爬虫下载
+	upState       xweb.Mapper `xweb:"/center/spider/upstate"`       //爬虫状态更新
+	assort        xweb.Mapper `xweb:"/center/spider/assort"`        //审核人员分类(无发布、需登录、无法处理、需删除)
+	batchShelves  xweb.Mapper `xweb:"/center/spider/batchShelves"`  //批量上下架
+	checktime     xweb.Mapper `xweb:"/center/spider/checktime"`     //爬虫核对
+	disables      xweb.Mapper `xweb:"/center/spider/disable"`       //批量作废
+	changeEvent   xweb.Mapper `xweb:"/center/changeEvent"`          //节点更新
+	getJson       xweb.Mapper `xweb:"/center/spider/json"`          //
 	//delRedis        xweb.Mapper `xweb:"/center/spider/delRedis"`        //清理Redis
 	updateESP       xweb.Mapper `xweb:"/center/spider/updateesp"`       //修改爬虫的节点/状态/平台
 	updatePendState xweb.Mapper `xweb:"/center/spider/updatePendState"` //更新爬虫挂起状态
@@ -75,7 +77,6 @@ type Front struct {
 	StepRe3   StepRe3
 	U         U
 
-	luaList       xweb.Mapper `xweb:"/center/lualist.html"`       //脚本管理
 	user          xweb.Mapper `xweb:"/center/user.html"`          //用户管理
 	delUser       xweb.Mapper `xweb:"/center/user/del"`           //删除用户
 	updateUser    xweb.Mapper `xweb:"/center/user/updateUser"`    //修改用户信息
@@ -95,6 +96,10 @@ type Front struct {
 	//爬虫认领
 	claimCode  xweb.Mapper `xweb:"/center/claim/claimcode"`  //爬虫认领
 	returnCode xweb.Mapper `xweb:"/center/claim/returncode"` //爬虫归还
+
+	//通用爬虫
+	commCodesCenter xweb.Mapper `xweb:"/center/commMonitor"`               //通用爬虫管理中心
+	updateCodeBase  xweb.Mapper `xweb:"/center/commspider/updatecodebase"` //
 }
 
 const Sp_state_0, Sp_state_1, Sp_state_2, Sp_state_3, Sp_state_4, Sp_state_5, Sp_state_6, Sp_state_7, Sp_state_8, Sp_state_9, Sp_state_10 = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 //0待完成，1待审核，2打回，3发布，4作废，5已上架，6已下架，7无发布，8需登录，9转python，10已删除
@@ -559,7 +564,9 @@ func saveLua(o map[string]interface{}) bool {
 
 	ok := spider.SaveSpider(o["code"].(string), param)
 	if ok { //保存成功，校验新导入的爬虫对应站点是否存在，否则加站点记录
-		site, _ := u.MgoEB.FindOneByField("site", map[string]interface{}{"site": o["name"]}, map[string]interface{}{"important": 1})
+		site, _ := u.MgoEB.FindOneByField("site", map[string]interface{}{"site": o["name"]}, map[string]interface{}{"important": 1, "site_toptype": 1, "site_subtype": 1})
+		var site_toptype string
+		var site_subtype string
 		if len(*site) == 0 {
 			qu.Debug("补充站点信息：", o["name"])
 			domain := u.DomainReg.FindString(qu.ObjToString(AutoTpl["Base.SpiderTargetChannelUrl"]))
@@ -605,21 +612,40 @@ func saveLua(o map[string]interface{}) bool {
 				"type_plate":     "",
 			}
 			u.MgoEB.Save("site", siteInfo)
-		} else if qu.IntAll((*site)["important"]) == 1 { //重点网站
-			u.MgoEB.Update("luaconfig", map[string]interface{}{"code": o["code"]}, map[string]interface{}{"$set": map[string]interface{}{"spiderimportant": true}}, false, false)
-			u.MgoEB.Update("site_code_baseinfo", map[string]interface{}{"spidercode": o["code"]}, map[string]interface{}{"$set": map[string]interface{}{
-				"site":       o["name"],
-				"channel":    o["channel"],
-				"spidercode": o["code"],
-				"platform":   o["platform"],
-				"modifyuser": "",
-				"state":      0,
-			}}, true, false)
+		} else {
+			site_toptype = qu.ObjToString((*site)["site_toptype"])
+			site_subtype = qu.ObjToString((*site)["site_subtype"])
+			if qu.IntAll((*site)["important"]) == 1 { //重点网站
+				u.MgoEB.Update("luaconfig", map[string]interface{}{"code": o["code"]}, map[string]interface{}{"$set": map[string]interface{}{"spiderimportant": true}}, false, false)
+				u.MgoEB.Update("site_code_baseinfo", map[string]interface{}{"spidercode": o["code"]}, map[string]interface{}{"$set": map[string]interface{}{
+					"site":       o["name"],
+					"channel":    o["channel"],
+					"spidercode": o["code"],
+					"platform":   o["platform"],
+					"modifyuser": "",
+					"state":      0,
+				}}, true, false)
+			}
 		}
 		//生成认领日志
 		if len(claimLog) > 0 {
 			u.MgoEB.Save("lua_logs_claim", claimLog)
 		}
+		//爬虫基本属性
+		u.MgoEB.Save("luaconfig_cfg", map[string]interface{}{
+			"site_toptype":   site_toptype,
+			"site_subtype":   site_subtype,
+			"infoformat":     infoformat,
+			"site":           o["name"],
+			"channel":        o["channel"],
+			"code":           o["code"],
+			"platform":       o["platform"],
+			"priority":       o["priority"],
+			"href":           o["channeladdr"],
+			"property_top":   "",
+			"property_sub":   "",
+			"property_third": "",
+		})
 	}
 	return ok
 }
@@ -705,11 +731,11 @@ func (f *Front) ImportAi() {
 							if infoformat == 5 {
 								stype = "news"
 							}
-							u.CommCodesCache <- map[string]string{
+							commutil.CommCodesCache <- map[string]interface{}{
 								"code":    code,
 								"href":    href,
-								"site":    qu.ObjToString(o["name"]),
-								"channel": qu.ObjToString(o["channel"]),
+								"site":    o["name"],
+								"channel": o["channel"],
 								"stype":   stype,
 							}
 						}
@@ -842,7 +868,9 @@ func saveLuaAi(o map[string]interface{}) bool {
 	}
 	ok := spider.SaveSpider(o["code"].(string), param) //爬虫保存
 	if ok {                                            //保存成功，校验新导入的爬虫对应站点是否存在，否则加站点记录
-		site, _ := u.MgoEB.FindOneByField("site", map[string]interface{}{"site": o["name"]}, map[string]interface{}{"important": 1})
+		site, _ := u.MgoEB.FindOneByField("site", map[string]interface{}{"site": o["name"]}, map[string]interface{}{"important": 1, "site_toptype": 1, "site_subtype": 1})
+		var site_toptype string
+		var site_subtype string
 		if len(*site) == 0 {
 			qu.Debug("补充站点信息：", o["name"])
 			domain := u.DomainReg.FindString(qu.ObjToString(AutoTpl["Base.SpiderTargetChannelUrl"]))
@@ -888,21 +916,40 @@ func saveLuaAi(o map[string]interface{}) bool {
 				"type_plate":     "",
 			}
 			u.MgoEB.Save("site", siteInfo)
-		} else if qu.IntAll((*site)["important"]) == 1 { //重点网站
-			u.MgoEB.Update("luaconfig", map[string]interface{}{"code": o["code"]}, map[string]interface{}{"$set": map[string]interface{}{"spiderimportant": true}}, false, false)
-			u.MgoEB.Update("site_code_baseinfo", map[string]interface{}{"spidercode": o["code"]}, map[string]interface{}{"$set": map[string]interface{}{
-				"site":       o["name"],
-				"channel":    o["channel"],
-				"spidercode": o["code"],
-				"platform":   o["platform"],
-				"modifyuser": "",
-				"state":      0,
-			}}, true, false)
+		} else {
+			site_toptype = qu.ObjToString((*site)["site_toptype"])
+			site_subtype = qu.ObjToString((*site)["site_subtype"])
+			if qu.IntAll((*site)["important"]) == 1 { //重点网站
+				u.MgoEB.Update("luaconfig", map[string]interface{}{"code": o["code"]}, map[string]interface{}{"$set": map[string]interface{}{"spiderimportant": true}}, false, false)
+				u.MgoEB.Update("site_code_baseinfo", map[string]interface{}{"spidercode": o["code"]}, map[string]interface{}{"$set": map[string]interface{}{
+					"site":       o["name"],
+					"channel":    o["channel"],
+					"spidercode": o["code"],
+					"platform":   o["platform"],
+					"modifyuser": "",
+					"state":      0,
+				}}, true, false)
+			}
 		}
 		//生成认领日志
 		if len(claimLog) > 0 {
 			u.MgoEB.Save("lua_logs_claim", claimLog)
 		}
+		//爬虫基本属性
+		u.MgoEB.Save("luaconfig_cfg", map[string]interface{}{
+			"site_toptype":   site_toptype,
+			"site_subtype":   site_subtype,
+			"infoformat":     infoformat,
+			"site":           o["name"],
+			"channel":        o["channel"],
+			"code":           o["code"],
+			"platform":       o["platform"],
+			"priority":       o["priority"],
+			"href":           o["channeladdr"],
+			"property_top":   "",
+			"property_sub":   "",
+			"property_third": "",
+		})
 	}
 	return ok
 }

src/front/spider.go

@@ -107,7 +107,7 @@ func (f *Front) LoadSpider(codeTaskIdReState string) error {
 		code := sp.Se.Decode4Hex(code)
 		f.T["actiontext"] = "编辑"
 		//lua, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
-		lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
+		lua, _ := u.MgoEB.FindOneByField("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"cssmark": 0})
 		//爬虫开发人员编辑爬虫，修改爬虫信息
 		if auth == u.Role_Dev && text != "bu" && qu.ObjToString((*lua)["spidertype"]) == "increment" {
 			(*lua)["spidertype"] = "history"
@@ -1893,7 +1893,7 @@ func IsHasUpState(auth, state int) bool {
 var list_fields = `{"_id":1,"code":1,"createuser":1,"modifyuser":1,"modifytime":1,"l_uploadtime":1,"l_checktime":1,"state":1,"param_common":1,"event":1,"urgency":1,"platform":1,"pendstate":1}`
 
 // 脚本管理，结合爬虫运行信息
-func (f *Front) LuaList() {
+func (f *Front) MonitorCenter() {
 	auth := qu.IntAll(f.GetSession("auth"))
 	if auth != u.Role_Admin {
 		f.ServeJson("没有权限！")

src/front/spider2.go

@@ -1,7 +1,11 @@
 package front
 
 import (
+	"commutil"
+	"fmt"
 	qu "qfw/util"
+	"sort"
+	sp "spiderutil"
 	"strings"
 	"time"
 	"util"
@@ -48,3 +52,102 @@ func (f *Front) SpiderRemark() {
 	code := f.GetString("code")
 	util.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"spiderremark": strings.Replace(remark, "采集说明：", "", -1)}}, false, false)
 }
+
+func (f *Front) CommCodesCenter() {
+	auth := qu.IntAll(f.GetSession("auth"))
+	if auth != util.Role_Admin {
+		f.ServeJson("没有权限！")
+		return
+	}
+	if f.Method() == "POST" {
+		start, _ := f.GetInteger("start")
+		limit, _ := f.GetInteger("length")
+		draw, _ := f.GetInteger("draw")
+		searchStr := f.GetString("search[value]")
+		platform := f.GetString("platform")
+		search := strings.TrimSpace(searchStr)
+		query := map[string]interface{}{}
+		//搜索条件
+		if search != "" {
+			query["$or"] = []interface{}{
+				map[string]interface{}{"code": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"site": map[string]interface{}{"$regex": search}},
+			}
+		}
+		if platform != "-1" {
+			query["platform"] = platform
+		}
+		sort := `{"%s":%d}`
+		orderIndex := f.GetString("order[0][column]")
+		orderName := f.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
+		orderType := 1
+		if f.GetString("order[0][dir]") != "asc" {
+			orderType = -1
+		}
+		sort = fmt.Sprintf(sort, orderName, orderType)
+		luaBases, _ := util.MgoEB.Find("luaconfig_cfg", query, sort, nil, false, start, limit)
+		count := util.MgoEB.Count("luaconfig_cfg", query)
+		qu.Debug("query:", query, start, limit, count, len(*luaBases))
+		for _, lb := range *luaBases {
+			l, _ := util.MgoEB.FindOneByField("luaconfig", map[string]interface{}{"code": lb["code"]}, map[string]interface{}{"state": 1, "workmgr": 1})
+			lb["state"] = (*l)["state"]
+			lb["workmgr"] = (*l)["workmgr"]
+			//var wkText string
+			//if workmgr, ok := (*l)["workmgr"].(map[string]interface{}); ok {
+			//	wkByte, _ := json.Marshal(workmgr)
+			//	wkText = string(wkByte)
+			//}
+			//lb["workmgr"] = wkText
+		}
+		f.ServeJson(map[string]interface{}{"draw": draw, "data": luaBases, "recordsFiltered": count, "recordsTotal": count})
+	} else {
+		events := []string{}
+		for k, _ := range sp.Config.Uploadevents {
+			events = append(events, k)
+		}
+		sort.Strings(events)
+		f.T["events"] = events
+		f.Render("commcodes.html", &f.T)
+	}
+}
+
+func (f *Front) UpdateCodeBase() {
+	itype, _ := f.GetInteger("itype")
+	code := f.GetString("code")
+	work_start, _ := f.GetInteger("work_start")
+	work_end, _ := f.GetInteger("work_end")
+	work_timeout, _ := f.GetInteger("work_timeout")
+	interval, _ := f.GetInteger("interval")
+	params := f.GetString("params")
+	qu.Debug(code, itype, work_start, work_end, work_timeout, interval, params)
+	var msg string
+	set := map[string]interface{}{
+		"workmgr": map[string]interface{}{
+			"script_code": code,
+			"task_code":   code,
+			"work_start":  work_start,
+			"work_end":    work_end,
+			"worktimeout": work_timeout,
+			"interval":    interval,
+			"params":      params,
+		},
+	}
+	util.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": set}, false, false)
+	util.MgoEB.Update("luaconfig_online", map[string]interface{}{"code": code}, map[string]interface{}{"$set": set}, false, false)
+	if itype == 1 { //发布
+		one, _ := util.MgoEB.FindOneByField("luaconfig",
+			map[string]interface{}{"code": code},
+			map[string]interface{}{"str_list": 0, "str_content": 0},
+		)
+		if len(*one) > 0 { //保存线上表
+			err := commutil.CommCodeMgr(*one)
+			if err != nil {
+				msg = err.Error()
+			}
+		} else {
+			msg = "爬虫查询异常，发布失败!"
+			qu.Debug("爬虫查询异常，发布失败：", code)
+		}
+	}
+	f.ServeJson(map[string]interface{}{"msg": msg})
+}

src/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"commutil"
 	_ "filter"
 	"front"
 	"luacheck"
@@ -101,7 +102,7 @@ func main() {
 	//爬虫质检
 	go luacheck.LuaCheckStart()
 	//万能爬虫校验
-	go u.CommCodeCheck()
+	go commutil.CommCodeCheck()
 	//提供接口，接收其他数据
 	http.HandleFunc("/spider/infos", func(w http.ResponseWriter, req *http.Request) {
 		data := req.FormValue("data")

src/web/templates/commcodes.html

@@ -0,0 +1,300 @@
+{{include "head.html"}}
+<div class="modal fade" id="myModal-codebase" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
+	<div class="modal-dialog">
+		<div class="modal-content">
+			<div class="modal-header">
+				<div class="modal-header">
+					<button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
+					<div class="edit-form">
+						<div class="edit-info">
+							<span class="glyphicon glyphicon-briefcase code-assign" aria-hidden="true"></span>
+							<span class="info">编辑属性<span>
+						</div>
+						<form class="form-horizontal" role="form">
+							<div class="form-group">
+								<label for="work_start" class="col-sm-2 control-label">起始时间:</label>
+								<div class="col-sm-10">
+									<input type="text" class="form-control" id="work_start" placeholder="请输入起始时间">
+								</div>
+							</div>
+							<div class="form-group">
+								<label for="work_end" class="col-sm-2 control-label">结束时间:</label>
+								<div class="col-sm-10">
+									<input type="text" class="form-control" id="work_end" placeholder="请输入结束时间">
+								</div>
+							</div>
+							<div class="form-group">
+								<label for="work_timeout" class="col-sm-2 control-label">超时时间:</label>
+								<div class="col-sm-10">
+									<input type="text" class="form-control" id="work_timeout" placeholder="请输入超时时间（单位:分钟）">
+								</div>
+							</div>
+							<div class="form-group">
+								<label for="interval" class="col-sm-2 control-label">间隔周期:</label>
+								<div class="col-sm-10">
+									<input type="text" class="form-control" id="interval" placeholder="请输入间隔周期（单位:分钟）">
+								</div>
+							</div>
+							<div class="form-group">
+								<label for="params" class="col-sm-2 control-label">其它参数:</label>
+								<div class="col-sm-10">
+									<textarea class="form-control" id="params" placeholder="请输入其它参数"></textarea>
+								</div>
+							</div>
+							<div class="form-group">
+								<div class="col-sm-offset-2 col-sm-10 operateStyle">
+									<input type="button" onclick="publishWorker(0)" class="btn btn-primary" value="保存">
+									<input type="button" onclick="publishWorker(1)" class="btn btn-success" value="发布">
+									<input type="button" data-dismiss="modal" class="btn btn-default" value="取消">
+								</div>
+							</div>
+						</form>
+					</div>
+				</div>
+			</div>
+		</div><!-- /.modal-content -->
+	</div><!-- /.modal -->
+</div>
+<div class="content-wrapper">
+	<section class="content-header">
+		<h1>
+			爬虫属性
+			<small>
+			</small>
+		</h1>
+		<ol class="breadcrumb">
+			<li><a href="#"><i class="fa fa-dashboard"></i> 首页</a></li>
+			<li class="active">列表</li>
+		</ol>
+	</section>
+	<section class="content">
+		<div class="box">
+			<!-- /.box-header -->
+			<div class="box-body">
+				<table id="spider" class="table table-bordered table-striped">
+					<thead>
+					<tr>
+						<th>网站</th>
+						<th>栏目</th>
+						<th>爬虫</th>
+						<th>类型</th>
+						<th>平台</th>
+						<th>状态</th>
+						<th>评分</th>
+						<th>站点一级</th>
+						<th>站点二级</th>
+						<th>属性一级</th>
+						<th>属性二级</th>
+						<th>属性三级</th>
+						<th class="hidden-xs">操作</th>
+					</tr>
+					</thead>
+				</table>
+			</div>
+			<!-- /.box-body -->
+		</div>
+	</section>
+
+	<script>
+		var spidercode ;
+		$(function(){
+			ttable=$('#spider').DataTable({
+				"language": {
+					"url": "/js/dataTables.chinese.lang"
+				},
+				"columnDefs": [
+					{ "orderable": false, "targets": [0,1,2,3,4,5,7,8,9,10,11] },
+				],
+				"order": [[6,"desc"]],
+				"processing":true,
+				"serverSide": true,
+				"searching": true,
+				"ordering": true,
+				"info": true,
+				"autoWidth": true,
+				"ajax": {
+					"url": "/center/commMonitor",
+					"type": "POST"
+				},
+				"columns": [
+					{ "data": "site"},
+					{ "data": "channel",render:function (val,a,row) {
+							var href ="javascript:void(0)" ;
+							if(row["href"] != ""){
+								href = row["href"];
+							}
+							var vals="<a href='"+href+"' title='"+val+"' target='_blank'>"+val+"</a>"
+							return vals
+						}},
+					{ "data": "code"},
+					{ "data": "infoformat",render:function (val, a, row){
+							if(val == 1){
+								return "招标";
+							}else if (val == 2){
+								return "拟建/审批";
+							}else if (val == 3){
+								return "产权";
+							}else if (val == 4){
+								return "舆情";
+							}else if (val == 5){
+								return "新闻";
+							}
+						}},
+					{ "data": "platform"},
+					{ "data": "state",render:function (val,a,row){
+							if(val==0){
+								val = "待完成";
+							}else if(val==1){
+								val = "待审核";
+							}else if(val==2){
+								val = "未通过";
+							}else if(val==3){
+								val = "已通过";
+							}else if(val==4){
+								val = "已作废";
+							}else if(val==5){
+								val = "已上架";
+							}else if(val==6){
+								val = "已下架";
+							}else if(val==7){
+								val = "无发布";
+							}else if(val==8){
+								val = "需登录";
+							}else if(val==9){
+								val = "转python";
+							}else if(val==10){
+								val = "已删除";
+							}else if(val==11){
+								val = "已上线";
+							}else if(val==12){
+								val = "无法标注";
+							}
+							return val;
+						}},
+					{ "data": "priority"},
+					{ "data": "site_toptype"},
+					{ "data": "site_subtype"},
+					{ "data": "property_top"},
+					{ "data": "property_sub"},
+					{ "data": "property_third"},
+					{ "data": "_id",render:function(val,a,row){
+							var dbutton=$(`<button class="btn btn-sm btn-success" disabled>更新发布</button>`);
+							if (row.state == 11 && row.platform == "通用平台"){
+								var workmgrText = JSON.stringify(row.workmgr);
+								dbutton=$(`<button class="btn btn-sm btn-success" data-workmgr='${workmgrText}' onclick="publish('${row.code}', this)">更新发布</button>`);
+							}
+							var div=$("<div><div class=\"btn-group\"></div></div>")
+							div.find(".btn-group").append(dbutton);
+							return div.html()
+						}}
+				],
+				"fnDrawCallback": function(table) {
+
+				},
+				"fnServerParams": function (e) {
+					var platform = $("#platform").val();
+					//平台
+					if(platform){
+						e.platform = platform;
+					}else{
+						e.platform="-1";
+					}
+				}
+			});
+			common.menuActive("commMonitor");
+			ttable.on( 'init.dt', function () {
+				//平台
+				var platform="<option value='-1'>全部</option>"
+				platform+="<option value='golua平台'>golua平台</option>";
+				platform+="<option value='chrome'>chrome</option>";
+				platform+="<option value='python'>python</option>";
+				platform+="<option value='通用平台'>通用平台</option>";
+				platform+="<option value='jschrome'>jschrome</option>";
+				var selectPlatform="<div class='form-group'><label for='name'>平台：</label>"+
+						"<select id='platform' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+						platform+
+						"</select></div>"
+				$("#spider_filter").prepend("&nbsp;&nbsp;");
+				$("#spider_filter").prepend(selectPlatform);
+				$("#spider_wrapper .col-sm-6").css({width:"100%"})//样式
+			});
+		})
+		//发布
+		function publish(code,row) {
+			spidercode = code;
+			var workmgrMap = JSON.parse($(row).attr('data-workmgr'));
+			if (workmgrMap != null){
+				$("#work_start").val(workmgrMap.work_start)
+				$("#work_end").val(workmgrMap.work_end)
+				$("#work_timeout").val(workmgrMap.worktimeout)
+				$("#interval").val(workmgrMap.interval)
+				$("#params").val(workmgrMap.params)
+			}
+			$("#myModal-codebase").modal("show");
+		}
+
+		function publishWorker(itype) {
+			var work_start =  parseInt($("#work_start").val());
+			var work_end =  parseInt($("#work_end").val());
+			var work_timeout =  parseInt($("#work_timeout").val());
+			var interval =  parseInt($("#interval").val());
+			var params =  $("#params").val();
+
+			if (isNaN(work_start) || isNaN(work_end) || work_start < 0 || work_end < 0 || work_start > work_end || work_end > 23) {
+				alert("起始或结束时间异常！");
+				return
+			}else if (isNaN(work_timeout) || work_timeout < 0){
+				alert("超时时间异常！");
+				return
+			}else if (isNaN(interval)){
+				alert("间隔周期异常！");
+				return
+			}else if (params != "" && !isNonEmptyJSON(params)){
+				alert("其它参数格式异常！")
+				return
+			}
+			$.ajax({
+				url:"/center/commspider/updatecodebase",
+				type:"post",
+				data:{"itype":itype,"code":spidercode,"work_start":work_start,"work_end":work_end,"work_timeout":work_timeout,"interval":interval,"params":params},
+				success:function (r){
+					var msg = "保存";
+					if (itype == 1){
+						msg = "发布";
+					}
+					if(r&&r.msg == ""){
+						showTip(msg+"成功", 1000);
+						$("#myModal-codebase").modal("hide");
+						ttable.draw(false)//只刷新信息，不重置页面（返回第一页）
+					}else{
+						showTip(msg+"失败"+r.msg, 1000);
+					}
+				}
+			})
+		}
+
+
+
+		function checkclick(){
+			ttable.ajax.reload();
+		}
+
+		function isNonEmptyJSON(text) {
+			try {
+				// 尝试解析文本
+				const parsed = JSON.parse(text);
+				// 检查是否为空对象或空数组
+				if (Object.keys(parsed).length === 0 || Array.isArray(parsed)) {
+					// 是空对象
+					return false;
+				}
+				// 如果不是空对象或空数组，返回 true
+				return true;
+			} catch (e) {
+				// 如果解析失败，捕获错误并返回 false
+				return false;
+			}
+		}
+	</script>
+</div>
+{{include "bottom.html"}}

src/web/templates/head.html

@@ -200,10 +200,10 @@
 	</script>
   <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->
   <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
-  <!--[if lt IE 9]>
-  <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
-  <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-  <![endif]-->
+  <!--[if lt IE 9]>-->
+	<!--<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>-->
+    <!--<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>-->
+      <!--<![endif]-->
 </head>
 <body class="sidebar-collapse skin-blue sidebar-mini" style="overflow-hidden;">
 <!-- 新建任务模态框（Modal） -->
@@ -627,11 +627,26 @@
 		  </li>
 		{{else if eq (session "auth") 3}}
 		  	<li data="index"><a href="/center"><i class="glyphicon glyphicon-eye-open"></i> <span>脚本列表</span></a></li>
-		  	<li data="index"><a href="/center/managerTask"><i class="glyphicon glyphicon-tasks"></i> <span>日常任务</span></a></li>
-		  	{{if eq (session "identity") 1}}
-		  	<li data="index"><a href="/center/sitetask"><i class="glyphicon glyphicon-align-justify"></i> <span>重点任务</span></a></li>
-		  	{{end}}
-			<li data="index_fbgl"><a href="/center/lualist.html"><i class="glyphicon glyphicon-credit-card"></i> <span>运行监控中心</span></a></li>
+			<li class="treeview" data="index_rwgl">
+			  <a href="#"><i class="glyphicon glyphicon-th-list"></i> <span>任务管理</span>
+				  <span class="pull-right-container"><i class="fa fa-angle-left pull-right"></i></span>
+			  </a>
+			  <ul class="treeview-menu">
+				  <li><a href="/center/managerTask"><i class="glyphicon glyphicon-tasks"></i>日常任务</a></li>
+				  {{if eq (session "identity") 1}}
+				  <li><a href="/center/sitetask"><i class="glyphicon glyphicon-align-justify"></i>重点任务</a></li>
+				  {{end}}
+			  </ul>
+			</li>
+			<li class="treeview" data="index_jkzx">
+			  <a href="#"><i class="glyphicon glyphicon-facetime-video"></i> <span>管理中心</span>
+				  <span class="pull-right-container"><i class="fa fa-angle-left pull-right"></i></span>
+			  </a>
+			  <ul class="treeview-menu">
+				  <li><a href="/center/luaMonitor"><i class="glyphicon glyphicon-font"></i>Lua爬虫管理</a></li>
+				  <li><a href="/center/commMonitor"><i class="glyphicon glyphicon-bold"></i>通用爬虫管理</a></li>
+			  </ul>
+			</li>
 			<li data="index_pcdr"><a href="/center/importdata"><i class="glyphicon glyphicon-import"></i> <span>爬虫导入</span></a></li>
 			<li data="index_pcqy"><a href="/center/luamove"><i class="glyphicon glyphicon-move"></i> <span>爬虫迁移管理</span></a></li>
 			{{if eq (session "identity") 1}}
@@ -640,7 +655,6 @@
 					  <span class="pull-right-container"><i class="fa fa-angle-left pull-right"></i></span>
 				  </a>
 				  <ul class="treeview-menu">
-
 					  <li><a href="/center/supplement/daylist"><i class="fa fa-circle-o"></i>日补采</a></li>
 					  <li><a href="/center/supplement/weeklist"><i class="fa fa-circle-o"></i>周补采</a></li>
 				  </ul>

src/web/templates/heart.html

@@ -14,13 +14,13 @@
  </section>
  <section class="content">
 	<div class="box">
-            <!-- /.box-header -->
+           <!-- /.box-header -->
             <div class="box-body">
               <table id="spider" class="table table-bordered table-striped">
                 <thead>
                 <tr>
-                 <th>站点</th>
-                 <th>栏目</th>
+                  <th>站点</th>
+                  <th>栏目</th>
                   <th>爬虫</th>
                   <th>节点</th>
           		  <th>维护人</th>
@@ -47,7 +47,7 @@
               "url": "/js/dataTables.chinese.lang"
           },
 		"columnDefs": [
-		  { "orderable": false, "targets": [0,1,2,3,4] } //设置列不可排序
+		  { "orderable": false, "targets": [0,1,2,3,4]} //设置列不可排序
 		],
         "order": [[6,"asc"]], //默认排序列
 		"processing":true,
@@ -64,46 +64,46 @@
 			"type": "POST"
 		 },
 		"columns": [
-         { "data": "param_common",render:function(val,a,row){
-            vals=val[1];
-            if(vals.length>15){
-                vals=vals.substr(0,15)+"...";
-            }
-            return  vals
-        }},
-        { "data": "param_common",render:function(val,a,row){
-            if(val.length<3){
-                val="暂无";
-            }else{
-                var href="#"
-                if(val[11]){
-                    href=val[11]
+            { "data": "param_common",render:function(val,a,row){
+                vals=val[1];
+                if(vals.length>15){
+                    vals=vals.substr(0,15)+"...";
+                }
+                return  vals
+            }},
+            { "data": "param_common",render:function(val,a,row){
+                if(val.length<3){
+                    val="暂无";
+                }else{
+                    var href="#"
+                    if(val[11]){
+                        href=val[11]
+                    }
+                    val="<a href='"+href+"' title='"+val[2]+"' target='_blank'>"+val[2]+"</a>"
                 }
-                val="<a href='"+href+"' title='"+val[2]+"' target='_blank'>"+val[2]+"</a>"
+                return val
+            }},
+            { "data": "code"},
+            { "data": "event"},
+            { "data": "modifyuser"},
+            { "data": "firstpage"},
+            { "data": "list"},
+            { "data": "detail"},
+            { "data": "findlist"},
+            { "data": "detailexecute",width:"72px"},
+            { "data": "pendtime"},
+            { "data": "updatetime"}
+        ],
+        "fnServerParams": function (e) {
+            var event = $("#heart_event").val();
+            //节点
+            if(event){
+                e.event = event;
+            }else{
+                e.event="-1";
             }
-            return val
-        }},
-        { "data": "code"},
-        { "data": "event"},
-		{ "data": "modifyuser"},
-        { "data": "firstpage"},
-        { "data": "list"},
-        { "data": "detail"},
-        { "data": "findlist"},
-        { "data": "detailexecute",width:"72px"},
-        { "data": "pendtime"},
-        { "data": "updatetime"}
-    ],
-    "fnServerParams": function (e) {  
-				var event = $("#heart_event").val();
-				//节点
-				if(event){
-					e.event = event;
-				}else{
-					e.event="-1";
-				}
-      }
-	});
+          }
+        });
 		common.menuActive("heart");
 		heartable.on( 'init.dt', function () {
 			var opt="<option value='-1'>全部</option>"

src/web/templates/lualist.html

@@ -58,7 +58,7 @@
                 "info": true,
                 "autoWidth": true,
                 "ajax": {
-                    "url": "/center/lualist.html",
+                    "url": "/center/luaMonitor",
                     "type": "POST"
                 },
                 "columns": [
@@ -223,7 +223,7 @@
                     }
                 }
             });
-            common.setActive("index_fbgl");
+            common.menuActive("luaMonitor");
             ttable.on( 'init.dt', function () {
                 var search=getCookie("input");
                 if(search){

src/web/templates/spideredit.html

@@ -1,8 +1,8 @@
 <head>
 	<meta charset="utf-8"> 
-	<link rel="stylesheet" href="https://cdn.staticfile.org/twitter-bootstrap/3.3.7/css/bootstrap.min.css">  
-	<script src="https://cdn.staticfile.org/jquery/2.1.1/jquery.min.js"></script>
-	<script src="https://cdn.staticfile.org/twitter-bootstrap/3.3.7/js/bootstrap.min.js"></script>
+<!--	<link rel="stylesheet" href="https://cdn.staticfile.org/twitter-bootstrap/3.3.7/css/bootstrap.min.css">-->
+<!--	<script src="https://cdn.staticfile.org/jquery/2.1.1/jquery.min.js"></script>-->
+<!--	<script src="https://cdn.staticfile.org/twitter-bootstrap/3.3.7/js/bootstrap.min.js"></script>-->
 </head>
 {{include "head.html"}}