jy-data-extract/src/jy/cluster/aliecs.go

/**
阿里云ecs实例自动申请、部署、释放
**/
package cluster

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	db "jy/mongodbutil"
	ju "jy/util"
	"log"
	"net/http"
	"net/url"
	qu "qfw/util"
	"regexp"
	"sort"
	"strings"
	"time"
)

const (
	Password = "Jy_ExtractBid_2019"
	URL      = "https://ecs.aliyuncs.com"
)

//批量创建实例
func RunInstances(taskName, computer, flow string, num, hours int) {
	if esconfig, ok := ju.Config["esconfig"].(map[string]interface{}); ok {
		widthOut := "0"
		if flow == "true" {
			widthOut = "10"
		}
		log.Println(esconfig["LaunchTemplateId"+computer], widthOut)
		if b, ok := esconfig["available"].(bool); ok && b {
			if zoneIds, _ := esconfig["ZoneIds"].([]interface{}); ok {
				pernum := num / len(zoneIds)
				if pernum < 1 {
					kv, _ := zoneIds[len(zoneIds)-1].(map[string]interface{})
					runInstances(kv, taskName, widthOut, computer, num, hours)
				} else {
					for k, v := range zoneIds {
						if (k == len(zoneIds)-1) && (num%len(zoneIds) != 0) {
							pernum = num - pernum*(len(zoneIds)-1)
						}
						kv, _ := v.(map[string]interface{})
						runInstances(kv, taskName, widthOut, computer, pernum, hours)
					}
				}
			}
		}
	}
}

func runInstances(kv map[string]interface{}, taskName, widthOut, computer string, pernum, hours int) {
	log.Println(kv, taskName, widthOut, computer, pernum, hours)
	res := GET("RunInstances", [][]string{
		[]string{"RegionId", "cn-beijing"},
		[]string{"ZoneId", qu.ObjToString(kv["zoneid"])},
		[]string{"VSwitchId", qu.ObjToString(kv["vswitchid"])},
		[]string{"LaunchTemplateId", qu.ObjToString(kv["LaunchTemplateId"+computer])},
		//[]string{"ImageId", "centos_7_06_64_20G_alibase_20181212.vhd"},
		//[]string{"InstanceType", "ecs.ic5.large"},
		//[]string{"SecurityGroupId", "sg-bp16x3td2evrejhkshp7"},[]string{"InternetMaxBandwidthIn", "50"},
		[]string{"InternetMaxBandwidthOut", widthOut},
		[]string{"InstanceChargeType", "PostPaid"},
		[]string{"SpotStrategy", "SpotWithPriceLimit"},
		[]string{"SpotPriceLimit", "4.99"},
		[]string{"InstanceName", "extract"},
		[]string{"UniqueSuffix", "true"},
		[]string{"Password", Password},
		[]string{"Amount", fmt.Sprint(pernum)},
		[]string{"AutoReleaseTime", time.Now().Add(time.Duration(hours) * time.Hour).UTC().Format("2006-01-02T15:04:05Z")},
	})
	if tmp, ok := res["InstanceIdSets"].(map[string]interface{}); ok {
		if t, ok := tmp["InstanceIdSet"].([]interface{}); ok {
			//实例id持久化
			for _, v := range t {
				db.Mgo.Save("ecs", map[string]interface{}{
					"InstanceId": v,
					"TaskName":   taskName,
					"UseFor":     "extract",
				})
			}
		}
	}
	log.Println(res)
}

var aireg = regexp.MustCompile("^ai")

//查询多台实例的详细信息
func DescribeInstances() {
	res := GET("DescribeInstances", [][]string{
		[]string{"RegionId", "cn-beijing"},
		[]string{"InstanceChargeType", "PostPaid"},
		[]string{"PageSize", "100"},
	})
	if res["Instances"] == nil {
		return
	}
	for _, ins := range res["Instances"].(map[string]interface{}) {
		for _, val := range ins.([]interface{}) {
			if tmp, ok := val.(map[string]interface{}); ok {
				if t, ok := tmp["VpcAttributes"].(map[string]interface{}); ok {
					if tt, ok := t["PrivateIpAddress"].(map[string]interface{}); ok {
						ttt := tt["IpAddress"].([]interface{})
						tmp["ip_nw"] = ttt[0]
					}
				}
				if t, ok := tmp["PublicIpAddress"].(map[string]interface{}); ok {
					if tt, ok := t["IpAddress"].([]interface{}); ok && len(tt) > 0 {
						tmp["ip_ww"] = tt[0]
					}
				}
				if aireg.MatchString(qu.ObjToString(tmp["InstanceName"])) || strings.Contains(qu.ObjToString(tmp["InstanceName"]), "ocr_task") {
					continue
				}
				//更新实例信息
				db.Mgo.Update("ecs", `{"InstanceId":"`+qu.ObjToString(tmp["InstanceId"])+`"}`, map[string]interface{}{"$set": tmp}, true, false)
			}
		}
	}
	log.Println(res)
}

//停止实例
func StopInstance(InstanceId string) {
	res := GET("StopInstance", [][]string{
		[]string{"InstanceId", InstanceId},
	})
	db.Mgo.Update("ecs", `{"InstanceId":"`+InstanceId+`"}`, map[string]interface{}{"$set": map[string]interface{}{"Status": "Released"}}, true, false)
	log.Println("StopInstance", res)
}

//释放实例
func DeleteInstance(InstanceId string) {
	res := GET("DeleteInstance", [][]string{
		[]string{"InstanceId", InstanceId},
		[]string{"Force", "true"},
	})
	db.Mgo.Update("ecs", `{"InstanceId":"`+InstanceId+`"}`, map[string]interface{}{"$set": map[string]interface{}{"Status": "Deleted"}}, true, false)
	log.Println("DeleteInstance", res)
}

//实例自动释放时间
func ModifyInstanceAutoReleaseTime(InstanceId string, hours int) {
	res := GET("ModifyInstanceAutoReleaseTime", [][]string{
		[]string{"InstanceId", InstanceId},
		[]string{"AutoReleaseTime", time.Now().Add(time.Duration(hours) * time.Hour).UTC().Format("2006-01-02T15:04:05Z")},
	})
	log.Println("ModifyInstanceAutoReleaseTime", res)
}

//GET请求
func GET(action string, param [][]string) (mres map[string]interface{}) {
	esconfig, _ := ju.Config["esconfig"].(map[string]interface{})
	ps := &paramSorter{[]string{
		"Format",
		"Version",
		"SignatureMethod",
		"SignatureNonce",
		"SignatureVersion",
		"AccessKeyId",
		"Timestamp",
	}, []string{
		"JSON",
		"2014-05-26",
		"HMAC-SHA1",
		fmt.Sprintf("%d", time.Now().UnixNano()/1000),
		"1.0",
		qu.ObjToString(esconfig["AccessID"]),
		time.Now().UTC().Format("2006-01-02T15:04:05Z"),
	}}
	ps.Keys = append(ps.Keys, "Action")
	ps.Vals = append(ps.Vals, action)
	if len(param) > 0 {
		for _, v := range param {
			ps.Keys = append(ps.Keys, v[0])
			ps.Vals = append(ps.Vals, v[1])
		}
	}
	ps.Sort()
	reqStr := ps.String()
	str := "GET&" +
		percentEncode("/") + "&" +
		percentEncode(reqStr)
	str = SP(str, "%3A", "%253A", -1)
	h := hmac.New(func() hash.Hash { return sha1.New() }, []byte(qu.ObjToString(esconfig["AccessSecret"])+"&"))
	io.WriteString(h, str)
	signedStr := base64.StdEncoding.EncodeToString(h.Sum(nil))
	ps.Keys = append(ps.Keys, "Signature")
	ps.Vals = append(ps.Vals, signedStr)
	reqStr = ps.Query()
	res, err := http.Get(fmt.Sprintf("%s?%s", URL, reqStr))
	if err != nil {
		log.Println(err.Error())
	} else {
		defer res.Body.Close()
		bt, err := ioutil.ReadAll(res.Body)
		if err == nil {
			//log.Println(string(bt))
			err := json.Unmarshal(bt, &mres)
			if err != nil {
				log.Println(err.Error())
			}
		}
	}
	return
}

var SP = strings.Replace

func percentEncode(str string) string {
	str = url.QueryEscape(str)
	str = SP(SP(SP(str, "+", "%20", -1), "*", "%2A", -1), "%7E", "~", -1)
	return str
}

type paramSorter struct {
	Keys []string
	Vals []string
}

func (ps *paramSorter) String() string {
	str := ""
	for n, k := range ps.Keys {
		str += k + "=" + ps.Vals[n]
		if n < len(ps.Keys)-1 {
			str += "&"
		}
	}
	return str
}
func (ps *paramSorter) Query() string {
	str := ""
	for n, k := range ps.Keys {
		str += k + "=" + url.QueryEscape(ps.Vals[n])
		if n < len(ps.Keys)-1 {
			str += "&"
		}
	}
	return str
}
func (ps *paramSorter) Sort() {
	sort.Sort(ps)
}
func (ps *paramSorter) Len() int {
	return len(ps.Vals)
}
func (ps *paramSorter) Less(i, j int) bool {
	return bytes.Compare([]byte(ps.Keys[i]), []byte(ps.Keys[j])) < 0
}
func (ps *paramSorter) Swap(i, j int) {
	ps.Vals[i], ps.Vals[j] = ps.Vals[j], ps.Vals[i]
	ps.Keys[i], ps.Keys[j] = ps.Keys[j], ps.Keys[i]
}