1、调整 医疗领域项目

2、新增kvtext
3、数据质量-规则引擎

data_kvtext/remark

@@ -0,0 +1 @@
+标签发现...

data_kvtext/src/initvar.go

@@ -0,0 +1,51 @@
+package main
+
+var (
+	Bid_Mgo, Ext_Mgo *MongodbSim
+	IsLocal          bool
+	KvArr            []string
+	KvMap            map[string]string
+)
+
+// 加载mgo
+func initmgo() {
+	if IsLocal {
+		Bid_Mgo = &MongodbSim{
+			MongodbAddr: "127.0.0.1:12005",
+			DbName:      "extract_2021",
+			Size:        10,
+			UserName:    "zhengkun",
+			Password:    "zk@123123",
+		}
+		//Bid_Mgo.InitPool()
+		Bid_Mgo.InitPoolDirect()
+
+		Ext_Mgo = &MongodbSim{
+			MongodbAddr: "127.0.0.1:12001",
+			DbName:      "qfw",
+			Size:        10,
+			UserName:    "",
+			Password:    "",
+		}
+		//Ext_Mgo.InitPool()
+		Ext_Mgo.InitPoolDirect()
+	} else {
+		Bid_Mgo = &MongodbSim{
+			MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
+			DbName:      "qfw",
+			Size:        10,
+			UserName:    "zhengkun",
+			Password:    "zk@123123",
+		}
+		Bid_Mgo.InitPool()
+
+		Ext_Mgo = &MongodbSim{
+			MongodbAddr: "172.17.4.85:27080",
+			DbName:      "qfw",
+			Size:        10,
+			UserName:    "",
+			Password:    "",
+		}
+		Ext_Mgo.InitPool()
+	}
+}

data_kvtext/src/kvtag.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"fmt"
+	log "github.com/donnie4w/go-logger/logger"
+	qu "qfw/util"
+	"strings"
+)
+
+var (
+	ExistsTag map[string]string
+)
+
+func initExtKvTag() {
+	ExistsTag = make(map[string]string, 0)
+	query := map[string]interface{}{
+		"s_version": "V3.1.2",
+		"s_type":    "string",
+		"delete":    false,
+	}
+	dataArr, _ := Bid_Mgo.Find("tagdetailinfo", query, nil, nil)
+	for _, v := range dataArr {
+		content := IsMarkInterfaceArr(v["content"])
+		for _, v1 := range content {
+			if v1 != "" {
+				ExistsTag[v1] = v1
+			}
+		}
+	}
+	log.Debug("is over kvtag ", len(ExistsTag))
+}
+
+// 给kvtag打标记
+func updateKvTagInfo() {
+	initExtKvTag() //加载现有的标签信息
+	if len(ExistsTag) <= 0 {
+		return
+	}
+	sess := Ext_Mgo.GetMgoConn()
+	defer Ext_Mgo.DestoryMongoConn(sess)
+	q, total := map[string]interface{}{}, 0
+	it := sess.DB(Ext_Mgo.DbName).C("zktest_kvtag_info").Find(&q).Sort("-total").Limit(int64(10000)).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%1000 == 0 {
+			log.Debug("cur index ", total)
+		}
+		name := qu.ObjToString(tmp["name"])
+		tmpid := BsonTOStringId(tmp["_id"])
+		exist := false
+		if ExistsTag[name] != "" {
+			exist = true
+		}
+		Ext_Mgo.UpdateById("zktest_kvtag_info", tmpid, map[string]interface{}{
+			"$set": map[string]interface{}{
+				"exist": exist,
+			},
+		})
+		tmp = make(map[string]interface{})
+	}
+	log.Debug("is over ", total)
+}
+
+// 导出标签信息
+func exportKvTagInfo() {
+	sess := Ext_Mgo.GetMgoConn()
+	defer Ext_Mgo.DestoryMongoConn(sess)
+	q, total := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gte": StringTOBsonId("64c7da800000000000000000"),
+			"$lt":  StringTOBsonId("64f0b9000000000000000000"),
+		},
+	}, 0
+	it := sess.DB(Ext_Mgo.DbName).C("result_20220218").Find(&q).Sort("-_id").Select(map[string]interface{}{
+		"kvtext": 1,
+	}).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%10000 == 0 {
+			log.Debug("cur index ", total, "~", len(KvArr), "~", len(KvMap))
+		}
+		kvtext := qu.ObjToString(tmp["kvtext"])
+		referInfoTransform(kvtext)
+		tmp = make(map[string]interface{})
+	}
+	log.Debug("is over ", total, "~", len(KvArr), "~", len(KvMap))
+
+	saveKvTagInfo()
+}
+func referInfoTransform(kvtext string) {
+	textArr := strings.Split(kvtext, "\n")
+	temp := map[string]string{}
+	for _, v := range textArr {
+		if v != "" {
+			nv := strings.Split(v, ":")[0]
+			if temp[nv] == "" && nv != "" {
+				temp[nv] = nv
+			}
+		}
+	}
+	//存全量
+	for k, _ := range temp {
+		if KvMap[k] == "" {
+			KvMap[k] = "1"
+			KvArr = append(KvArr, k)
+		} else {
+			num := qu.IntAll(KvMap[k])
+			KvMap[k] = fmt.Sprintf("%d", num+1)
+		}
+	}
+}
+func saveKvTagInfo() {
+	log.Debug("开始保存数据...")
+	for k, v := range KvArr {
+		if k%10000 == 0 {
+			log.Debug("is save index ", k, "~", v)
+		}
+		Ext_Mgo.Save("zktest_kvtag_info", map[string]interface{}{
+			"name":  v,
+			"total": qu.IntAll(KvMap[v]),
+		})
+	}
+	log.Debug("is save over ", len(KvArr))
+}

data_kvtext/src/main.go

@@ -0,0 +1,11 @@
+package main
+
+func init() {
+	IsLocal = true
+	initmgo()
+	KvArr, KvMap = []string{}, map[string]string{}
+}
+
+func main() {
+	updateKvTagInfo()
+}

data_kvtext/src/mgo.go

@@ -0,0 +1,431 @@
+package main
+
+import (
+	"context"
+	"log"
+	qu "qfw/util"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type MgoSess struct {
+	Db     string
+	Coll   string
+	Query  interface{}
+	Sorts  []string
+	fields interface{}
+	limit  int64
+	skip   int64
+	M      *MongodbSim
+}
+
+type MgoIter struct {
+	Cursor *mongo.Cursor
+}
+
+func (mt *MgoIter) Next(result interface{}) bool {
+	if mt.Cursor != nil {
+		if mt.Cursor.Next(nil) {
+			err := mt.Cursor.Decode(result)
+			if err != nil {
+				log.Println("mgo cur err", err.Error())
+				mt.Cursor.Close(nil)
+				return false
+			}
+			return true
+		} else {
+			mt.Cursor.Close(nil)
+			return false
+		}
+	} else {
+		return false
+	}
+
+}
+
+func (ms *MgoSess) DB(name string) *MgoSess {
+	ms.Db = name
+	return ms
+}
+
+func (ms *MgoSess) C(name string) *MgoSess {
+	ms.Coll = name
+	return ms
+}
+
+func (ms *MgoSess) Find(q interface{}) *MgoSess {
+	ms.Query = q
+	return ms
+}
+
+func (ms *MgoSess) Select(fields interface{}) *MgoSess {
+	ms.fields = fields
+	return ms
+}
+
+func (ms *MgoSess) Limit(limit int64) *MgoSess {
+	ms.limit = limit
+	return ms
+}
+func (ms *MgoSess) Skip(skip int64) *MgoSess {
+	ms.skip = skip
+	return ms
+}
+
+func (ms *MgoSess) Sort(sorts ...string) *MgoSess {
+	ms.Sorts = sorts
+	return ms
+}
+
+func (ms *MgoSess) Iter() *MgoIter {
+	it := &MgoIter{}
+	find := options.Find()
+	if ms.skip > 0 {
+		find.SetSkip(ms.skip)
+	}
+	if ms.limit > 0 {
+		find.SetLimit(ms.limit)
+	}
+	find.SetBatchSize(100)
+	if len(ms.Sorts) > 0 {
+		sort := bson.M{}
+		for _, k := range ms.Sorts {
+			switch k[:1] {
+			case "-":
+				sort[k[1:]] = -1
+			case "+":
+				sort[k[1:]] = 1
+			default:
+				sort[k] = 1
+			}
+		}
+		find.SetSort(sort)
+	}
+	if ms.fields != nil {
+		find.SetProjection(ms.fields)
+	}
+	cur, err := ms.M.C.Database(ms.Db).Collection(ms.Coll).Find(ms.M.Ctx, ms.Query, find)
+	if err != nil {
+		log.Println("mgo find err", err.Error())
+	} else {
+		it.Cursor = cur
+	}
+	return it
+}
+
+type MongodbSim struct {
+	MongodbAddr string
+	Size        int
+	//	MinSize     int
+	DbName   string
+	C        *mongo.Client
+	Ctx      context.Context
+	ShortCtx context.Context
+	pool     chan bool
+	UserName string
+	Password string
+}
+
+func (m *MongodbSim) GetMgoConn() *MgoSess {
+	//m.Open()
+	ms := &MgoSess{}
+	ms.M = m
+	return ms
+}
+
+func (m *MongodbSim) DestoryMongoConn(ms *MgoSess) {
+	//m.Close()
+	ms.M = nil
+	ms = nil
+}
+
+func (m *MongodbSim) InitPoolDirect() {
+	opts := options.Client()
+	opts.SetConnectTimeout(3 * time.Second)
+	opts.ApplyURI("mongodb://" + m.MongodbAddr)
+	opts.SetMaxPoolSize(uint64(m.Size))
+	opts.SetDirect(true)
+	m.pool = make(chan bool, m.Size)
+
+	if m.UserName != "" && m.Password != "" {
+		cre := options.Credential{
+			Username:   m.UserName,
+			Password:   m.Password,
+			AuthSource: "admin",
+		}
+		opts.SetAuth(cre)
+	}
+
+	opts.SetMaxConnIdleTime(2 * time.Hour)
+	m.Ctx, _ = context.WithTimeout(context.Background(), 99999*time.Hour)
+	m.ShortCtx, _ = context.WithTimeout(context.Background(), 1*time.Minute)
+	client, err := mongo.Connect(m.ShortCtx, opts)
+	if err != nil {
+		log.Println("mgo init error:", err.Error())
+	} else {
+		m.C = client
+		log.Println("init success")
+	}
+}
+
+func (m *MongodbSim) InitPool() {
+	opts := options.Client()
+	opts.SetConnectTimeout(3 * time.Second)
+	opts.ApplyURI("mongodb://" + m.MongodbAddr)
+	opts.SetMaxPoolSize(uint64(m.Size))
+	//opts.SetDirect(true)
+	m.pool = make(chan bool, m.Size)
+
+	if m.UserName != "" && m.Password != "" {
+		cre := options.Credential{
+			Username:   m.UserName,
+			Password:   m.Password,
+			AuthSource: "admin",
+		}
+		opts.SetAuth(cre)
+	}
+
+	opts.SetMaxConnIdleTime(2 * time.Hour)
+	m.Ctx, _ = context.WithTimeout(context.Background(), 99999*time.Hour)
+	m.ShortCtx, _ = context.WithTimeout(context.Background(), 1*time.Minute)
+	client, err := mongo.Connect(m.ShortCtx, opts)
+	if err != nil {
+		log.Println("mgo init error:", err.Error())
+	} else {
+		m.C = client
+		log.Println("init success")
+	}
+}
+
+func (m *MongodbSim) Open() {
+	m.pool <- true
+}
+func (m *MongodbSim) Close() {
+	<-m.pool
+}
+
+// 新建表并生成索引
+func (m *MongodbSim) CreateIndex(c string, models []mongo.IndexModel) bool {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	names, err := coll.Indexes().CreateMany(m.Ctx, models)
+	if err == nil && len(names) > 0 {
+		return true
+	} else {
+		log.Println("CreateIndex Error:", err)
+		return false
+	}
+}
+
+// 批量插入
+func (m *MongodbSim) UpSertBulk(c string, doc ...[]map[string]interface{}) (map[int64]interface{}, bool) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	for _, d := range doc {
+		write := mongo.NewUpdateOneModel()
+		write.SetFilter(d[0])
+		write.SetUpdate(d[1])
+		write.SetUpsert(true)
+		writes = append(writes, write)
+	}
+	r, e := coll.BulkWrite(m.Ctx, writes)
+	if e != nil {
+		log.Println("mgo upsert error:", e.Error())
+		return nil, false
+	}
+	//	else {
+	//		if r.UpsertedCount != int64(len(doc)) {
+	//			log.Println("mgo upsert uncomplete:uc/dc", r.UpsertedCount, len(doc))
+	//		}
+	//		return true
+	//	}
+	return r.UpsertedIDs, true
+}
+
+// 批量插入
+func (m *MongodbSim) SaveBulk(c string, doc ...map[string]interface{}) bool {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	for _, d := range doc {
+		write := mongo.NewInsertOneModel()
+		write.SetDocument(d)
+		writes = append(writes, write)
+	}
+	_, e := coll.BulkWrite(m.Ctx, writes)
+	if e != nil {
+		log.Println("mgo savebulk error:", e.Error())
+		return false
+	}
+	return true
+}
+
+// 保存
+func (m *MongodbSim) Save(c string, doc map[string]interface{}) interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r, err := coll.InsertOne(m.Ctx, doc)
+	if err != nil {
+		return nil
+	}
+	return r.InsertedID
+}
+
+// 更新by Id
+func (m *MongodbSim) UpdateById(c, id string, doc map[string]interface{}) bool {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	_, err := coll.UpdateOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)}, doc)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+func (m *MongodbSim) UpdateStrId(c, id string, doc map[string]interface{}) bool {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	_, err := coll.UpdateOne(m.Ctx, map[string]interface{}{"_id": id}, doc)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+func (m *MongodbSim) UpdateQueryData(c string, query map[string]interface{}, doc map[string]interface{}) bool {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	_, err := coll.UpdateOne(m.Ctx, query, doc)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+// 删除by id
+func (m *MongodbSim) DeleteById(c, id string) int64 {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r, err := coll.DeleteOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)})
+	if err != nil {
+		return 0
+	}
+	return r.DeletedCount
+}
+
+// 通过条件删除
+func (m *MongodbSim) Delete(c string, query map[string]interface{}) int64 {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r, err := coll.DeleteMany(m.Ctx, query)
+	if err != nil {
+		return 0
+	}
+	return r.DeletedCount
+}
+
+// findbyid
+func (m *MongodbSim) FindById(c, id string) map[string]interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r := coll.FindOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)})
+	v := map[string]interface{}{}
+	r.Decode(&v)
+	return v
+}
+
+// findone
+func (m *MongodbSim) FindOne(c string, query map[string]interface{}) map[string]interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r := coll.FindOne(m.Ctx, query)
+	v := map[string]interface{}{}
+	r.Decode(&v)
+	return v
+}
+
+// find
+func (m *MongodbSim) Find(c string, query map[string]interface{}, sort, fields interface{}) ([]map[string]interface{}, error) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	op := options.Find()
+	r, err := coll.Find(m.Ctx, query, op.SetSort(sort), op.SetProjection(fields))
+	if err != nil {
+		log.Fatal(err)
+		return nil, err
+	}
+
+	var results []map[string]interface{}
+	if err = r.All(m.Ctx, &results); err != nil {
+		log.Fatal(err)
+		return nil, err
+	}
+	return results, nil
+}
+
+// find
+func (m *MongodbSim) FindLimit(c string, query map[string]interface{}, sort, fields interface{}, limit int64) ([]map[string]interface{}, error) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	op := options.Find()
+	r, err := coll.Find(m.Ctx, query, op.SetSort(sort), op.SetProjection(fields), op.SetLimit(limit))
+	if err != nil {
+		log.Fatal(err)
+		return nil, err
+	}
+
+	var results []map[string]interface{}
+	if err = r.All(m.Ctx, &results); err != nil {
+		log.Fatal(err)
+		return nil, err
+	}
+	return results, nil
+}
+
+// 创建_id
+func NewObjectId() primitive.ObjectID {
+	return primitive.NewObjectID()
+}
+
+func StringTOBsonId(id string) primitive.ObjectID {
+	objectId, _ := primitive.ObjectIDFromHex(id)
+	return objectId
+}
+
+func BsonTOStringId(id interface{}) string {
+	return id.(primitive.ObjectID).Hex()
+}
+
+func IsMarkInterfaceArr(t interface{}) []string {
+	sub_list := []string{}
+	if list_3, ok_3 := t.([]string); ok_3 {
+		sub_list = list_3
+		return sub_list
+	}
+	if list_1, ok_1 := t.(primitive.A); ok_1 {
+		sub_list = qu.ObjArrToStringArr(list_1)
+	} else {
+		if list_2, ok_2 := t.([]interface{}); ok_2 {
+			sub_list = qu.ObjArrToStringArr(list_2)
+		}
+	}
+	return sub_list
+}

data_quality/src/figure/figure.go

@@ -11,6 +11,37 @@ import (
 	"unicode/utf8"
 )
 
+//打通用分
+func FigureGlobalScoreInfo() {
+	q, total := map[string]interface{}{}, 0
+	sess := u.Ext_Mgo.GetMgoConn()
+	defer u.Ext_Mgo.DestoryMongoConn(sess)
+	it := sess.DB(u.Ext_Mgo.DbName).C("zktest_marked_info").Find(&q).Sort("_id").Select(map[string]interface{}{
+		"qua_res": 1,
+	}).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%10000 == 0 {
+			log.Debug("cur index ", total)
+		}
+		tmpid := u.BsonTOStringId(tmp["_id"])
+		qua_res := *qu.ObjToMap(tmp["qua_res"])
+		qua_score := *qu.ObjToMap(qua_res["qua_score"])
+		ns := 0
+		for _, v := range qua_score {
+			nv := *qu.ObjToMap(v)
+			ns += qu.IntAll(nv["score"])
+		}
+		u.Ext_Mgo.UpdateById("zktest_marked_info", tmpid, map[string]interface{}{
+			"$set": map[string]interface{}{
+				"qua_score": ns,
+			},
+		})
+		tmp = make(map[string]interface{})
+	}
+	log.Debug("is over ...", total)
+
+}
+
 func FigureUnLevelScreenNumber() {
 	dataArr, _ := u.Ext_Mgo.Find("zktest_marked_info", map[string]interface{}{
 		"qua_rejecty": 1,

data_quality/src/figure/govaluate/.gitignore

@@ -0,0 +1,28 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+coverage.out
+
+manual_test.go
+*.out
+*.err

data_quality/src/figure/govaluate/.travis.yml

@@ -0,0 +1,10 @@
+language: go
+
+script: ./test.sh
+
+go:
+  - 1.2
+  - 1.3
+  - 1.4
+  - 1.5
+  - 1.6

data_quality/src/figure/govaluate/CONTRIBUTORS

@@ -0,0 +1,15 @@
+This library was authored by George Lester, and contains contributions from:
+
+vjeantet (regex support)
+iasci (ternary operator)
+oxtoacart (parameter structures, deferred parameter retrieval)
+wmiller848 (bitwise operators)
+prashantv (optimization of bools)
+dpaolella (exposure of variables used in an expression)
+benpaxton (fix for missing type checks during literal elide process)
+abrander (panic-finding testing tool, float32 conversions)
+xfennec (fix for dates being parsed in the current Location)
+bgaifullin (lifting restriction on complex/struct types)
+gautambt (hexadecimal literals)
+felixonmars (fix multiple typos in test names)
+sambonfire (automatic type conversion for accessor function calls)

data_quality/src/figure/govaluate/EvaluableExpression.go

@@ -0,0 +1,276 @@
+package govaluate
+
+import (
+	"errors"
+	"fmt"
+)
+
+const isoDateFormat string = "2006-01-02T15:04:05.999999999Z0700"
+const shortCircuitHolder int = -1
+
+var DUMMY_PARAMETERS = MapParameters(map[string]interface{}{})
+
+/*
+	EvaluableExpression represents a set of ExpressionTokens which, taken together,
+	are an expression that can be evaluated down into a single value.
+*/
+type EvaluableExpression struct {
+
+	/*
+		Represents the query format used to output dates. Typically only used when creating SQL or Mongo queries from an expression.
+		Defaults to the complete ISO8601 format, including nanoseconds.
+	*/
+	QueryDateFormat string
+
+	/*
+		Whether or not to safely check types when evaluating.
+		If true, this library will return error messages when invalid types are used.
+		If false, the library will panic when operators encounter types they can't use.
+
+		This is exclusively for users who need to squeeze every ounce of speed out of the library as they can,
+		and you should only set this to false if you know exactly what you're doing.
+	*/
+	ChecksTypes bool
+
+	tokens           []ExpressionToken
+	evaluationStages *evaluationStage
+	inputExpression  string
+}
+
+/*
+	Parses a new EvaluableExpression from the given [expression] string.
+	Returns an error if the given expression has invalid syntax.
+*/
+func NewEvaluableExpression(expression string) (*EvaluableExpression, error) {
+
+	functions := make(map[string]ExpressionFunction)
+	return NewEvaluableExpressionWithFunctions(expression, functions)
+}
+
+/*
+	Similar to [NewEvaluableExpression], except that instead of a string, an already-tokenized expression is given.
+	This is useful in cases where you may be generating an expression automatically, or using some other parser (e.g., to parse from a query language)
+*/
+func NewEvaluableExpressionFromTokens(tokens []ExpressionToken) (*EvaluableExpression, error) {
+
+	var ret *EvaluableExpression
+	var err error
+
+	ret = new(EvaluableExpression)
+	ret.QueryDateFormat = isoDateFormat
+
+	err = checkBalance(tokens)
+	if err != nil {
+		return nil, err
+	}
+
+	err = checkExpressionSyntax(tokens)
+	if err != nil {
+		return nil, err
+	}
+
+	ret.tokens, err = optimizeTokens(tokens)
+	if err != nil {
+		return nil, err
+	}
+
+	ret.evaluationStages, err = planStages(ret.tokens)
+	if err != nil {
+		return nil, err
+	}
+
+	ret.ChecksTypes = true
+	return ret, nil
+}
+
+/*
+	Similar to [NewEvaluableExpression], except enables the use of user-defined functions.
+	Functions passed into this will be available to the expression.
+*/
+func NewEvaluableExpressionWithFunctions(expression string, functions map[string]ExpressionFunction) (*EvaluableExpression, error) {
+
+	var ret *EvaluableExpression
+	var err error
+
+	ret = new(EvaluableExpression)
+	ret.QueryDateFormat = isoDateFormat
+	ret.inputExpression = expression
+
+	ret.tokens, err = parseTokens(expression, functions)
+	if err != nil {
+		return nil, err
+	}
+
+	err = checkBalance(ret.tokens)
+	if err != nil {
+		return nil, err
+	}
+
+	err = checkExpressionSyntax(ret.tokens)
+	if err != nil {
+		return nil, err
+	}
+
+	ret.tokens, err = optimizeTokens(ret.tokens)
+	if err != nil {
+		return nil, err
+	}
+
+	ret.evaluationStages, err = planStages(ret.tokens)
+	if err != nil {
+		return nil, err
+	}
+
+	ret.ChecksTypes = true
+	return ret, nil
+}
+
+/*
+	Same as `Eval`, but automatically wraps a map of parameters into a `govalute.Parameters` structure.
+*/
+func (this EvaluableExpression) Evaluate(parameters map[string]interface{}) (interface{}, error) {
+
+	if parameters == nil {
+		return this.Eval(nil)
+	}
+
+	return this.Eval(MapParameters(parameters))
+}
+
+/*
+	Runs the entire expression using the given [parameters].
+	e.g., If the expression contains a reference to the variable "foo", it will be taken from `parameters.Get("foo")`.
+
+	This function returns errors if the combination of expression and parameters cannot be run,
+	such as if a variable in the expression is not present in [parameters].
+
+	In all non-error circumstances, this returns the single value result of the expression and parameters given.
+	e.g., if the expression is "1 + 1", this will return 2.0.
+	e.g., if the expression is "foo + 1" and parameters contains "foo" = 2, this will return 3.0
+*/
+func (this EvaluableExpression) Eval(parameters Parameters) (interface{}, error) {
+
+	if this.evaluationStages == nil {
+		return nil, nil
+	}
+
+	if parameters != nil {
+		parameters = &sanitizedParameters{parameters}
+	} else {
+		parameters = DUMMY_PARAMETERS
+	}
+
+	return this.evaluateStage(this.evaluationStages, parameters)
+}
+
+func (this EvaluableExpression) evaluateStage(stage *evaluationStage, parameters Parameters) (interface{}, error) {
+
+	var left, right interface{}
+	var err error
+
+	if stage.leftStage != nil {
+		left, err = this.evaluateStage(stage.leftStage, parameters)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if stage.isShortCircuitable() {
+		switch stage.symbol {
+		case AND:
+			if left == false {
+				return false, nil
+			}
+		case OR:
+			if left == true {
+				return true, nil
+			}
+		case COALESCE:
+			if left != nil {
+				return left, nil
+			}
+
+		case TERNARY_TRUE:
+			if left == false {
+				right = shortCircuitHolder
+			}
+		case TERNARY_FALSE:
+			if left != nil {
+				right = shortCircuitHolder
+			}
+		}
+	}
+
+	if right != shortCircuitHolder && stage.rightStage != nil {
+		right, err = this.evaluateStage(stage.rightStage, parameters)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if this.ChecksTypes {
+		if stage.typeCheck == nil {
+
+			err = typeCheck(stage.leftTypeCheck, left, stage.symbol, stage.typeErrorFormat)
+			if err != nil {
+				return nil, err
+			}
+
+			err = typeCheck(stage.rightTypeCheck, right, stage.symbol, stage.typeErrorFormat)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			// special case where the type check needs to know both sides to determine if the operator can handle it
+			if !stage.typeCheck(left, right) {
+				errorMsg := fmt.Sprintf(stage.typeErrorFormat, left, stage.symbol.String())
+				return nil, errors.New(errorMsg)
+			}
+		}
+	}
+
+	return stage.operator(left, right, parameters)
+}
+
+func typeCheck(check stageTypeCheck, value interface{}, symbol OperatorSymbol, format string) error {
+
+	if check == nil {
+		return nil
+	}
+
+	if check(value) {
+		return nil
+	}
+
+	errorMsg := fmt.Sprintf(format, value, symbol.String())
+	return errors.New(errorMsg)
+}
+
+/*
+	Returns an array representing the ExpressionTokens that make up this expression.
+*/
+func (this EvaluableExpression) Tokens() []ExpressionToken {
+
+	return this.tokens
+}
+
+/*
+	Returns the original expression used to create this EvaluableExpression.
+*/
+func (this EvaluableExpression) String() string {
+
+	return this.inputExpression
+}
+
+/*
+	Returns an array representing the variables contained in this EvaluableExpression.
+*/
+func (this EvaluableExpression) Vars() []string {
+	var varlist []string
+	for _, val := range this.Tokens() {
+		if val.Kind == VARIABLE {
+			varlist = append(varlist, val.Value.(string))
+		}
+	}
+	return varlist
+}

data_quality/src/figure/govaluate/EvaluableExpression_sql.go

@@ -0,0 +1,167 @@
+package govaluate
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"time"
+)
+
+/*
+	Returns a string representing this expression as if it were written in SQL.
+	This function assumes that all parameters exist within the same table, and that the table essentially represents
+	a serialized object of some sort (e.g., hibernate).
+	If your data model is more normalized, you may need to consider iterating through each actual token given by `Tokens()`
+	to create your query.
+
+	Boolean values are considered to be "1" for true, "0" for false.
+
+	Times are formatted according to this.QueryDateFormat.
+*/
+func (this EvaluableExpression) ToSQLQuery() (string, error) {
+
+	var stream *tokenStream
+	var transactions *expressionOutputStream
+	var transaction string
+	var err error
+
+	stream = newTokenStream(this.tokens)
+	transactions = new(expressionOutputStream)
+
+	for stream.hasNext() {
+
+		transaction, err = this.findNextSQLString(stream, transactions)
+		if err != nil {
+			return "", err
+		}
+
+		transactions.add(transaction)
+	}
+
+	return transactions.createString(" "), nil
+}
+
+func (this EvaluableExpression) findNextSQLString(stream *tokenStream, transactions *expressionOutputStream) (string, error) {
+
+	var token ExpressionToken
+	var ret string
+
+	token = stream.next()
+
+	switch token.Kind {
+
+	case STRING:
+		ret = fmt.Sprintf("'%v'", token.Value)
+	case PATTERN:
+		ret = fmt.Sprintf("'%s'", token.Value.(*regexp.Regexp).String())
+	case TIME:
+		ret = fmt.Sprintf("'%s'", token.Value.(time.Time).Format(this.QueryDateFormat))
+
+	case LOGICALOP:
+		switch logicalSymbols[token.Value.(string)] {
+
+		case AND:
+			ret = "AND"
+		case OR:
+			ret = "OR"
+		}
+
+	case BOOLEAN:
+		if token.Value.(bool) {
+			ret = "1"
+		} else {
+			ret = "0"
+		}
+
+	case VARIABLE:
+		ret = fmt.Sprintf("[%s]", token.Value.(string))
+
+	case NUMERIC:
+		ret = fmt.Sprintf("%g", token.Value.(float64))
+
+	case COMPARATOR:
+		switch comparatorSymbols[token.Value.(string)] {
+
+		case EQ:
+			ret = "="
+		case NEQ:
+			ret = "<>"
+		case REQ:
+			ret = "RLIKE"
+		case NREQ:
+			ret = "NOT RLIKE"
+		default:
+			ret = fmt.Sprintf("%s", token.Value.(string))
+		}
+
+	case TERNARY:
+
+		switch ternarySymbols[token.Value.(string)] {
+
+		case COALESCE:
+
+			left := transactions.rollback()
+			right, err := this.findNextSQLString(stream, transactions)
+			if err != nil {
+				return "", err
+			}
+
+			ret = fmt.Sprintf("COALESCE(%v, %v)", left, right)
+		case TERNARY_TRUE:
+			fallthrough
+		case TERNARY_FALSE:
+			return "", errors.New("Ternary operators are unsupported in SQL output")
+		}
+	case PREFIX:
+		switch prefixSymbols[token.Value.(string)] {
+
+		case INVERT:
+			ret = fmt.Sprintf("NOT")
+		default:
+
+			right, err := this.findNextSQLString(stream, transactions)
+			if err != nil {
+				return "", err
+			}
+
+			ret = fmt.Sprintf("%s%s", token.Value.(string), right)
+		}
+	case MODIFIER:
+
+		switch modifierSymbols[token.Value.(string)] {
+
+		case EXPONENT:
+
+			left := transactions.rollback()
+			right, err := this.findNextSQLString(stream, transactions)
+			if err != nil {
+				return "", err
+			}
+
+			ret = fmt.Sprintf("POW(%s, %s)", left, right)
+		case MODULUS:
+
+			left := transactions.rollback()
+			right, err := this.findNextSQLString(stream, transactions)
+			if err != nil {
+				return "", err
+			}
+
+			ret = fmt.Sprintf("MOD(%s, %s)", left, right)
+		default:
+			ret = fmt.Sprintf("%s", token.Value.(string))
+		}
+	case CLAUSE:
+		ret = "("
+	case CLAUSE_CLOSE:
+		ret = ")"
+	case SEPARATOR:
+		ret = ","
+
+	default:
+		errorMsg := fmt.Sprintf("Unrecognized query token '%s' of kind '%s'", token.Value, token.Kind)
+		return "", errors.New(errorMsg)
+	}
+
+	return ret, nil
+}

data_quality/src/figure/govaluate/ExpressionToken.go

@@ -0,0 +1,9 @@
+package govaluate
+
+/*
+	Represents a single parsed token.
+*/
+type ExpressionToken struct {
+	Kind  TokenKind
+	Value interface{}
+}

data_quality/src/figure/govaluate/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014-2016 George Lester
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data_quality/src/figure/govaluate/MANUAL.md

@@ -0,0 +1,176 @@
+govaluate
+====
+
+This library contains quite a lot of functionality, this document is meant to be formal documentation on the operators and features of it.
+Some of this documentation may duplicate what's in README.md, but should never conflict.
+
+# Types
+
+This library only officially deals with four types; `float64`, `bool`, `string`, and arrays.
+
+All numeric literals, with or without a radix, will be converted to `float64` for evaluation. For instance; in practice, there is no difference between the literals "1.0" and "1", they both end up as `float64`. This matters to users because if you intend to return numeric values from your expressions, then the returned value will be `float64`, not any other numeric type.
+
+Any string _literal_ (not parameter) which is interpretable as a date will be converted to a `float64` representation of that date's unix time. Any `time.Time` parameters will not be operable with these date literals; such parameters will need to use the `time.Time.Unix()` method to get a numeric representation.
+
+Arrays are untyped, and can be mixed-type. Internally they're all just `interface{}`. Only two operators can interact with arrays, `IN` and `,`. All other operators will refuse to operate on arrays.
+
+# Operators
+
+## Modifiers
+
+### Addition, concatenation `+`
+
+If either left or right sides of the `+` operator are a `string`, then this operator will perform string concatenation and return that result. If neither are string, then both must be numeric, and this will return a numeric result.
+
+Any other case is invalid.
+
+### Arithmetic `-` `*` `/` `**` `%`
+
+`**` refers to "take to the power of". For instance, `3 ** 4` == 81.
+
+* _Left side_: numeric
+* _Right side_: numeric
+* _Returns_: numeric
+
+### Bitwise shifts, masks `>>` `<<` `|` `&` `^`
+
+All of these operators convert their `float64` left and right sides to `int64`, perform their operation, and then convert back.
+Given how this library assumes numeric are represented (as `float64`), it is unlikely that this behavior will change, even though it may cause havoc with extremely large or small numbers.
+
+* _Left side_: numeric
+* _Right side_: numeric
+* _Returns_: numeric
+
+### Negation `-`
+
+Prefix only. This can never have a left-hand value.
+
+* _Right side_: numeric
+* _Returns_: numeric
+
+### Inversion `!`
+
+Prefix only. This can never have a left-hand value.
+
+* _Right side_: bool
+* _Returns_: bool
+
+### Bitwise NOT `~`
+
+Prefix only. This can never have a left-hand value.
+
+* _Right side_: numeric
+* _Returns_: numeric
+
+## Logical Operators
+
+For all logical operators, this library will short-circuit the operation if the left-hand side is sufficient to determine what to do. For instance, `true || expensiveOperation()` will not actually call `expensiveOperation()`, since it knows the left-hand side is `true`.
+
+### Logical AND/OR `&&` `||`
+
+* _Left side_: bool
+* _Right side_: bool
+* _Returns_: bool
+
+### Ternary true `?`
+
+Checks if the left side is `true`. If so, returns the right side. If the left side is `false`, returns `nil`.
+In practice, this is commonly used with the other ternary operator.
+
+* _Left side_: bool
+* _Right side_: Any type.
+* _Returns_: Right side or `nil`
+
+### Ternary false `:`
+
+Checks if the left side is `nil`. If so, returns the right side. If the left side is non-nil, returns the left side.
+In practice, this is commonly used with the other ternary operator.
+
+* _Left side_: Any type.
+* _Right side_: Any type.
+* _Returns_: Right side or `nil`
+
+### Null coalescence `??`
+
+Similar to the C# operator. If the left value is non-nil, it returns that. If not, then the right-value is returned.
+
+* _Left side_: Any type.
+* _Right side_: Any type.
+* _Returns_: No specific type - whichever is passed to it.
+
+## Comparators
+
+### Numeric/lexicographic comparators `>` `<` `>=` `<=`
+
+If both sides are numeric, this returns the usual greater/lesser behavior that would be expected.
+If both sides are string, this returns the lexicographic comparison of the strings. This uses Go's standard lexicographic compare.
+
+* _Accepts_: Left and right side must either be both string, or both numeric.
+* _Returns_: bool
+
+### Regex comparators `=~` `!~`
+
+These use go's standard `regexp` flavor of regex. The left side is expected to be the candidate string, the right side is the pattern. `=~` returns whether or not the candidate string matches the regex pattern given on the right. `!~` is the inverted version of the same logic.
+
+* _Left side_: string
+* _Right side_: string
+* _Returns_: bool
+
+## Arrays
+
+### Separator `,`
+
+The separator, always paired with parenthesis, creates arrays. It must always have both a left and right-hand value, so for instance `(, 0)` and `(0,)` are invalid uses of it.
+
+Again, this should always be used with parenthesis; like `(1, 2, 3, 4)`.
+
+### Membership `IN`
+
+The only operator with a text name, this operator checks the right-hand side array to see if it contains a value that is equal to the left-side value.
+Equality is determined by the use of the `==` operator, and this library doesn't check types between the values. Any two values, when cast to `interface{}`, and can still be checked for equality with `==` will act as expected.
+
+Note that you can use a parameter for the array, but it must be an `[]interface{}`.
+
+* _Left side_: Any type.
+* _Right side_: array
+* _Returns_: bool
+
+# Parameters
+
+Parameters must be passed in every time the expression is evaluated. Parameters can be of any type, but will not cause errors unless actually used in an erroneous way. There is no difference in behavior for any of the above operators for parameters - they are type checked when used.
+
+All `int` and `float` values of any width will be converted to `float64` before use.
+
+At no point is the parameter structure, or any value thereof, modified by this library.
+
+## Alternates to maps
+
+The default form of parameters as a map may not serve your use case. You may have parameters in some other structure, you may want to change the no-parameter-found behavior, or maybe even just have some debugging print statements invoked when a parameter is accessed.
+
+To do this, define a type that implements the `govaluate.Parameters` interface. When you want to evaluate, instead call `EvaluableExpression.Eval` and pass your parameter structure.
+
+# Functions
+
+During expression parsing (_not_ evaluation), a map of functions can be given to `govaluate.NewEvaluableExpressionWithFunctions` (the lengthiest and finest of function names). The resultant expression will be able to invoke those functions during evaluation. Once parsed, an expression cannot have functions added or removed - a new expression will need to be created if you want to change the functions, or behavior of said functions.
+
+Functions always take the form `<name>(<parameters>)`, including parens. Functions can have an empty list of parameters, like `<name>()`, but still must have parens.
+
+If the expression contains something that looks like it ought to be a function (such as `foo()`), but no such function was given to it, it will error on parsing.
+
+Functions must be of type `map[string]govaluate.ExpressionFunction`. `ExpressionFunction`, for brevity, has the following signature:
+
+`func(args ...interface{}) (interface{}, error)`
+
+Where `args` is whatever is passed to the function when called. If a non-nil error is returned from a function during evaluation, the evaluation stops and ultimately returns that error to the caller of `Evaluate()` or `Eval()`.
+
+## Built-in functions
+
+There aren't any builtin functions. The author is opposed to maintaining a standard library of functions to be used.
+
+Every use case of this library is different, and even in simple use cases (such as parameters, see above) different users need different behavior, naming, or even functionality. The author prefers that users make their own decisions about what functions they need, and how they operate.
+
+# Equality
+
+The `==` and `!=` operators involve a moderately complex workflow. They use [`reflect.DeepEqual`](https://golang.org/pkg/reflect/#DeepEqual). This is for complicated reasons, but there are some types in Go that cannot be compared with the native `==` operator. Arrays, in particular, cannot be compared - Go will panic if you try. One might assume this could be handled with the type checking system in `govaluate`, but unfortunately without reflection there is no way to know if a variable is a slice/array. Worse, structs can be incomparable if they _contain incomparable types_.
+
+It's all very complicated. Fortunately, Go includes the `reflect.DeepEqual` function to handle all the edge cases. Currently, `govaluate` uses that for all equality/inequality.

data_quality/src/figure/govaluate/OperatorSymbol.go

@@ -0,0 +1,309 @@
+package govaluate
+
+/*
+	Represents the valid symbols for operators.
+
+*/
+type OperatorSymbol int
+
+const (
+	VALUE OperatorSymbol = iota
+	LITERAL
+	NOOP
+	EQ
+	NEQ
+	GT
+	LT
+	GTE
+	LTE
+	REQ
+	NREQ
+	IN
+
+	AND
+	OR
+
+	PLUS
+	MINUS
+	BITWISE_AND
+	BITWISE_OR
+	BITWISE_XOR
+	BITWISE_LSHIFT
+	BITWISE_RSHIFT
+	MULTIPLY
+	DIVIDE
+	MODULUS
+	EXPONENT
+
+	NEGATE
+	INVERT
+	BITWISE_NOT
+
+	TERNARY_TRUE
+	TERNARY_FALSE
+	COALESCE
+
+	FUNCTIONAL
+	ACCESS
+	SEPARATE
+)
+
+type operatorPrecedence int
+
+const (
+	noopPrecedence operatorPrecedence = iota
+	valuePrecedence
+	functionalPrecedence
+	prefixPrecedence
+	exponentialPrecedence
+	additivePrecedence
+	bitwisePrecedence
+	bitwiseShiftPrecedence
+	multiplicativePrecedence
+	comparatorPrecedence
+	ternaryPrecedence
+	logicalAndPrecedence
+	logicalOrPrecedence
+	separatePrecedence
+)
+
+func findOperatorPrecedenceForSymbol(symbol OperatorSymbol) operatorPrecedence {
+
+	switch symbol {
+	case NOOP:
+		return noopPrecedence
+	case VALUE:
+		return valuePrecedence
+	case EQ:
+		fallthrough
+	case NEQ:
+		fallthrough
+	case GT:
+		fallthrough
+	case LT:
+		fallthrough
+	case GTE:
+		fallthrough
+	case LTE:
+		fallthrough
+	case REQ:
+		fallthrough
+	case NREQ:
+		fallthrough
+	case IN:
+		return comparatorPrecedence
+	case AND:
+		return logicalAndPrecedence
+	case OR:
+		return logicalOrPrecedence
+	case BITWISE_AND:
+		fallthrough
+	case BITWISE_OR:
+		fallthrough
+	case BITWISE_XOR:
+		return bitwisePrecedence
+	case BITWISE_LSHIFT:
+		fallthrough
+	case BITWISE_RSHIFT:
+		return bitwiseShiftPrecedence
+	case PLUS:
+		fallthrough
+	case MINUS:
+		return additivePrecedence
+	case MULTIPLY:
+		fallthrough
+	case DIVIDE:
+		fallthrough
+	case MODULUS:
+		return multiplicativePrecedence
+	case EXPONENT:
+		return exponentialPrecedence
+	case BITWISE_NOT:
+		fallthrough
+	case NEGATE:
+		fallthrough
+	case INVERT:
+		return prefixPrecedence
+	case COALESCE:
+		fallthrough
+	case TERNARY_TRUE:
+		fallthrough
+	case TERNARY_FALSE:
+		return ternaryPrecedence
+	case ACCESS:
+		fallthrough
+	case FUNCTIONAL:
+		return functionalPrecedence
+	case SEPARATE:
+		return separatePrecedence
+	}
+
+	return valuePrecedence
+}
+
+/*
+	Map of all valid comparators, and their string equivalents.
+	Used during parsing of expressions to determine if a symbol is, in fact, a comparator.
+	Also used during evaluation to determine exactly which comparator is being used.
+*/
+var comparatorSymbols = map[string]OperatorSymbol{
+	"==": EQ,
+	"!=": NEQ,
+	">":  GT,
+	">=": GTE,
+	"<":  LT,
+	"<=": LTE,
+	"=~": REQ,
+	"!~": NREQ,
+	"in": IN,
+}
+
+var logicalSymbols = map[string]OperatorSymbol{
+	"&&": AND,
+	"||": OR,
+}
+
+var bitwiseSymbols = map[string]OperatorSymbol{
+	"^": BITWISE_XOR,
+	"&": BITWISE_AND,
+	"|": BITWISE_OR,
+}
+
+var bitwiseShiftSymbols = map[string]OperatorSymbol{
+	">>": BITWISE_RSHIFT,
+	"<<": BITWISE_LSHIFT,
+}
+
+var additiveSymbols = map[string]OperatorSymbol{
+	"+": PLUS,
+	"-": MINUS,
+}
+
+var multiplicativeSymbols = map[string]OperatorSymbol{
+	"*": MULTIPLY,
+	"/": DIVIDE,
+	"%": MODULUS,
+}
+
+var exponentialSymbolsS = map[string]OperatorSymbol{
+	"**": EXPONENT,
+}
+
+var prefixSymbols = map[string]OperatorSymbol{
+	"-": NEGATE,
+	"!": INVERT,
+	"~": BITWISE_NOT,
+}
+
+var ternarySymbols = map[string]OperatorSymbol{
+	"?":  TERNARY_TRUE,
+	":":  TERNARY_FALSE,
+	"??": COALESCE,
+}
+
+// this is defined separately from additiveSymbols et al because it's needed for parsing, not stage planning.
+var modifierSymbols = map[string]OperatorSymbol{
+	"+":  PLUS,
+	"-":  MINUS,
+	"*":  MULTIPLY,
+	"/":  DIVIDE,
+	"%":  MODULUS,
+	"**": EXPONENT,
+	"&":  BITWISE_AND,
+	"|":  BITWISE_OR,
+	"^":  BITWISE_XOR,
+	">>": BITWISE_RSHIFT,
+	"<<": BITWISE_LSHIFT,
+}
+
+var separatorSymbols = map[string]OperatorSymbol{
+	",": SEPARATE,
+}
+
+/*
+	Returns true if this operator is contained by the given array of candidate symbols.
+	False otherwise.
+*/
+func (this OperatorSymbol) IsModifierType(candidate []OperatorSymbol) bool {
+
+	for _, symbolType := range candidate {
+		if this == symbolType {
+			return true
+		}
+	}
+
+	return false
+}
+
+/*
+	Generally used when formatting type check errors.
+	We could store the stringified symbol somewhere else and not require a duplicated codeblock to translate
+	OperatorSymbol to string, but that would require more memory, and another field somewhere.
+	Adding operators is rare enough that we just stringify it here instead.
+*/
+func (this OperatorSymbol) String() string {
+
+	switch this {
+	case NOOP:
+		return "NOOP"
+	case VALUE:
+		return "VALUE"
+	case EQ:
+		return "="
+	case NEQ:
+		return "!="
+	case GT:
+		return ">"
+	case LT:
+		return "<"
+	case GTE:
+		return ">="
+	case LTE:
+		return "<="
+	case REQ:
+		return "=~"
+	case NREQ:
+		return "!~"
+	case AND:
+		return "&&"
+	case OR:
+		return "||"
+	case IN:
+		return "in"
+	case BITWISE_AND:
+		return "&"
+	case BITWISE_OR:
+		return "|"
+	case BITWISE_XOR:
+		return "^"
+	case BITWISE_LSHIFT:
+		return "<<"
+	case BITWISE_RSHIFT:
+		return ">>"
+	case PLUS:
+		return "+"
+	case MINUS:
+		return "-"
+	case MULTIPLY:
+		return "*"
+	case DIVIDE:
+		return "/"
+	case MODULUS:
+		return "%"
+	case EXPONENT:
+		return "**"
+	case NEGATE:
+		return "-"
+	case INVERT:
+		return "!"
+	case BITWISE_NOT:
+		return "~"
+	case TERNARY_TRUE:
+		return "?"
+	case TERNARY_FALSE:
+		return ":"
+	case COALESCE:
+		return "??"
+	}
+	return ""
+}

data_quality/src/figure/govaluate/README.md

@@ -0,0 +1,233 @@
+govaluate
+====
+
+[![Build Status](https://travis-ci.org/Knetic/govaluate.svg?branch=master)](https://travis-ci.org/Knetic/govaluate)
+[![Godoc](https://img.shields.io/badge/godoc-reference-5272B4.svg)](https://godoc.org/github.com/Knetic/govaluate)
+[![Go Report Card](https://goreportcard.com/badge/github.com/Knetic/govaluate)](https://goreportcard.com/report/github.com/Knetic/govaluate) 
+[![Gocover](https://gocover.io/_badge/github.com/Knetic/govaluate)](https://gocover.io/github.com/Knetic/govaluate)
+
+Provides support for evaluating arbitrary C-like artithmetic/string expressions.
+
+Why can't you just write these expressions in code?
+--
+
+Sometimes, you can't know ahead-of-time what an expression will look like, or you want those expressions to be configurable.
+Perhaps you've got a set of data running through your application, and you want to allow your users to specify some validations to run on it before committing it to a database. Or maybe you've written a monitoring framework which is capable of gathering a bunch of metrics, then evaluating a few expressions to see if any metrics should be alerted upon, but the conditions for alerting are different for each monitor.
+
+A lot of people wind up writing their own half-baked style of evaluation language that fits their needs, but isn't complete. Or they wind up baking the expression into the actual executable, even if they know it's subject to change. These strategies may work, but they take time to implement, time for users to learn, and induce technical debt as requirements change. This library is meant to cover all the normal C-like expressions, so that you don't have to reinvent one of the oldest wheels on a computer.
+
+How do I use it?
+--
+
+You create a new EvaluableExpression, then call "Evaluate" on it.
+
+```go
+	expression, err := govaluate.NewEvaluableExpression("10 > 0");
+	result, err := expression.Evaluate(nil);
+	// result is now set to "true", the bool value.
+```
+
+Cool, but how about with parameters?
+
+```go
+	expression, err := govaluate.NewEvaluableExpression("foo > 0");
+
+	parameters := make(map[string]interface{}, 8)
+	parameters["foo"] = -1;
+
+	result, err := expression.Evaluate(parameters);
+	// result is now set to "false", the bool value.
+```
+
+That's cool, but we can almost certainly have done all that in code. What about a complex use case that involves some math?
+
+```go
+	expression, err := govaluate.NewEvaluableExpression("(requests_made * requests_succeeded / 100) >= 90");
+
+	parameters := make(map[string]interface{}, 8)
+	parameters["requests_made"] = 100;
+	parameters["requests_succeeded"] = 80;
+
+	result, err := expression.Evaluate(parameters);
+	// result is now set to "false", the bool value.
+```
+
+Or maybe you want to check the status of an alive check ("smoketest") page, which will be a string?
+
+```go
+	expression, err := govaluate.NewEvaluableExpression("http_response_body == 'service is ok'");
+
+	parameters := make(map[string]interface{}, 8)
+	parameters["http_response_body"] = "service is ok";
+
+	result, err := expression.Evaluate(parameters);
+	// result is now set to "true", the bool value.
+```
+
+These examples have all returned boolean values, but it's equally possible to return numeric ones.
+
+```go
+	expression, err := govaluate.NewEvaluableExpression("(mem_used / total_mem) * 100");
+
+	parameters := make(map[string]interface{}, 8)
+	parameters["total_mem"] = 1024;
+	parameters["mem_used"] = 512;
+
+	result, err := expression.Evaluate(parameters);
+	// result is now set to "50.0", the float64 value.
+```
+
+You can also do date parsing, though the formats are somewhat limited. Stick to RF3339, ISO8061, unix date, or ruby date formats. If you're having trouble getting a date string to parse, check the list of formats actually used: [parsing.go:248](https://github.com/Knetic/govaluate/blob/0580e9b47a69125afa0e4ebd1cf93c49eb5a43ec/parsing.go#L258).
+
+```go
+	expression, err := govaluate.NewEvaluableExpression("'2014-01-02' > '2014-01-01 23:59:59'");
+	result, err := expression.Evaluate(nil);
+
+	// result is now set to true
+```
+
+Expressions are parsed once, and can be re-used multiple times. Parsing is the compute-intensive phase of the process, so if you intend to use the same expression with different parameters, just parse it once. Like so;
+
+```go
+	expression, err := govaluate.NewEvaluableExpression("response_time <= 100");
+	parameters := make(map[string]interface{}, 8)
+
+	for {
+		parameters["response_time"] = pingSomething();
+		result, err := expression.Evaluate(parameters)
+	}
+```
+
+The normal C-standard order of operators is respected. When writing an expression, be sure that you either order the operators correctly, or use parenthesis to clarify which portions of an expression should be run first.
+
+Escaping characters
+--
+
+Sometimes you'll have parameters that have spaces, slashes, pluses, ampersands or some other character
+that this library interprets as something special. For example, the following expression will not
+act as one might expect:
+
+	"response-time < 100"
+
+As written, the library will parse it as "[response] minus [time] is less than 100". In reality,
+"response-time" is meant to be one variable that just happens to have a dash in it.
+
+There are two ways to work around this. First, you can escape the entire parameter name:
+
+ 	"[response-time] < 100"
+
+Or you can use backslashes to escape only the minus sign.
+
+	"response\\-time < 100"
+
+Backslashes can be used anywhere in an expression to escape the very next character. Square bracketed parameter names can be used instead of plain parameter names at any time.
+
+Functions
+--
+
+You may have cases where you want to call a function on a parameter during execution of the expression. Perhaps you want to aggregate some set of data, but don't know the exact aggregation you want to use until you're writing the expression itself. Or maybe you have a mathematical operation you want to perform, for which there is no operator; like `log` or `tan` or `sqrt`. For cases like this, you can provide a map of functions to `NewEvaluableExpressionWithFunctions`, which will then be able to use them during execution. For instance;
+
+```go
+	functions := map[string]govaluate.ExpressionFunction {
+		"strlen": func(args ...interface{}) (interface{}, error) {
+			length := len(args[0].(string))
+			return (float64)(length), nil
+		},
+	}
+
+	expString := "strlen('someReallyLongInputString') <= 16"
+	expression, _ := govaluate.NewEvaluableExpressionWithFunctions(expString, functions)
+
+	result, _ := expression.Evaluate(nil)
+	// result is now "false", the boolean value
+```
+
+Functions can accept any number of arguments, correctly handles nested functions, and arguments can be of any type (even if none of this library's operators support evaluation of that type). For instance, each of these usages of functions in an expression are valid (assuming that the appropriate functions and parameters are given):
+
+```go
+"sqrt(x1 ** y1, x2 ** y2)"
+"max(someValue, abs(anotherValue), 10 * lastValue)"
+```
+
+Functions cannot be passed as parameters, they must be known at the time when the expression is parsed, and are unchangeable after parsing.
+
+Accessors
+--
+
+If you have structs in your parameters, you can access their fields and methods in the usual way. For instance, given a struct that has a method "Echo", present in the parameters as `foo`, the following is valid:
+
+	"foo.Echo('hello world')"
+
+Fields are accessed in a similar way. Assuming `foo` has a field called "Length":
+
+	"foo.Length > 9000"
+
+Accessors can be nested to any depth, like the following
+
+	"foo.Bar.Baz.SomeFunction()"
+
+However it is not _currently_ supported to access values in `map`s. So the following will not work
+
+	"foo.SomeMap['key']"
+
+This may be convenient, but note that using accessors involves a _lot_ of reflection. This makes the expression about four times slower than just using a parameter (consult the benchmarks for more precise measurements on your system).
+If at all reasonable, the author recommends extracting the values you care about into a parameter map beforehand, or defining a struct that implements the `Parameters` interface, and which grabs fields as required. If there are functions you want to use, it's better to pass them as expression functions (see the above section). These approaches use no reflection, and are designed to be fast and clean.
+
+What operators and types does this support?
+--
+
+* Modifiers: `+` `-` `/` `*` `&` `|` `^` `**` `%` `>>` `<<`
+* Comparators: `>` `>=` `<` `<=` `==` `!=` `=~` `!~`
+* Logical ops: `||` `&&`
+* Numeric constants, as 64-bit floating point (`12345.678`)
+* String constants (single quotes: `'foobar'`)
+* Date constants (single quotes, using any permutation of RFC3339, ISO8601, ruby date, or unix date; date parsing is automatically tried with any string constant)
+* Boolean constants: `true` `false`
+* Parenthesis to control order of evaluation `(` `)`
+* Arrays (anything separated by `,` within parenthesis: `(1, 2, 'foo')`)
+* Prefixes: `!` `-` `~`
+* Ternary conditional: `?` `:`
+* Null coalescence: `??`
+
+See [MANUAL.md](https://github.com/Knetic/govaluate/blob/master/MANUAL.md) for exacting details on what types each operator supports.
+
+Types
+--
+
+Some operators don't make sense when used with some types. For instance, what does it mean to get the modulo of a string? What happens if you check to see if two numbers are logically AND'ed together?
+
+Everyone has a different intuition about the answers to these questions. To prevent confusion, this library will _refuse to operate_ upon types for which there is not an unambiguous meaning for the operation. See [MANUAL.md](https://github.com/Knetic/govaluate/blob/master/MANUAL.md) for details about what operators are valid for which types.
+
+Benchmarks
+--
+
+If you're concerned about the overhead of this library, a good range of benchmarks are built into this repo. You can run them with `go test -bench=.`. The library is built with an eye towards being quick, but has not been aggressively profiled and optimized. For most applications, though, it is completely fine.
+
+For a very rough idea of performance, here are the results output from a benchmark run on a 3rd-gen Macbook Pro (Linux Mint 17.1).
+
+```
+BenchmarkSingleParse-12                          1000000              1382 ns/op
+BenchmarkSimpleParse-12                           200000             10771 ns/op
+BenchmarkFullParse-12                              30000             49383 ns/op
+BenchmarkEvaluationSingle-12                    50000000                30.1 ns/op
+BenchmarkEvaluationNumericLiteral-12            10000000               119 ns/op
+BenchmarkEvaluationLiteralModifiers-12          10000000               236 ns/op
+BenchmarkEvaluationParameters-12                 5000000               260 ns/op
+BenchmarkEvaluationParametersModifiers-12        3000000               547 ns/op
+BenchmarkComplexExpression-12                    2000000               963 ns/op
+BenchmarkRegexExpression-12                       100000             20357 ns/op
+BenchmarkConstantRegexExpression-12              1000000              1392 ns/op
+ok
+```
+
+API Breaks
+--
+
+While this library has very few cases which will ever result in an API break, it can (and [has](https://github.com/Knetic/govaluate/releases/tag/v2.0.0)) happened. If you are using this in production, vendor the commit you've tested against, or use gopkg.in to redirect your import (e.g., `import "gopkg.in/Knetic/govaluate.v2"`). Master branch (while infrequent) _may_ at some point contain API breaking changes, and the author will have no way to communicate these to downstreams, other than creating a new major release.
+
+Releases will explicitly state when an API break happens, and if they do not specify an API break it should be safe to upgrade.
+
+License
+--
+
+This project is licensed under the MIT general use license. You're free to integrate, fork, and play with this code as you feel fit without consulting the author, as long as you provide proper credit to the author in your works.

data_quality/src/figure/govaluate/TokenKind.go

@@ -0,0 +1,75 @@
+package govaluate
+
+/*
+	Represents all valid types of tokens that a token can be.
+*/
+type TokenKind int
+
+const (
+	UNKNOWN TokenKind = iota
+
+	PREFIX
+	NUMERIC
+	BOOLEAN
+	STRING
+	PATTERN
+	TIME
+	VARIABLE
+	FUNCTION
+	SEPARATOR
+	ACCESSOR
+
+	COMPARATOR
+	LOGICALOP
+	MODIFIER
+
+	CLAUSE
+	CLAUSE_CLOSE
+
+	TERNARY
+)
+
+/*
+	GetTokenKindString returns a string that describes the given TokenKind.
+	e.g., when passed the NUMERIC TokenKind, this returns the string "NUMERIC".
+*/
+func (kind TokenKind) String() string {
+
+	switch kind {
+
+	case PREFIX:
+		return "PREFIX"
+	case NUMERIC:
+		return "NUMERIC"
+	case BOOLEAN:
+		return "BOOLEAN"
+	case STRING:
+		return "STRING"
+	case PATTERN:
+		return "PATTERN"
+	case TIME:
+		return "TIME"
+	case VARIABLE:
+		return "VARIABLE"
+	case FUNCTION:
+		return "FUNCTION"
+	case SEPARATOR:
+		return "SEPARATOR"
+	case COMPARATOR:
+		return "COMPARATOR"
+	case LOGICALOP:
+		return "LOGICALOP"
+	case MODIFIER:
+		return "MODIFIER"
+	case CLAUSE:
+		return "CLAUSE"
+	case CLAUSE_CLOSE:
+		return "CLAUSE_CLOSE"
+	case TERNARY:
+		return "TERNARY"
+	case ACCESSOR:
+		return "ACCESSOR"
+	}
+
+	return "UNKNOWN"
+}

data_quality/src/figure/govaluate/benchmarks_test.go

@@ -0,0 +1,251 @@
+package govaluate
+
+import (
+	"testing"
+)
+
+/*
+  Serves as a "water test" to give an idea of the general overhead of parsing
+*/
+func BenchmarkSingleParse(bench *testing.B) {
+
+	for i := 0; i < bench.N; i++ {
+		NewEvaluableExpression("1")
+	}
+}
+
+/*
+  The most common use case, a single variable, modified slightly, compared to a constant.
+  This is the "expected" use case of govaluate.
+*/
+func BenchmarkSimpleParse(bench *testing.B) {
+
+	for i := 0; i < bench.N; i++ {
+		NewEvaluableExpression("(requests_made * requests_succeeded / 100) >= 90")
+	}
+}
+
+/*
+  Benchmarks all syntax possibilities in one expression.
+*/
+func BenchmarkFullParse(bench *testing.B) {
+
+	var expression string
+
+	// represents all the major syntax possibilities.
+	expression = "2 > 1 &&" +
+		"'something' != 'nothing' || " +
+		"'2014-01-20' < 'Wed Jul  8 23:07:35 MDT 2015' && " +
+		"[escapedVariable name with spaces] <= unescaped\\-variableName &&" +
+		"modifierTest + 1000 / 2 > (80 * 100 % 2)"
+
+	for i := 0; i < bench.N; i++ {
+		NewEvaluableExpression(expression)
+	}
+}
+
+/*
+  Benchmarks the bare-minimum evaluation time
+*/
+func BenchmarkEvaluationSingle(bench *testing.B) {
+
+	expression, _ := NewEvaluableExpression("1")
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(nil)
+	}
+}
+
+/*
+  Benchmarks evaluation times of literals (no variables, no modifiers)
+*/
+func BenchmarkEvaluationNumericLiteral(bench *testing.B) {
+
+	expression, _ := NewEvaluableExpression("(2) > (1)")
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(nil)
+	}
+}
+
+/*
+  Benchmarks evaluation times of literals with modifiers
+*/
+func BenchmarkEvaluationLiteralModifiers(bench *testing.B) {
+
+	expression, _ := NewEvaluableExpression("(2) + (2) == (4)")
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(nil)
+	}
+}
+
+func BenchmarkEvaluationParameter(bench *testing.B) {
+
+	expression, _ := NewEvaluableExpression("requests_made")
+	parameters := map[string]interface{}{
+		"requests_made": 99.0,
+	}
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(parameters)
+	}
+}
+
+/*
+  Benchmarks evaluation times of parameters
+*/
+func BenchmarkEvaluationParameters(bench *testing.B) {
+
+	expression, _ := NewEvaluableExpression("requests_made > requests_succeeded")
+	parameters := map[string]interface{}{
+		"requests_made":      99.0,
+		"requests_succeeded": 90.0,
+	}
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(parameters)
+	}
+}
+
+/*
+  Benchmarks evaluation times of parameters + literals with modifiers
+*/
+func BenchmarkEvaluationParametersModifiers(bench *testing.B) {
+
+	expression, _ := NewEvaluableExpression("(requests_made * requests_succeeded / 100) >= 90")
+	parameters := map[string]interface{}{
+		"requests_made":      99.0,
+		"requests_succeeded": 90.0,
+	}
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(parameters)
+	}
+}
+
+/*
+  Benchmarks the ludicrously-unlikely worst-case expression,
+  one which uses all features.
+  This is largely a canary benchmark to make sure that any syntax additions don't
+  unnecessarily bloat the evaluation time.
+*/
+func BenchmarkComplexExpression(bench *testing.B) {
+
+	var expressionString string
+
+	expressionString = "2 > 1 &&" +
+		"'something' != 'nothing' || " +
+		"'2014-01-20' < 'Wed Jul  8 23:07:35 MDT 2015' && " +
+		"[escapedVariable name with spaces] <= unescaped\\-variableName &&" +
+		"modifierTest + 1000 / 2 > (80 * 100 % 2)"
+
+	expression, _ := NewEvaluableExpression(expressionString)
+	parameters := map[string]interface{}{
+		"escapedVariable name with spaces": 99.0,
+		"unescaped\\-variableName":         90.0,
+		"modifierTest":                     5.0,
+	}
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(parameters)
+	}
+}
+
+/*
+  Benchmarks uncompiled parameter regex operators, which are the most expensive of the lot.
+  Note that regex compilation times are unpredictable and wily things. The regex engine has a lot of edge cases
+  and possible performance pitfalls. This test doesn't aim to be comprehensive against all possible regex scenarios,
+  it is primarily concerned with tracking how much longer it takes to compile a regex at evaluation-time than during parse-time.
+*/
+func BenchmarkRegexExpression(bench *testing.B) {
+
+	var expressionString string
+
+	expressionString = "(foo !~ bar) && (foobar =~ oba)"
+
+	expression, _ := NewEvaluableExpression(expressionString)
+	parameters := map[string]interface{}{
+		"foo": "foo",
+		"bar": "bar",
+		"baz": "baz",
+		"oba": ".*oba.*",
+	}
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(parameters)
+	}
+}
+
+/*
+	Benchmarks pre-compilable regex patterns. Meant to serve as a sanity check that constant strings used as regex patterns
+	are actually being precompiled.
+	Also demonstrates that (generally) compiling a regex at evaluation-time takes an order of magnitude more time than pre-compiling.
+*/
+func BenchmarkConstantRegexExpression(bench *testing.B) {
+
+	expressionString := "(foo !~ '[bB]az') && (bar =~ '[bB]ar')"
+	expression, _ := NewEvaluableExpression(expressionString)
+
+	parameters := map[string]interface{}{
+		"foo": "foo",
+		"bar": "bar",
+	}
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(parameters)
+	}
+}
+
+func BenchmarkAccessors(bench *testing.B) {
+
+	expressionString := "foo.Int"
+	expression, _ := NewEvaluableExpression(expressionString)
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(fooFailureParameters)
+	}
+}
+
+func BenchmarkAccessorMethod(bench *testing.B) {
+
+	expressionString := "foo.Func()"
+	expression, _ := NewEvaluableExpression(expressionString)
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(fooFailureParameters)
+	}
+}
+
+func BenchmarkAccessorMethodParams(bench *testing.B) {
+
+	expressionString := "foo.FuncArgStr('bonk')"
+	expression, _ := NewEvaluableExpression(expressionString)
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(fooFailureParameters)
+	}
+}
+
+func BenchmarkNestedAccessors(bench *testing.B) {
+
+	expressionString := "foo.Nested.Funk"
+	expression, _ := NewEvaluableExpression(expressionString)
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		expression.Evaluate(fooFailureParameters)
+	}
+}

data_quality/src/figure/govaluate/dummies_test.go

@@ -0,0 +1,85 @@
+package govaluate
+
+import (
+	"errors"
+	"fmt"
+)
+
+/*
+	Struct used to test "parameter calls".
+*/
+type dummyParameter struct {
+	String    string
+	Int       int
+	BoolFalse bool
+	Nil       interface{}
+	Nested    dummyNestedParameter
+}
+
+func (this dummyParameter) Func() string {
+	return "funk"
+}
+
+func (this dummyParameter) Func2() (string, error) {
+	return "frink", nil
+}
+
+func (this *dummyParameter) Func3() string {
+	return "fronk"
+}
+
+func (this dummyParameter) FuncArgStr(arg1 string) string {
+	return arg1
+}
+
+func (this dummyParameter) TestArgs(str string, ui uint, ui8 uint8, ui16 uint16, ui32 uint32, ui64 uint64, i int, i8 int8, i16 int16, i32 int32, i64 int64, f32 float32, f64 float64, b bool) string {
+	
+	var sum float64
+	
+	sum = float64(ui) + float64(ui8) + float64(ui16) + float64(ui32) + float64(ui64)
+	sum += float64(i) + float64(i8) + float64(i16) + float64(i32) + float64(i64)
+	sum += float64(f32)
+
+	if b {
+		sum += f64
+	}
+
+	return fmt.Sprintf("%v: %v", str, sum)
+}
+
+func (this dummyParameter) AlwaysFail() (interface{}, error) {
+	return nil, errors.New("function should always fail")
+}
+
+type dummyNestedParameter struct {
+	Funk string
+}
+
+func (this dummyNestedParameter) Dunk(arg1 string) string {
+	return arg1 + "dunk"
+}
+
+var dummyParameterInstance = dummyParameter{
+	String:    "string!",
+	Int:       101,
+	BoolFalse: false,
+	Nil:       nil,
+	Nested: dummyNestedParameter{
+		Funk: "funkalicious",
+	},
+}
+
+var fooParameter = EvaluationParameter{
+	Name:  "foo",
+	Value: dummyParameterInstance,
+}
+
+var fooPtrParameter = EvaluationParameter{
+	Name:  "fooptr",
+	Value: &dummyParameterInstance,
+}
+
+var fooFailureParameters = map[string]interface{}{
+	"foo":    fooParameter.Value,
+	"fooptr": &fooPtrParameter.Value,
+}

data_quality/src/figure/govaluate/evaluationFailure_test.go

@@ -0,0 +1,543 @@
+package govaluate
+
+/*
+  Tests to make sure evaluation fails in the expected ways.
+*/
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+)
+
+type DebugStruct struct {
+	x int
+}
+
+/*
+	Represents a test for parsing failures
+*/
+type EvaluationFailureTest struct {
+	Name       string
+	Input      string
+	Functions  map[string]ExpressionFunction
+	Parameters map[string]interface{}
+	Expected   string
+}
+
+const (
+	INVALID_MODIFIER_TYPES   string = "cannot be used with the modifier"
+	INVALID_COMPARATOR_TYPES        = "cannot be used with the comparator"
+	INVALID_LOGICALOP_TYPES         = "cannot be used with the logical operator"
+	INVALID_TERNARY_TYPES           = "cannot be used with the ternary operator"
+	ABSENT_PARAMETER                = "No parameter"
+	INVALID_REGEX                   = "Unable to compile regexp pattern"
+	INVALID_PARAMETER_CALL          = "No method or field"
+	TOO_FEW_ARGS                    = "Too few arguments to parameter call"
+	TOO_MANY_ARGS                   = "Too many arguments to parameter call"
+	MISMATCHED_PARAMETERS           = "Argument type conversion failed"
+)
+
+// preset parameter map of types that can be used in an evaluation failure test to check typing.
+var EVALUATION_FAILURE_PARAMETERS = map[string]interface{}{
+	"number": 1,
+	"string": "foo",
+	"bool":   true,
+}
+
+func TestComplexParameter(test *testing.T) {
+
+	var expression *EvaluableExpression
+	var err error
+	var v interface{}
+
+	parameters := map[string]interface{}{
+		"complex64":  complex64(0),
+		"complex128": complex128(0),
+	}
+
+	expression, _ = NewEvaluableExpression("complex64")
+	v, err = expression.Evaluate(parameters)
+	if err != nil {
+		test.Errorf("Expected no error, but have %s", err)
+	}
+	if v.(complex64) != complex64(0) {
+		test.Errorf("Expected %v == %v", v, complex64(0))
+	}
+
+	expression, _ = NewEvaluableExpression("complex128")
+	v, err = expression.Evaluate(parameters)
+	if err != nil {
+		test.Errorf("Expected no error, but have %s", err)
+	}
+	if v.(complex128) != complex128(0) {
+		test.Errorf("Expected %v == %v", v, complex128(0))
+	}
+}
+
+func TestStructParameter(t *testing.T) {
+	expected := DebugStruct{}
+	expression, _ := NewEvaluableExpression("foo")
+	parameters := map[string]interface{}{"foo": expected}
+	v, err := expression.Evaluate(parameters)
+	if err != nil {
+		t.Errorf("Expected no error, but have %s", err)
+	} else if v.(DebugStruct) != expected {
+		t.Errorf("Values mismatch: %v != %v", expected, v)
+	}
+}
+
+func TestNilParameterUsage(test *testing.T) {
+
+	expression, err := NewEvaluableExpression("2 > 1")
+	_, err = expression.Evaluate(nil)
+
+	if err != nil {
+		test.Errorf("Expected no error from nil parameter evaluation, got %v\n", err)
+		return
+	}
+}
+
+func TestModifierTyping(test *testing.T) {
+
+	evaluationTests := []EvaluationFailureTest{
+		EvaluationFailureTest{
+
+			Name:     "PLUS literal number to literal bool",
+			Input:    "1 + true",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "PLUS number to bool",
+			Input:    "number + bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "MINUS number to bool",
+			Input:    "number - bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "MINUS number to bool",
+			Input:    "number - bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "MULTIPLY number to bool",
+			Input:    "number * bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "DIVIDE number to bool",
+			Input:    "number / bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "EXPONENT number to bool",
+			Input:    "number ** bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "MODULUS number to bool",
+			Input:    "number % bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "XOR number to bool",
+			Input:    "number % bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "BITWISE_OR number to bool",
+			Input:    "number | bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "BITWISE_AND number to bool",
+			Input:    "number & bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "BITWISE_XOR number to bool",
+			Input:    "number ^ bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "BITWISE_LSHIFT number to bool",
+			Input:    "number << bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "BITWISE_RSHIFT number to bool",
+			Input:    "number >> bool",
+			Expected: INVALID_MODIFIER_TYPES,
+		},
+	}
+
+	runEvaluationFailureTests(evaluationTests, test)
+}
+
+func TestLogicalOperatorTyping(test *testing.T) {
+
+	evaluationTests := []EvaluationFailureTest{
+		EvaluationFailureTest{
+
+			Name:     "AND number to number",
+			Input:    "number && number",
+			Expected: INVALID_LOGICALOP_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "OR number to number",
+			Input:    "number || number",
+			Expected: INVALID_LOGICALOP_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "AND string to string",
+			Input:    "string && string",
+			Expected: INVALID_LOGICALOP_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "OR string to string",
+			Input:    "string || string",
+			Expected: INVALID_LOGICALOP_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "AND number to string",
+			Input:    "number && string",
+			Expected: INVALID_LOGICALOP_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "OR number to string",
+			Input:    "number || string",
+			Expected: INVALID_LOGICALOP_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "AND bool to string",
+			Input:    "bool && string",
+			Expected: INVALID_LOGICALOP_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "OR string to bool",
+			Input:    "string || bool",
+			Expected: INVALID_LOGICALOP_TYPES,
+		},
+	}
+
+	runEvaluationFailureTests(evaluationTests, test)
+}
+
+/*
+	While there is type-safe transitions checked at parse-time, tested in the "parsing_test" and "parsingFailure_test" files,
+	we also need to make sure that we receive type mismatch errors during evaluation.
+*/
+func TestComparatorTyping(test *testing.T) {
+
+	evaluationTests := []EvaluationFailureTest{
+		EvaluationFailureTest{
+
+			Name:     "GT literal bool to literal bool",
+			Input:    "true > true",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "GT bool to bool",
+			Input:    "bool > bool",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "GTE bool to bool",
+			Input:    "bool >= bool",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "LT bool to bool",
+			Input:    "bool < bool",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "LTE bool to bool",
+			Input:    "bool <= bool",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+
+		EvaluationFailureTest{
+
+			Name:     "GT number to string",
+			Input:    "number > string",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "GTE number to string",
+			Input:    "number >= string",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "LT number to string",
+			Input:    "number < string",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "REQ number to string",
+			Input:    "number =~ string",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "REQ number to bool",
+			Input:    "number =~ bool",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "REQ bool to number",
+			Input:    "bool =~ number",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "REQ bool to string",
+			Input:    "bool =~ string",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "NREQ number to string",
+			Input:    "number !~ string",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "NREQ number to bool",
+			Input:    "number !~ bool",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "NREQ bool to number",
+			Input:    "bool !~ number",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "NREQ bool to string",
+			Input:    "bool !~ string",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "IN non-array numeric",
+			Input:    "1 in 2",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "IN non-array string",
+			Input:    "1 in 'foo'",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "IN non-array boolean",
+			Input:    "1 in true",
+			Expected: INVALID_COMPARATOR_TYPES,
+		},
+	}
+
+	runEvaluationFailureTests(evaluationTests, test)
+}
+
+func TestTernaryTyping(test *testing.T) {
+
+	evaluationTests := []EvaluationFailureTest{
+		EvaluationFailureTest{
+
+			Name:     "Ternary with number",
+			Input:    "10 ? true",
+			Expected: INVALID_TERNARY_TYPES,
+		},
+		EvaluationFailureTest{
+
+			Name:     "Ternary with string",
+			Input:    "'foo' ? true",
+			Expected: INVALID_TERNARY_TYPES,
+		},
+	}
+
+	runEvaluationFailureTests(evaluationTests, test)
+}
+
+func TestRegexParameterCompilation(test *testing.T) {
+
+	evaluationTests := []EvaluationFailureTest{
+		EvaluationFailureTest{
+
+			Name:  "Regex equality runtime parsing",
+			Input: "'foo' =~ foo",
+			Parameters: map[string]interface{}{
+				"foo": "[foo",
+			},
+			Expected: INVALID_REGEX,
+		},
+		EvaluationFailureTest{
+
+			Name:  "Regex inequality runtime parsing",
+			Input: "'foo' =~ foo",
+			Parameters: map[string]interface{}{
+				"foo": "[foo",
+			},
+			Expected: INVALID_REGEX,
+		},
+	}
+
+	runEvaluationFailureTests(evaluationTests, test)
+}
+
+func TestFunctionExecution(test *testing.T) {
+
+	evaluationTests := []EvaluationFailureTest{
+		EvaluationFailureTest{
+
+			Name:  "Function error bubbling",
+			Input: "error()",
+			Functions: map[string]ExpressionFunction{
+				"error": func(arguments ...interface{}) (interface{}, error) {
+					return nil, errors.New("Huge problems")
+				},
+			},
+			Expected: "Huge problems",
+		},
+	}
+
+	runEvaluationFailureTests(evaluationTests, test)
+}
+
+func TestInvalidParameterCalls(test *testing.T) {
+
+	evaluationTests := []EvaluationFailureTest{
+		EvaluationFailureTest{
+
+			Name:       "Missing parameter field reference",
+			Input:      "foo.NotExists",
+			Parameters: fooFailureParameters,
+			Expected:   INVALID_PARAMETER_CALL,
+		},
+		EvaluationFailureTest{
+
+			Name:       "Parameter method call on missing function",
+			Input:      "foo.NotExist()",
+			Parameters: fooFailureParameters,
+			Expected:   INVALID_PARAMETER_CALL,
+		},
+		EvaluationFailureTest{
+
+			Name:       "Nested missing parameter field reference",
+			Input:      "foo.Nested.NotExists",
+			Parameters: fooFailureParameters,
+			Expected:   INVALID_PARAMETER_CALL,
+		},
+		EvaluationFailureTest{
+
+			Name:       "Parameter method call returns error",
+			Input:      "foo.AlwaysFail()",
+			Parameters: fooFailureParameters,
+			Expected:   "function should always fail",
+		},
+		EvaluationFailureTest{
+
+			Name:       "Too few arguments to parameter call",
+			Input:      "foo.FuncArgStr()",
+			Parameters: fooFailureParameters,
+			Expected:   TOO_FEW_ARGS,
+		},
+		EvaluationFailureTest{
+
+			Name:       "Too many arguments to parameter call",
+			Input:      "foo.FuncArgStr('foo', 'bar', 15)",
+			Parameters: fooFailureParameters,
+			Expected:   TOO_MANY_ARGS,
+		},
+		EvaluationFailureTest{
+
+			Name:       "Mismatched parameters",
+			Input:      "foo.FuncArgStr(5)",
+			Parameters: fooFailureParameters,
+			Expected:   MISMATCHED_PARAMETERS,
+		},
+	}
+
+	runEvaluationFailureTests(evaluationTests, test)
+}
+
+func runEvaluationFailureTests(evaluationTests []EvaluationFailureTest, test *testing.T) {
+
+	var expression *EvaluableExpression
+	var err error
+
+	fmt.Printf("Running %d negative parsing test cases...\n", len(evaluationTests))
+
+	for _, testCase := range evaluationTests {
+
+		if len(testCase.Functions) > 0 {
+			expression, err = NewEvaluableExpressionWithFunctions(testCase.Input, testCase.Functions)
+		} else {
+			expression, err = NewEvaluableExpression(testCase.Input)
+		}
+
+		if err != nil {
+
+			test.Logf("Test '%s' failed", testCase.Name)
+			test.Logf("Expected evaluation error, but got parsing error: '%s'", err)
+			test.Fail()
+			continue
+		}
+
+		if testCase.Parameters == nil {
+			testCase.Parameters = EVALUATION_FAILURE_PARAMETERS
+		}
+
+		_, err = expression.Evaluate(testCase.Parameters)
+
+		if err == nil {
+
+			test.Logf("Test '%s' failed", testCase.Name)
+			test.Logf("Expected error, received none.")
+			test.Fail()
+			continue
+		}
+
+		if !strings.Contains(err.Error(), testCase.Expected) {
+
+			test.Logf("Test '%s' failed", testCase.Name)
+			test.Logf("Got error: '%s', expected '%s'", err.Error(), testCase.Expected)
+			test.Fail()
+			continue
+		}
+	}
+}

data_quality/src/figure/govaluate/evaluationStage.go

@@ -0,0 +1,516 @@
+package govaluate
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+	"regexp"
+	"strings"
+)
+
+const (
+	logicalErrorFormat    string = "Value '%v' cannot be used with the logical operator '%v', it is not a bool"
+	modifierErrorFormat   string = "Value '%v' cannot be used with the modifier '%v', it is not a number"
+	comparatorErrorFormat string = "Value '%v' cannot be used with the comparator '%v', it is not a number"
+	ternaryErrorFormat    string = "Value '%v' cannot be used with the ternary operator '%v', it is not a bool"
+	prefixErrorFormat     string = "Value '%v' cannot be used with the prefix '%v'"
+)
+
+type evaluationOperator func(left interface{}, right interface{}, parameters Parameters) (interface{}, error)
+type stageTypeCheck func(value interface{}) bool
+type stageCombinedTypeCheck func(left interface{}, right interface{}) bool
+
+type evaluationStage struct {
+	symbol OperatorSymbol
+
+	leftStage, rightStage *evaluationStage
+
+	// the operation that will be used to evaluate this stage (such as adding [left] to [right] and return the result)
+	operator evaluationOperator
+
+	// ensures that both left and right values are appropriate for this stage. Returns an error if they aren't operable.
+	leftTypeCheck  stageTypeCheck
+	rightTypeCheck stageTypeCheck
+
+	// if specified, will override whatever is used in "leftTypeCheck" and "rightTypeCheck".
+	// primarily used for specific operators that don't care which side a given type is on, but still requires one side to be of a given type
+	// (like string concat)
+	typeCheck stageCombinedTypeCheck
+
+	// regardless of which type check is used, this string format will be used as the error message for type errors
+	typeErrorFormat string
+}
+
+var (
+	_true  = interface{}(true)
+	_false = interface{}(false)
+)
+
+func (this *evaluationStage) swapWith(other *evaluationStage) {
+
+	temp := *other
+	other.setToNonStage(*this)
+	this.setToNonStage(temp)
+}
+
+func (this *evaluationStage) setToNonStage(other evaluationStage) {
+
+	this.symbol = other.symbol
+	this.operator = other.operator
+	this.leftTypeCheck = other.leftTypeCheck
+	this.rightTypeCheck = other.rightTypeCheck
+	this.typeCheck = other.typeCheck
+	this.typeErrorFormat = other.typeErrorFormat
+}
+
+func (this *evaluationStage) isShortCircuitable() bool {
+
+	switch this.symbol {
+	case AND:
+		fallthrough
+	case OR:
+		fallthrough
+	case TERNARY_TRUE:
+		fallthrough
+	case TERNARY_FALSE:
+		fallthrough
+	case COALESCE:
+		return true
+	}
+
+	return false
+}
+
+func noopStageRight(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return right, nil
+}
+
+func addStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+
+	// string concat if either are strings
+	if isString(left) || isString(right) {
+		return fmt.Sprintf("%v%v", left, right), nil
+	}
+
+	return left.(float64) + right.(float64), nil
+}
+func subtractStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return left.(float64) - right.(float64), nil
+}
+func multiplyStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return left.(float64) * right.(float64), nil
+}
+func divideStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return left.(float64) / right.(float64), nil
+}
+func exponentStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return math.Pow(left.(float64), right.(float64)), nil
+}
+func modulusStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return math.Mod(left.(float64), right.(float64)), nil
+}
+func gteStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	if isString(left) && isString(right) {
+		return boolIface(left.(string) >= right.(string)), nil
+	}
+	return boolIface(left.(float64) >= right.(float64)), nil
+}
+func gtStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	if isString(left) && isString(right) {
+		return boolIface(left.(string) > right.(string)), nil
+	}
+	return boolIface(left.(float64) > right.(float64)), nil
+}
+func lteStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	if isString(left) && isString(right) {
+		return boolIface(left.(string) <= right.(string)), nil
+	}
+	return boolIface(left.(float64) <= right.(float64)), nil
+}
+func ltStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	if isString(left) && isString(right) {
+		return boolIface(left.(string) < right.(string)), nil
+	}
+	return boolIface(left.(float64) < right.(float64)), nil
+}
+func equalStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return boolIface(reflect.DeepEqual(left, right)), nil
+}
+func notEqualStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return boolIface(!reflect.DeepEqual(left, right)), nil
+}
+func andStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return boolIface(left.(bool) && right.(bool)), nil
+}
+func orStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return boolIface(left.(bool) || right.(bool)), nil
+}
+func negateStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return -right.(float64), nil
+}
+func invertStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return boolIface(!right.(bool)), nil
+}
+func bitwiseNotStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return float64(^int64(right.(float64))), nil
+}
+func ternaryIfStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	if left.(bool) {
+		return right, nil
+	}
+	return nil, nil
+}
+func ternaryElseStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	if left != nil {
+		return left, nil
+	}
+	return right, nil
+}
+
+func regexStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+
+	var pattern *regexp.Regexp
+	var err error
+
+	switch right.(type) {
+	case string:
+		pattern, err = regexp.Compile(right.(string))
+		if err != nil {
+			return nil, errors.New(fmt.Sprintf("Unable to compile regexp pattern '%v': %v", right, err))
+		}
+	case *regexp.Regexp:
+		pattern = right.(*regexp.Regexp)
+	}
+
+	return pattern.Match([]byte(left.(string))), nil
+}
+
+func notRegexStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+
+	ret, err := regexStage(left, right, parameters)
+	if err != nil {
+		return nil, err
+	}
+
+	return !(ret.(bool)), nil
+}
+
+func bitwiseOrStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return float64(int64(left.(float64)) | int64(right.(float64))), nil
+}
+func bitwiseAndStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return float64(int64(left.(float64)) & int64(right.(float64))), nil
+}
+func bitwiseXORStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return float64(int64(left.(float64)) ^ int64(right.(float64))), nil
+}
+func leftShiftStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return float64(uint64(left.(float64)) << uint64(right.(float64))), nil
+}
+func rightShiftStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+	return float64(uint64(left.(float64)) >> uint64(right.(float64))), nil
+}
+
+func makeParameterStage(parameterName string) evaluationOperator {
+
+	return func(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+		value, err := parameters.Get(parameterName)
+		if err != nil {
+			return nil, err
+		}
+
+		return value, nil
+	}
+}
+
+func makeLiteralStage(literal interface{}) evaluationOperator {
+	return func(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+		return literal, nil
+	}
+}
+
+func makeFunctionStage(function ExpressionFunction) evaluationOperator {
+
+	return func(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+
+		if right == nil {
+			return function()
+		}
+
+		switch right.(type) {
+		case []interface{}:
+			return function(right.([]interface{})...)
+		default:
+			return function(right)
+		}
+	}
+}
+
+func typeConvertParam(p reflect.Value, t reflect.Type) (ret reflect.Value, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			errorMsg := fmt.Sprintf("Argument type conversion failed: failed to convert '%s' to '%s'", p.Kind().String(), t.Kind().String())
+			err = errors.New(errorMsg)
+			ret = p
+		}
+	}()
+
+	return p.Convert(t), nil
+}
+
+func typeConvertParams(method reflect.Value, params []reflect.Value) ([]reflect.Value, error) {
+
+	methodType := method.Type()
+	numIn := methodType.NumIn()
+	numParams := len(params)
+
+	if numIn != numParams {
+		if numIn > numParams {
+			return nil, fmt.Errorf("Too few arguments to parameter call: got %d arguments, expected %d", len(params), numIn)
+		}
+		return nil, fmt.Errorf("Too many arguments to parameter call: got %d arguments, expected %d", len(params), numIn)
+	}
+
+	for i := 0; i < numIn; i++ {
+		t := methodType.In(i)
+		p := params[i]
+		pt := p.Type()
+
+		if t.Kind() != pt.Kind() {
+			np, err := typeConvertParam(p, t)
+			if err != nil {
+				return nil, err
+			}
+			params[i] = np
+		}
+	}
+
+	return params, nil
+}
+
+func makeAccessorStage(pair []string) evaluationOperator {
+
+	reconstructed := strings.Join(pair, ".")
+
+	return func(left interface{}, right interface{}, parameters Parameters) (ret interface{}, err error) {
+
+		var params []reflect.Value
+
+		value, err := parameters.Get(pair[0])
+		if err != nil {
+			return nil, err
+		}
+
+		// while this library generally tries to handle panic-inducing cases on its own,
+		// accessors are a sticky case which have a lot of possible ways to fail.
+		// therefore every call to an accessor sets up a defer that tries to recover from panics, converting them to errors.
+		defer func() {
+			if r := recover(); r != nil {
+				errorMsg := fmt.Sprintf("Failed to access '%s': %v", reconstructed, r.(string))
+				err = errors.New(errorMsg)
+				ret = nil
+			}
+		}()
+
+		for i := 1; i < len(pair); i++ {
+
+			coreValue := reflect.ValueOf(value)
+
+			var corePtrVal reflect.Value
+
+			// if this is a pointer, resolve it.
+			if coreValue.Kind() == reflect.Ptr {
+				corePtrVal = coreValue
+				coreValue = coreValue.Elem()
+			}
+
+			if coreValue.Kind() != reflect.Struct {
+				return nil, errors.New("Unable to access '" + pair[i] + "', '" + pair[i-1] + "' is not a struct")
+			}
+
+			field := coreValue.FieldByName(pair[i])
+			if field != (reflect.Value{}) {
+				value = field.Interface()
+				continue
+			}
+
+			method := coreValue.MethodByName(pair[i])
+			if method == (reflect.Value{}) {
+				if corePtrVal.IsValid() {
+					method = corePtrVal.MethodByName(pair[i])
+				}
+				if method == (reflect.Value{}) {
+					return nil, errors.New("No method or field '" + pair[i] + "' present on parameter '" + pair[i-1] + "'")
+				}
+			}
+
+			switch right.(type) {
+			case []interface{}:
+
+				givenParams := right.([]interface{})
+				params = make([]reflect.Value, len(givenParams))
+				for idx, _ := range givenParams {
+					params[idx] = reflect.ValueOf(givenParams[idx])
+				}
+
+			default:
+
+				if right == nil {
+					params = []reflect.Value{}
+					break
+				}
+
+				params = []reflect.Value{reflect.ValueOf(right.(interface{}))}
+			}
+
+			params, err = typeConvertParams(method, params)
+
+			if err != nil {
+				return nil, errors.New("Method call failed - '" + pair[0] + "." + pair[1] + "': " + err.Error())
+			}
+
+			returned := method.Call(params)
+			retLength := len(returned)
+
+			if retLength == 0 {
+				return nil, errors.New("Method call '" + pair[i-1] + "." + pair[i] + "' did not return any values.")
+			}
+
+			if retLength == 1 {
+
+				value = returned[0].Interface()
+				continue
+			}
+
+			if retLength == 2 {
+
+				errIface := returned[1].Interface()
+				err, validType := errIface.(error)
+
+				if validType && errIface != nil {
+					return returned[0].Interface(), err
+				}
+
+				value = returned[0].Interface()
+				continue
+			}
+
+			return nil, errors.New("Method call '" + pair[0] + "." + pair[1] + "' did not return either one value, or a value and an error. Cannot interpret meaning.")
+		}
+
+		value = castToFloat64(value)
+		return value, nil
+	}
+}
+
+func separatorStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+
+	var ret []interface{}
+
+	switch left.(type) {
+	case []interface{}:
+		ret = append(left.([]interface{}), right)
+	default:
+		ret = []interface{}{left, right}
+	}
+
+	return ret, nil
+}
+
+func inStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
+
+	for _, value := range right.([]interface{}) {
+		if left == value {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+//
+
+func isString(value interface{}) bool {
+
+	switch value.(type) {
+	case string:
+		return true
+	}
+	return false
+}
+
+func isRegexOrString(value interface{}) bool {
+
+	switch value.(type) {
+	case string:
+		return true
+	case *regexp.Regexp:
+		return true
+	}
+	return false
+}
+
+func isBool(value interface{}) bool {
+	switch value.(type) {
+	case bool:
+		return true
+	}
+	return false
+}
+
+func isFloat64(value interface{}) bool {
+	switch value.(type) {
+	case float64:
+		return true
+	}
+	return false
+}
+
+/*
+	Addition usually means between numbers, but can also mean string concat.
+	String concat needs one (or both) of the sides to be a string.
+*/
+func additionTypeCheck(left interface{}, right interface{}) bool {
+
+	if isFloat64(left) && isFloat64(right) {
+		return true
+	}
+	if !isString(left) && !isString(right) {
+		return false
+	}
+	return true
+}
+
+/*
+	Comparison can either be between numbers, or lexicographic between two strings,
+	but never between the two.
+*/
+func comparatorTypeCheck(left interface{}, right interface{}) bool {
+
+	if isFloat64(left) && isFloat64(right) {
+		return true
+	}
+	if isString(left) && isString(right) {
+		return true
+	}
+	return false
+}
+
+func isArray(value interface{}) bool {
+	switch value.(type) {
+	case []interface{}:
+		return true
+	}
+	return false
+}
+
+/*
+	Converting a boolean to an interface{} requires an allocation.
+	We can use interned bools to avoid this cost.
+*/
+func boolIface(b bool) interface{} {
+	if b {
+		return _true
+	}
+	return _false
+}

data_quality/src/figure/govaluate/evaluation_test.go

@@ -0,0 +1,1516 @@
+package govaluate
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"testing"
+	"time"
+)
+
+/*
+	Represents a test of expression evaluation
+*/
+type EvaluationTest struct {
+	Name       string
+	Input      string
+	Functions  map[string]ExpressionFunction
+	Parameters []EvaluationParameter
+	Expected   interface{}
+}
+
+type EvaluationParameter struct {
+	Name  string
+	Value interface{}
+}
+
+func TestNoParameterEvaluation(test *testing.T) {
+
+	evaluationTests := []EvaluationTest{
+
+		EvaluationTest{
+
+			Name:     "Single PLUS",
+			Input:    "51 + 49",
+			Expected: 100.0,
+		},
+		EvaluationTest{
+
+			Name:     "Single MINUS",
+			Input:    "100 - 51",
+			Expected: 49.0,
+		},
+		EvaluationTest{
+
+			Name:     "Single BITWISE AND",
+			Input:    "100 & 50",
+			Expected: 32.0,
+		},
+		EvaluationTest{
+
+			Name:     "Single BITWISE OR",
+			Input:    "100 | 50",
+			Expected: 118.0,
+		},
+		EvaluationTest{
+
+			Name:     "Single BITWISE XOR",
+			Input:    "100 ^ 50",
+			Expected: 86.0,
+		},
+		EvaluationTest{
+
+			Name:     "Single shift left",
+			Input:    "2 << 1",
+			Expected: 4.0,
+		},
+		EvaluationTest{
+
+			Name:     "Single shift right",
+			Input:    "2 >> 1",
+			Expected: 1.0,
+		},
+		EvaluationTest{
+
+			Name:     "Single BITWISE NOT",
+			Input:    "~10",
+			Expected: -11.0,
+		},
+		EvaluationTest{
+
+			Name:     "Single MULTIPLY",
+			Input:    "5 * 20",
+			Expected: 100.0,
+		},
+		EvaluationTest{
+
+			Name:     "Single DIVIDE",
+			Input:    "100 / 20",
+			Expected: 5.0,
+		},
+		EvaluationTest{
+
+			Name:     "Single even MODULUS",
+			Input:    "100 % 2",
+			Expected: 0.0,
+		},
+		EvaluationTest{
+
+			Name:     "Single odd MODULUS",
+			Input:    "101 % 2",
+			Expected: 1.0,
+		},
+		EvaluationTest{
+
+			Name:     "Single EXPONENT",
+			Input:    "10 ** 2",
+			Expected: 100.0,
+		},
+		EvaluationTest{
+
+			Name:     "Compound PLUS",
+			Input:    "20 + 30 + 50",
+			Expected: 100.0,
+		},
+		EvaluationTest{
+
+			Name:     "Compound BITWISE AND",
+			Input:    "20 & 30 & 50",
+			Expected: 16.0,
+		},
+		EvaluationTest{
+
+			Name:     "Mutiple operators",
+			Input:    "20 * 5 - 49",
+			Expected: 51.0,
+		},
+		EvaluationTest{
+
+			Name:     "Parenthesis usage",
+			Input:    "100 - (5 * 10)",
+			Expected: 50.0,
+		},
+		EvaluationTest{
+
+			Name:     "Nested parentheses",
+			Input:    "50 + (5 * (15 - 5))",
+			Expected: 100.0,
+		},
+		EvaluationTest{
+
+			Name:     "Nested parentheses with bitwise",
+			Input:    "100 ^ (23 * (2 | 5))",
+			Expected: 197.0,
+		},
+		EvaluationTest{
+
+			Name:     "Logical OR operation of two clauses",
+			Input:    "(1 == 1) || (true == true)",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Logical AND operation of two clauses",
+			Input:    "(1 == 1) && (true == true)",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Implicit boolean",
+			Input:    "2 > 1",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Compound boolean",
+			Input:    "5 < 10 && 1 < 5",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Evaluated true && false operation (for issue #8)",
+			Input:    "1 > 10 && 11 > 10",
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:     "Evaluated true && false operation (for issue #8)",
+			Input:    "true == true && false == true",
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:     "Parenthesis boolean",
+			Input:    "10 < 50 && (1 != 2 && 1 > 0)",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Comparison of string constants",
+			Input:    "'foo' == 'foo'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "NEQ comparison of string constants",
+			Input:    "'foo' != 'bar'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "REQ comparison of string constants",
+			Input:    "'foobar' =~ 'oba'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "NREQ comparison of string constants",
+			Input:    "'foo' !~ 'bar'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Multiplicative/additive order",
+			Input:    "5 + 10 * 2",
+			Expected: 25.0,
+		},
+		EvaluationTest{
+
+			Name:     "Multiple constant multiplications",
+			Input:    "10 * 10 * 10",
+			Expected: 1000.0,
+		},
+		EvaluationTest{
+
+			Name:     "Multiple adds/multiplications",
+			Input:    "10 * 10 * 10 + 1 * 10 * 10",
+			Expected: 1100.0,
+		},
+		EvaluationTest{
+
+			Name:     "Modulus precedence",
+			Input:    "1 + 101 % 2 * 5",
+			Expected: 6.0,
+		},
+		EvaluationTest{
+
+			Name:     "Exponent precedence",
+			Input:    "1 + 5 ** 3 % 2 * 5",
+			Expected: 6.0,
+		},
+		EvaluationTest{
+
+			Name:     "Bit shift precedence",
+			Input:    "50 << 1 & 90",
+			Expected: 64.0,
+		},
+		EvaluationTest{
+
+			Name:     "Bit shift precedence",
+			Input:    "90 & 50 << 1",
+			Expected: 64.0,
+		},
+		EvaluationTest{
+
+			Name:     "Bit shift precedence amongst non-bitwise",
+			Input:    "90 + 50 << 1 * 5",
+			Expected: 4480.0,
+		},
+		EvaluationTest{
+			Name:     "Order of non-commutative same-precedence operators (additive)",
+			Input:    "1 - 2 - 4 - 8",
+			Expected: -13.0,
+		},
+		EvaluationTest{
+			Name:     "Order of non-commutative same-precedence operators (multiplicative)",
+			Input:    "1 * 4 / 2 * 8",
+			Expected: 16.0,
+		},
+		EvaluationTest{
+			Name:     "Null coalesce precedence",
+			Input:    "true ?? true ? 100 + 200 : 400",
+			Expected: 300.0,
+		},
+		EvaluationTest{
+
+			Name:     "Identical date equivalence",
+			Input:    "'2014-01-02 14:12:22' == '2014-01-02 14:12:22'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Positive date GT",
+			Input:    "'2014-01-02 14:12:22' > '2014-01-02 12:12:22'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Negative date GT",
+			Input:    "'2014-01-02 14:12:22' > '2014-01-02 16:12:22'",
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:     "Positive date GTE",
+			Input:    "'2014-01-02 14:12:22' >= '2014-01-02 12:12:22'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Negative date GTE",
+			Input:    "'2014-01-02 14:12:22' >= '2014-01-02 16:12:22'",
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:     "Positive date LT",
+			Input:    "'2014-01-02 14:12:22' < '2014-01-02 16:12:22'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Negative date LT",
+			Input:    "'2014-01-02 14:12:22' < '2014-01-02 11:12:22'",
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:     "Positive date LTE",
+			Input:    "'2014-01-02 09:12:22' <= '2014-01-02 12:12:22'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Negative date LTE",
+			Input:    "'2014-01-02 14:12:22' <= '2014-01-02 11:12:22'",
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:     "Sign prefix comparison",
+			Input:    "-1 < 0",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Lexicographic LT",
+			Input:    "'ab' < 'abc'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Lexicographic LTE",
+			Input:    "'ab' <= 'abc'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Lexicographic GT",
+			Input:    "'aba' > 'abc'",
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:     "Lexicographic GTE",
+			Input:    "'aba' >= 'abc'",
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:     "Boolean sign prefix comparison",
+			Input:    "!true == false",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Inversion of clause",
+			Input:    "!(10 < 0)",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Negation after modifier",
+			Input:    "10 * -10",
+			Expected: -100.0,
+		},
+		EvaluationTest{
+
+			Name:     "Ternary with single boolean",
+			Input:    "true ? 10",
+			Expected: 10.0,
+		},
+		EvaluationTest{
+
+			Name:     "Ternary nil with single boolean",
+			Input:    "false ? 10",
+			Expected: nil,
+		},
+		EvaluationTest{
+
+			Name:     "Ternary with comparator boolean",
+			Input:    "10 > 5 ? 35.50",
+			Expected: 35.50,
+		},
+		EvaluationTest{
+
+			Name:     "Ternary nil with comparator boolean",
+			Input:    "1 > 5 ? 35.50",
+			Expected: nil,
+		},
+		EvaluationTest{
+
+			Name:     "Ternary with parentheses",
+			Input:    "(5 * (15 - 5)) > 5 ? 35.50",
+			Expected: 35.50,
+		},
+		EvaluationTest{
+
+			Name:     "Ternary precedence",
+			Input:    "true ? 35.50 > 10",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Ternary-else",
+			Input:    "false ? 35.50 : 50",
+			Expected: 50.0,
+		},
+		EvaluationTest{
+
+			Name:     "Ternary-else inside clause",
+			Input:    "(false ? 5 : 35.50) > 10",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Ternary-else (true-case) inside clause",
+			Input:    "(true ? 1 : 5) < 10",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Ternary-else before comparator (negative case)",
+			Input:    "true ? 1 : 5 > 10",
+			Expected: 1.0,
+		},
+		EvaluationTest{
+
+			Name:     "Nested ternaries (#32)",
+			Input:    "(2 == 2) ? 1 : (true ? 2 : 3)",
+			Expected: 1.0,
+		},
+		EvaluationTest{
+
+			Name:     "Nested ternaries, right case (#32)",
+			Input:    "false ? 1 : (true ? 2 : 3)",
+			Expected: 2.0,
+		},
+		EvaluationTest{
+
+			Name:     "Doubly-nested ternaries (#32)",
+			Input:    "true ? (false ? 1 : (false ? 2 : 3)) : (false ? 4 : 5)",
+			Expected: 3.0,
+		},
+		EvaluationTest{
+
+			Name:     "String to string concat",
+			Input:    "'foo' + 'bar' == 'foobar'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "String to float64 concat",
+			Input:    "'foo' + 123 == 'foo123'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Float64 to string concat",
+			Input:    "123 + 'bar' == '123bar'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "String to date concat",
+			Input:    "'foo' + '02/05/1970' == 'foobar'",
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:     "String to bool concat",
+			Input:    "'foo' + true == 'footrue'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Bool to string concat",
+			Input:    "true + 'bar' == 'truebar'",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Null coalesce left",
+			Input:    "1 ?? 2",
+			Expected: 1.0,
+		},
+		EvaluationTest{
+
+			Name:     "Array membership literals",
+			Input:    "1 in (1, 2, 3)",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Array membership literal with inversion",
+			Input:    "!(1 in (1, 2, 3))",
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:     "Logical operator reordering (#30)",
+			Input:    "(true && true) || (true && false)",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Logical operator reordering without parens (#30)",
+			Input:    "true && true || true && false",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Logical operator reordering with multiple OR (#30)",
+			Input:    "false || true && true || false",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Left-side multiple consecutive (should be reordered) operators",
+			Input:    "(10 * 10 * 10) > 10",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Three-part non-paren logical op reordering (#44)",
+			Input:    "false && true || true",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Three-part non-paren logical op reordering (#44), second one",
+			Input:    "true || false && true",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Logical operator reordering without parens (#45)",
+			Input:    "true && true || false && false",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Single function",
+			Input: "foo()",
+			Functions: map[string]ExpressionFunction{
+				"foo": func(arguments ...interface{}) (interface{}, error) {
+					return true, nil
+				},
+			},
+
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Function with argument",
+			Input: "passthrough(1)",
+			Functions: map[string]ExpressionFunction{
+				"passthrough": func(arguments ...interface{}) (interface{}, error) {
+					return arguments[0], nil
+				},
+			},
+
+			Expected: 1.0,
+		},
+
+		EvaluationTest{
+
+			Name:  "Function with arguments",
+			Input: "passthrough(1, 2)",
+			Functions: map[string]ExpressionFunction{
+				"passthrough": func(arguments ...interface{}) (interface{}, error) {
+					return arguments[0].(float64) + arguments[1].(float64), nil
+				},
+			},
+
+			Expected: 3.0,
+		},
+		EvaluationTest{
+
+			Name:  "Nested function with precedence",
+			Input: "sum(1, sum(2, 3), 2 + 2, true ? 4 : 5)",
+			Functions: map[string]ExpressionFunction{
+				"sum": func(arguments ...interface{}) (interface{}, error) {
+
+					sum := 0.0
+					for _, v := range arguments {
+						sum += v.(float64)
+					}
+					return sum, nil
+				},
+			},
+
+			Expected: 14.0,
+		},
+		EvaluationTest{
+
+			Name:  "Empty function and modifier, compared",
+			Input: "numeric()-1 > 0",
+			Functions: map[string]ExpressionFunction{
+				"numeric": func(arguments ...interface{}) (interface{}, error) {
+					return 2.0, nil
+				},
+			},
+
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Empty function comparator",
+			Input: "numeric() > 0",
+			Functions: map[string]ExpressionFunction{
+				"numeric": func(arguments ...interface{}) (interface{}, error) {
+					return 2.0, nil
+				},
+			},
+
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Empty function logical operator",
+			Input: "success() && !false",
+			Functions: map[string]ExpressionFunction{
+				"success": func(arguments ...interface{}) (interface{}, error) {
+					return true, nil
+				},
+			},
+
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Empty function ternary",
+			Input: "nope() ? 1 : 2.0",
+			Functions: map[string]ExpressionFunction{
+				"nope": func(arguments ...interface{}) (interface{}, error) {
+					return false, nil
+				},
+			},
+
+			Expected: 2.0,
+		},
+		EvaluationTest{
+
+			Name:  "Empty function null coalesce",
+			Input: "null() ?? 2",
+			Functions: map[string]ExpressionFunction{
+				"null": func(arguments ...interface{}) (interface{}, error) {
+					return nil, nil
+				},
+			},
+
+			Expected: 2.0,
+		},
+		EvaluationTest{
+
+			Name:  "Empty function with prefix",
+			Input: "-ten()",
+			Functions: map[string]ExpressionFunction{
+				"ten": func(arguments ...interface{}) (interface{}, error) {
+					return 10.0, nil
+				},
+			},
+
+			Expected: -10.0,
+		},
+		EvaluationTest{
+
+			Name:  "Empty function as part of chain",
+			Input: "10 - numeric() - 2",
+			Functions: map[string]ExpressionFunction{
+				"numeric": func(arguments ...interface{}) (interface{}, error) {
+					return 5.0, nil
+				},
+			},
+
+			Expected: 3.0,
+		},
+		EvaluationTest{
+
+			Name:  "Empty function near separator",
+			Input: "10 in (1, 2, 3, ten(), 8)",
+			Functions: map[string]ExpressionFunction{
+				"ten": func(arguments ...interface{}) (interface{}, error) {
+					return 10.0, nil
+				},
+			},
+
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Enclosed empty function with modifier and comparator (#28)",
+			Input: "(ten() - 1) > 3",
+			Functions: map[string]ExpressionFunction{
+				"ten": func(arguments ...interface{}) (interface{}, error) {
+					return 10.0, nil
+				},
+			},
+
+			Expected: true,
+		},
+		EvaluationTest{
+			
+			Name:  "Ternary/Java EL ambiguity",
+			Input: "false ? foo:length()",
+			Functions: map[string]ExpressionFunction{
+				"length": func(arguments ...interface{}) (interface{}, error) {
+					return 1.0, nil
+				},
+			},
+			Expected: 1.0,
+		},
+	}
+
+	runEvaluationTests(evaluationTests, test)
+}
+
+func TestParameterizedEvaluation(test *testing.T) {
+
+	evaluationTests := []EvaluationTest{
+
+		EvaluationTest{
+
+			Name:  "Single parameter modified by constant",
+			Input: "foo + 2",
+			Parameters: []EvaluationParameter{
+
+				EvaluationParameter{
+					Name:  "foo",
+					Value: 2.0,
+				},
+			},
+			Expected: 4.0,
+		},
+		EvaluationTest{
+
+			Name:  "Single parameter modified by variable",
+			Input: "foo * bar",
+			Parameters: []EvaluationParameter{
+
+				EvaluationParameter{
+					Name:  "foo",
+					Value: 5.0,
+				},
+				EvaluationParameter{
+					Name:  "bar",
+					Value: 2.0,
+				},
+			},
+			Expected: 10.0,
+		},
+		EvaluationTest{
+
+			Name:  "Multiple multiplications of the same parameter",
+			Input: "foo * foo * foo",
+			Parameters: []EvaluationParameter{
+
+				EvaluationParameter{
+					Name:  "foo",
+					Value: 10.0,
+				},
+			},
+			Expected: 1000.0,
+		},
+		EvaluationTest{
+
+			Name:  "Multiple additions of the same parameter",
+			Input: "foo + foo + foo",
+			Parameters: []EvaluationParameter{
+
+				EvaluationParameter{
+					Name:  "foo",
+					Value: 10.0,
+				},
+			},
+			Expected: 30.0,
+		},
+		EvaluationTest{
+
+			Name:  "Parameter name sensitivity",
+			Input: "foo + FoO + FOO",
+			Parameters: []EvaluationParameter{
+
+				EvaluationParameter{
+					Name:  "foo",
+					Value: 8.0,
+				},
+				EvaluationParameter{
+					Name:  "FoO",
+					Value: 4.0,
+				},
+				EvaluationParameter{
+					Name:  "FOO",
+					Value: 2.0,
+				},
+			},
+			Expected: 14.0,
+		},
+		EvaluationTest{
+
+			Name:  "Sign prefix comparison against prefixed variable",
+			Input: "-1 < -foo",
+			Parameters: []EvaluationParameter{
+
+				EvaluationParameter{
+					Name:  "foo",
+					Value: -8.0,
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Fixed-point parameter",
+			Input: "foo > 1",
+			Parameters: []EvaluationParameter{
+
+				EvaluationParameter{
+					Name:  "foo",
+					Value: 2,
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Modifier after closing clause",
+			Input:    "(2 + 2) + 2 == 6",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:     "Comparator after closing clause",
+			Input:    "(2 + 2) >= 4",
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Two-boolean logical operation (for issue #8)",
+			Input: "(foo == true) || (bar == true)",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: true,
+				},
+				EvaluationParameter{
+					Name:  "bar",
+					Value: false,
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Two-variable integer logical operation (for issue #8)",
+			Input: "foo > 10 && bar > 10",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: 1,
+				},
+				EvaluationParameter{
+					Name:  "bar",
+					Value: 11,
+				},
+			},
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:  "Regex against right-hand parameter",
+			Input: "'foobar' =~ foo",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: "obar",
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Not-regex against right-hand parameter",
+			Input: "'foobar' !~ foo",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: "baz",
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Regex against two parameters",
+			Input: "foo =~ bar",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: "foobar",
+				},
+				EvaluationParameter{
+					Name:  "bar",
+					Value: "oba",
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Not-regex against two parameters",
+			Input: "foo !~ bar",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: "foobar",
+				},
+				EvaluationParameter{
+					Name:  "bar",
+					Value: "baz",
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Pre-compiled regex",
+			Input: "foo =~ bar",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: "foobar",
+				},
+				EvaluationParameter{
+					Name:  "bar",
+					Value: regexp.MustCompile("[fF][oO]+"),
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Pre-compiled not-regex",
+			Input: "foo !~ bar",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: "foobar",
+				},
+				EvaluationParameter{
+					Name:  "bar",
+					Value: regexp.MustCompile("[fF][oO]+"),
+				},
+			},
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:  "Single boolean parameter",
+			Input: "commission ? 10",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "commission",
+					Value: true,
+				},
+			},
+			Expected: 10.0,
+		},
+		EvaluationTest{
+
+			Name:  "True comparator with a parameter",
+			Input: "partner == 'amazon' ? 10",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "partner",
+					Value: "amazon",
+				},
+			},
+			Expected: 10.0,
+		},
+		EvaluationTest{
+
+			Name:  "False comparator with a parameter",
+			Input: "partner == 'amazon' ? 10",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "partner",
+					Value: "ebay",
+				},
+			},
+			Expected: nil,
+		},
+		EvaluationTest{
+
+			Name:  "True comparator with multiple parameters",
+			Input: "theft && period == 24 ? 60",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "theft",
+					Value: true,
+				},
+				EvaluationParameter{
+					Name:  "period",
+					Value: 24,
+				},
+			},
+			Expected: 60.0,
+		},
+		EvaluationTest{
+
+			Name:  "False comparator with multiple parameters",
+			Input: "theft && period == 24 ? 60",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "theft",
+					Value: false,
+				},
+				EvaluationParameter{
+					Name:  "period",
+					Value: 24,
+				},
+			},
+			Expected: nil,
+		},
+		EvaluationTest{
+
+			Name:  "String concat with single string parameter",
+			Input: "foo + 'bar'",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: "baz",
+				},
+			},
+			Expected: "bazbar",
+		},
+		EvaluationTest{
+
+			Name:  "String concat with multiple string parameter",
+			Input: "foo + bar",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: "baz",
+				},
+				EvaluationParameter{
+					Name:  "bar",
+					Value: "quux",
+				},
+			},
+			Expected: "bazquux",
+		},
+		EvaluationTest{
+
+			Name:  "String concat with float parameter",
+			Input: "foo + bar",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: "baz",
+				},
+				EvaluationParameter{
+					Name:  "bar",
+					Value: 123.0,
+				},
+			},
+			Expected: "baz123",
+		},
+		EvaluationTest{
+
+			Name:  "Mixed multiple string concat",
+			Input: "foo + 123 + 'bar' + true",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: "baz",
+				},
+			},
+			Expected: "baz123bartrue",
+		},
+		EvaluationTest{
+
+			Name:  "Integer width spectrum",
+			Input: "uint8 + uint16 + uint32 + uint64 + int8 + int16 + int32 + int64",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "uint8",
+					Value: uint8(0),
+				},
+				EvaluationParameter{
+					Name:  "uint16",
+					Value: uint16(0),
+				},
+				EvaluationParameter{
+					Name:  "uint32",
+					Value: uint32(0),
+				},
+				EvaluationParameter{
+					Name:  "uint64",
+					Value: uint64(0),
+				},
+				EvaluationParameter{
+					Name:  "int8",
+					Value: int8(0),
+				},
+				EvaluationParameter{
+					Name:  "int16",
+					Value: int16(0),
+				},
+				EvaluationParameter{
+					Name:  "int32",
+					Value: int32(0),
+				},
+				EvaluationParameter{
+					Name:  "int64",
+					Value: int64(0),
+				},
+			},
+			Expected: 0.0,
+		},
+		EvaluationTest{
+
+			Name:  "Floats",
+			Input: "float32 + float64",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "float32",
+					Value: float32(0.0),
+				},
+				EvaluationParameter{
+					Name:  "float64",
+					Value: float64(0.0),
+				},
+			},
+			Expected: 0.0,
+		},
+		EvaluationTest{
+
+			Name:  "Null coalesce right",
+			Input: "foo ?? 1.0",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: nil,
+				},
+			},
+			Expected: 1.0,
+		},
+		EvaluationTest{
+
+			Name:  "Multiple comparator/logical operators (#30)",
+			Input: "(foo >= 2887057408 && foo <= 2887122943) || (foo >= 168100864 && foo <= 168118271)",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: 2887057409,
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Multiple comparator/logical operators, opposite order (#30)",
+			Input: "(foo >= 168100864 && foo <= 168118271) || (foo >= 2887057408 && foo <= 2887122943)",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: 2887057409,
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Multiple comparator/logical operators, small value (#30)",
+			Input: "(foo >= 2887057408 && foo <= 2887122943) || (foo >= 168100864 && foo <= 168118271)",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: 168100865,
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Multiple comparator/logical operators, small value, opposite order (#30)",
+			Input: "(foo >= 168100864 && foo <= 168118271) || (foo >= 2887057408 && foo <= 2887122943)",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "foo",
+					Value: 168100865,
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Incomparable array equality comparison",
+			Input: "arr == arr",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "arr",
+					Value: []int{0, 0, 0},
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Incomparable array not-equality comparison",
+			Input: "arr != arr",
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "arr",
+					Value: []int{0, 0, 0},
+				},
+			},
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:  "Mixed function and parameters",
+			Input: "sum(1.2, amount) + name",
+			Functions: map[string]ExpressionFunction{
+				"sum": func(arguments ...interface{}) (interface{}, error) {
+
+					sum := 0.0
+					for _, v := range arguments {
+						sum += v.(float64)
+					}
+					return sum, nil
+				},
+			},
+			Parameters: []EvaluationParameter{
+				EvaluationParameter{
+					Name:  "amount",
+					Value: .8,
+				},
+				EvaluationParameter{
+					Name:  "name",
+					Value: "awesome",
+				},
+			},
+
+			Expected: "2awesome",
+		},
+		EvaluationTest{
+
+			Name:  "Short-circuit OR",
+			Input: "true || fail()",
+			Functions: map[string]ExpressionFunction{
+				"fail": func(arguments ...interface{}) (interface{}, error) {
+					return nil, errors.New("Did not short-circuit")
+				},
+			},
+			Expected: true,
+		},
+		EvaluationTest{
+
+			Name:  "Short-circuit AND",
+			Input: "false && fail()",
+			Functions: map[string]ExpressionFunction{
+				"fail": func(arguments ...interface{}) (interface{}, error) {
+					return nil, errors.New("Did not short-circuit")
+				},
+			},
+			Expected: false,
+		},
+		EvaluationTest{
+
+			Name:  "Short-circuit ternary",
+			Input: "true ? 1 : fail()",
+			Functions: map[string]ExpressionFunction{
+				"fail": func(arguments ...interface{}) (interface{}, error) {
+					return nil, errors.New("Did not short-circuit")
+				},
+			},
+			Expected: 1.0,
+		},
+		EvaluationTest{
+
+			Name:  "Short-circuit coalesce",
+			Input: "'foo' ?? fail()",
+			Functions: map[string]ExpressionFunction{
+				"fail": func(arguments ...interface{}) (interface{}, error) {
+					return nil, errors.New("Did not short-circuit")
+				},
+			},
+			Expected: "foo",
+		},
+		EvaluationTest{
+
+			Name:       "Simple parameter call",
+			Input:      "foo.String",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   fooParameter.Value.(dummyParameter).String,
+		},
+		EvaluationTest{
+
+			Name:       "Simple parameter function call",
+			Input:      "foo.Func()",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   "funk",
+		},
+		EvaluationTest{
+
+			Name:       "Simple parameter call from pointer",
+			Input:      "fooptr.String",
+			Parameters: []EvaluationParameter{fooPtrParameter},
+			Expected:   fooParameter.Value.(dummyParameter).String,
+		},
+		EvaluationTest{
+
+			Name:       "Simple parameter function call from pointer",
+			Input:      "fooptr.Func()",
+			Parameters: []EvaluationParameter{fooPtrParameter},
+			Expected:   "funk",
+		},
+		EvaluationTest{
+
+			Name:       "Simple parameter function call from pointer",
+			Input:      "fooptr.Func3()",
+			Parameters: []EvaluationParameter{fooPtrParameter},
+			Expected:   "fronk",
+		},
+		EvaluationTest{
+
+			Name:       "Simple parameter call",
+			Input:      "foo.String == 'hi'",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   false,
+		},
+		EvaluationTest{
+
+			Name:       "Simple parameter call with modifier",
+			Input:      "foo.String + 'hi'",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   fooParameter.Value.(dummyParameter).String + "hi",
+		},
+		EvaluationTest{
+
+			Name:       "Simple parameter function call, two-arg return",
+			Input:      "foo.Func2()",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   "frink",
+		},
+		EvaluationTest{
+
+			Name:       "Parameter function call with all argument types",
+			Input:      "foo.TestArgs(\"hello\", 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1.0, 2.0, true)",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   "hello: 33",
+		},
+
+		EvaluationTest{
+
+			Name:       "Simple parameter function call, one arg",
+			Input:      "foo.FuncArgStr('boop')",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   "boop",
+		},
+		EvaluationTest{
+
+			Name:       "Simple parameter function call, one arg",
+			Input:      "foo.FuncArgStr('boop') + 'hi'",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   "boophi",
+		},
+		EvaluationTest{
+
+			Name:       "Nested parameter function call",
+			Input:      "foo.Nested.Dunk('boop')",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   "boopdunk",
+		},
+		EvaluationTest{
+
+			Name:       "Nested parameter call",
+			Input:      "foo.Nested.Funk",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   "funkalicious",
+		},
+		EvaluationTest{
+
+			Name:       "Parameter call with + modifier",
+			Input:      "1 + foo.Int",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   102.0,
+		},
+		EvaluationTest{
+
+			Name:       "Parameter string call with + modifier",
+			Input:      "'woop' + (foo.String)",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   "woopstring!",
+		},
+		EvaluationTest{
+
+			Name:       "Parameter call with && operator",
+			Input:      "true && foo.BoolFalse",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   false,
+		},
+		EvaluationTest{
+
+			Name:       "Null coalesce nested parameter",
+			Input:      "foo.Nil ?? false",
+			Parameters: []EvaluationParameter{fooParameter},
+			Expected:   false,
+		},
+	}
+
+	runEvaluationTests(evaluationTests, test)
+}
+
+/*
+	Tests the behavior of a nil set of parameters.
+*/
+func TestNilParameters(test *testing.T) {
+
+	expression, _ := NewEvaluableExpression("true")
+	_, err := expression.Evaluate(nil)
+
+	if err != nil {
+		test.Fail()
+	}
+}
+
+/*
+	Tests functionality related to using functions with a struct method receiver.
+	Created to test #54.
+*/
+func TestStructFunctions(test *testing.T) {
+
+	parseFormat := "2006"
+	y2k, _ := time.Parse(parseFormat, "2000")
+	y2k1, _ := time.Parse(parseFormat, "2001")
+
+	functions := map[string]ExpressionFunction{
+		"func1": func(args ...interface{}) (interface{}, error) {
+			return float64(y2k.Year()), nil
+		},
+		"func2": func(args ...interface{}) (interface{}, error) {
+			return float64(y2k1.Year()), nil
+		},
+	}
+
+	exp, _ := NewEvaluableExpressionWithFunctions("func1() + func2()", functions)
+	result, _ := exp.Evaluate(nil)
+
+	if result != 4001.0 {
+		test.Logf("Function calling method did not return the right value. Got: %v, expected %d\n", result, 4001)
+		test.Fail()
+	}
+}
+
+func runEvaluationTests(evaluationTests []EvaluationTest, test *testing.T) {
+
+	var expression *EvaluableExpression
+	var result interface{}
+	var parameters map[string]interface{}
+	var err error
+
+	fmt.Printf("Running %d evaluation test cases...\n", len(evaluationTests))
+
+	// Run the test cases.
+	for _, evaluationTest := range evaluationTests {
+
+		if evaluationTest.Functions != nil {
+			expression, err = NewEvaluableExpressionWithFunctions(evaluationTest.Input, evaluationTest.Functions)
+		} else {
+			expression, err = NewEvaluableExpression(evaluationTest.Input)
+		}
+
+		if err != nil {
+
+			test.Logf("Test '%s' failed to parse: '%s'", evaluationTest.Name, err)
+			test.Fail()
+			continue
+		}
+
+		parameters = make(map[string]interface{}, 8)
+
+		for _, parameter := range evaluationTest.Parameters {
+			parameters[parameter.Name] = parameter.Value
+		}
+
+		result, err = expression.Evaluate(parameters)
+
+		if err != nil {
+
+			test.Logf("Test '%s' failed", evaluationTest.Name)
+			test.Logf("Encountered error: %s", err.Error())
+			test.Fail()
+			continue
+		}
+
+		if result != evaluationTest.Expected {
+
+			test.Logf("Test '%s' failed", evaluationTest.Name)
+			test.Logf("Evaluation result '%v' does not match expected: '%v'", result, evaluationTest.Expected)
+			test.Fail()
+		}
+	}
+}

data_quality/src/figure/govaluate/expressionFunctions.go

@@ -0,0 +1,8 @@
+package govaluate
+
+/*
+	Represents a function that can be called from within an expression.
+	This method must return an error if, for any reason, it is unable to produce exactly one unambiguous result.
+	An error returned will halt execution of the expression.
+*/
+type ExpressionFunction func(arguments ...interface{}) (interface{}, error)

data_quality/src/figure/govaluate/expressionOutputStream.go

@@ -0,0 +1,46 @@
+package govaluate
+
+import (
+	"bytes"
+)
+
+/*
+	Holds a series of "transactions" which represent each token as it is output by an outputter (such as ToSQLQuery()).
+	Some outputs (such as SQL) require a function call or non-c-like syntax to represent an expression.
+	To accomplish this, this struct keeps track of each translated token as it is output, and can return and rollback those transactions.
+*/
+type expressionOutputStream struct {
+	transactions []string
+}
+
+func (this *expressionOutputStream) add(transaction string) {
+	this.transactions = append(this.transactions, transaction)
+}
+
+func (this *expressionOutputStream) rollback() string {
+
+	index := len(this.transactions) - 1
+	ret := this.transactions[index]
+
+	this.transactions = this.transactions[:index]
+	return ret
+}
+
+func (this *expressionOutputStream) createString(delimiter string) string {
+
+	var retBuffer bytes.Buffer
+	var transaction string
+
+	penultimate := len(this.transactions) - 1
+
+	for i := 0; i < penultimate; i++ {
+
+		transaction = this.transactions[i]
+
+		retBuffer.WriteString(transaction)
+		retBuffer.WriteString(delimiter)
+	}
+	retBuffer.WriteString(this.transactions[penultimate])
+
+	return retBuffer.String()
+}

data_quality/src/figure/govaluate/lexerState.go

@@ -0,0 +1,373 @@
+package govaluate
+
+import (
+	"errors"
+	"fmt"
+)
+
+type lexerState struct {
+	isEOF          bool
+	isNullable     bool
+	kind           TokenKind
+	validNextKinds []TokenKind
+}
+
+// lexer states.
+// Constant for all purposes except compiler.
+var validLexerStates = []lexerState{
+
+	lexerState{
+		kind:       UNKNOWN,
+		isEOF:      false,
+		isNullable: true,
+		validNextKinds: []TokenKind{
+
+			PREFIX,
+			NUMERIC,
+			BOOLEAN,
+			VARIABLE,
+			PATTERN,
+			FUNCTION,
+			ACCESSOR,
+			STRING,
+			TIME,
+			CLAUSE,
+		},
+	},
+
+	lexerState{
+
+		kind:       CLAUSE,
+		isEOF:      false,
+		isNullable: true,
+		validNextKinds: []TokenKind{
+
+			PREFIX,
+			NUMERIC,
+			BOOLEAN,
+			VARIABLE,
+			PATTERN,
+			FUNCTION,
+			ACCESSOR,
+			STRING,
+			TIME,
+			CLAUSE,
+			CLAUSE_CLOSE,
+		},
+	},
+
+	lexerState{
+
+		kind:       CLAUSE_CLOSE,
+		isEOF:      true,
+		isNullable: true,
+		validNextKinds: []TokenKind{
+
+			COMPARATOR,
+			MODIFIER,
+			NUMERIC,
+			BOOLEAN,
+			VARIABLE,
+			STRING,
+			PATTERN,
+			TIME,
+			CLAUSE,
+			CLAUSE_CLOSE,
+			LOGICALOP,
+			TERNARY,
+			SEPARATOR,
+		},
+	},
+
+	lexerState{
+
+		kind:       NUMERIC,
+		isEOF:      true,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+
+			MODIFIER,
+			COMPARATOR,
+			LOGICALOP,
+			CLAUSE_CLOSE,
+			TERNARY,
+			SEPARATOR,
+		},
+	},
+	lexerState{
+
+		kind:       BOOLEAN,
+		isEOF:      true,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+
+			MODIFIER,
+			COMPARATOR,
+			LOGICALOP,
+			CLAUSE_CLOSE,
+			TERNARY,
+			SEPARATOR,
+		},
+	},
+	lexerState{
+
+		kind:       STRING,
+		isEOF:      true,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+
+			MODIFIER,
+			COMPARATOR,
+			LOGICALOP,
+			CLAUSE_CLOSE,
+			TERNARY,
+			SEPARATOR,
+		},
+	},
+	lexerState{
+
+		kind:       TIME,
+		isEOF:      true,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+
+			MODIFIER,
+			COMPARATOR,
+			LOGICALOP,
+			CLAUSE_CLOSE,
+			SEPARATOR,
+		},
+	},
+	lexerState{
+
+		kind:       PATTERN,
+		isEOF:      true,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+
+			MODIFIER,
+			COMPARATOR,
+			LOGICALOP,
+			CLAUSE_CLOSE,
+			SEPARATOR,
+		},
+	},
+	lexerState{
+
+		kind:       VARIABLE,
+		isEOF:      true,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+
+			MODIFIER,
+			COMPARATOR,
+			LOGICALOP,
+			CLAUSE_CLOSE,
+			TERNARY,
+			SEPARATOR,
+		},
+	},
+	lexerState{
+
+		kind:       MODIFIER,
+		isEOF:      false,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+
+			PREFIX,
+			NUMERIC,
+			VARIABLE,
+			FUNCTION,
+			ACCESSOR,
+			STRING,
+			BOOLEAN,
+			CLAUSE,
+			CLAUSE_CLOSE,
+		},
+	},
+	lexerState{
+
+		kind:       COMPARATOR,
+		isEOF:      false,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+
+			PREFIX,
+			NUMERIC,
+			BOOLEAN,
+			VARIABLE,
+			FUNCTION,
+			ACCESSOR,
+			STRING,
+			TIME,
+			CLAUSE,
+			CLAUSE_CLOSE,
+			PATTERN,
+		},
+	},
+	lexerState{
+
+		kind:       LOGICALOP,
+		isEOF:      false,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+
+			PREFIX,
+			NUMERIC,
+			BOOLEAN,
+			VARIABLE,
+			FUNCTION,
+			ACCESSOR,
+			STRING,
+			TIME,
+			CLAUSE,
+			CLAUSE_CLOSE,
+		},
+	},
+	lexerState{
+
+		kind:       PREFIX,
+		isEOF:      false,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+
+			NUMERIC,
+			BOOLEAN,
+			VARIABLE,
+			FUNCTION,
+			ACCESSOR,
+			CLAUSE,
+			CLAUSE_CLOSE,
+		},
+	},
+
+	lexerState{
+
+		kind:       TERNARY,
+		isEOF:      false,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+
+			PREFIX,
+			NUMERIC,
+			BOOLEAN,
+			STRING,
+			TIME,
+			VARIABLE,
+			FUNCTION,
+			ACCESSOR,
+			CLAUSE,
+			SEPARATOR,
+		},
+	},
+	lexerState{
+
+		kind:       FUNCTION,
+		isEOF:      false,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+			CLAUSE,
+		},
+	},
+	lexerState{
+
+		kind:       ACCESSOR,
+		isEOF:      true,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+			CLAUSE,
+			MODIFIER,
+			COMPARATOR,
+			LOGICALOP,
+			CLAUSE_CLOSE,
+			TERNARY,
+			SEPARATOR,
+		},
+	},
+	lexerState{
+
+		kind:       SEPARATOR,
+		isEOF:      false,
+		isNullable: true,
+		validNextKinds: []TokenKind{
+
+			PREFIX,
+			NUMERIC,
+			BOOLEAN,
+			STRING,
+			TIME,
+			VARIABLE,
+			FUNCTION,
+			ACCESSOR,
+			CLAUSE,
+		},
+	},
+}
+
+func (this lexerState) canTransitionTo(kind TokenKind) bool {
+
+	for _, validKind := range this.validNextKinds {
+
+		if validKind == kind {
+			return true
+		}
+	}
+
+	return false
+}
+
+func checkExpressionSyntax(tokens []ExpressionToken) error {
+
+	var state lexerState
+	var lastToken ExpressionToken
+	var err error
+
+	state = validLexerStates[0]
+
+	for _, token := range tokens {
+
+		if !state.canTransitionTo(token.Kind) {
+
+			// call out a specific error for tokens looking like they want to be functions.
+			if lastToken.Kind == VARIABLE && token.Kind == CLAUSE {
+				return errors.New("Undefined function " + lastToken.Value.(string))
+			}
+
+			firstStateName := fmt.Sprintf("%s [%v]", state.kind.String(), lastToken.Value)
+			nextStateName := fmt.Sprintf("%s [%v]", token.Kind.String(), token.Value)
+
+			return errors.New("Cannot transition token types from " + firstStateName + " to " + nextStateName)
+		}
+
+		state, err = getLexerStateForToken(token.Kind)
+		if err != nil {
+			return err
+		}
+
+		if !state.isNullable && token.Value == nil {
+
+			errorMsg := fmt.Sprintf("Token kind '%v' cannot have a nil value", token.Kind.String())
+			return errors.New(errorMsg)
+		}
+
+		lastToken = token
+	}
+
+	if !state.isEOF {
+		return errors.New("Unexpected end of expression")
+	}
+	return nil
+}
+
+func getLexerStateForToken(kind TokenKind) (lexerState, error) {
+
+	for _, possibleState := range validLexerStates {
+
+		if possibleState.kind == kind {
+			return possibleState, nil
+		}
+	}
+
+	errorMsg := fmt.Sprintf("No lexer state found for token kind '%v'\n", kind.String())
+	return validLexerStates[0], errors.New(errorMsg)
+}

data_quality/src/figure/govaluate/lexerStream.go

@@ -0,0 +1,39 @@
+package govaluate
+
+type lexerStream struct {
+	source   []rune
+	position int
+	length   int
+}
+
+func newLexerStream(source string) *lexerStream {
+
+	var ret *lexerStream
+	var runes []rune
+
+	for _, character := range source {
+		runes = append(runes, character)
+	}
+
+	ret = new(lexerStream)
+	ret.source = runes
+	ret.length = len(runes)
+	return ret
+}
+
+func (this *lexerStream) readCharacter() rune {
+
+	var character rune
+
+	character = this.source[this.position]
+	this.position += 1
+	return character
+}
+
+func (this *lexerStream) rewind(amount int) {
+	this.position -= amount
+}
+
+func (this lexerStream) canRead() bool {
+	return this.position < this.length
+}

data_quality/src/figure/govaluate/parameters.go

@@ -0,0 +1,32 @@
+package govaluate
+
+import (
+	"errors"
+)
+
+/*
+	Parameters is a collection of named parameters that can be used by an EvaluableExpression to retrieve parameters
+	when an expression tries to use them.
+*/
+type Parameters interface {
+
+	/*
+		Get gets the parameter of the given name, or an error if the parameter is unavailable.
+		Failure to find the given parameter should be indicated by returning an error.
+	*/
+	Get(name string) (interface{}, error)
+}
+
+type MapParameters map[string]interface{}
+
+func (p MapParameters) Get(name string) (interface{}, error) {
+
+	value, found := p[name]
+
+	if !found {
+		errorMessage := "No parameter '" + name + "' found."
+		return nil, errors.New(errorMessage)
+	}
+
+	return value, nil
+}

data_quality/src/figure/govaluate/parsing.go

@@ -0,0 +1,526 @@
+package govaluate
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+)
+
+func parseTokens(expression string, functions map[string]ExpressionFunction) ([]ExpressionToken, error) {
+
+	var ret []ExpressionToken
+	var token ExpressionToken
+	var stream *lexerStream
+	var state lexerState
+	var err error
+	var found bool
+
+	stream = newLexerStream(expression)
+	state = validLexerStates[0]
+
+	for stream.canRead() {
+
+		token, err, found = readToken(stream, state, functions)
+
+		if err != nil {
+			return ret, err
+		}
+
+		if !found {
+			break
+		}
+
+		state, err = getLexerStateForToken(token.Kind)
+		if err != nil {
+			return ret, err
+		}
+
+		// append this valid token
+		ret = append(ret, token)
+	}
+
+	err = checkBalance(ret)
+	if err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+func readToken(stream *lexerStream, state lexerState, functions map[string]ExpressionFunction) (ExpressionToken, error, bool) {
+
+	var function ExpressionFunction
+	var ret ExpressionToken
+	var tokenValue interface{}
+	var tokenTime time.Time
+	var tokenString string
+	var kind TokenKind
+	var character rune
+	var found bool
+	var completed bool
+	var err error
+
+	// numeric is 0-9, or . or 0x followed by digits
+	// string starts with '
+	// variable is alphanumeric, always starts with a letter
+	// bracket always means variable
+	// symbols are anything non-alphanumeric
+	// all others read into a buffer until they reach the end of the stream
+	for stream.canRead() {
+
+		character = stream.readCharacter()
+
+		if unicode.IsSpace(character) {
+			continue
+		}
+
+		kind = UNKNOWN
+
+		// numeric constant
+		if isNumeric(character) {
+
+			if stream.canRead() && character == '0' {
+				character = stream.readCharacter()
+
+				if stream.canRead() && character == 'x' {
+					tokenString, _ = readUntilFalse(stream, false, true, true, isHexDigit)
+					tokenValueInt, err := strconv.ParseUint(tokenString, 16, 64)
+
+					if err != nil {
+						errorMsg := fmt.Sprintf("Unable to parse hex value '%v' to uint64\n", tokenString)
+						return ExpressionToken{}, errors.New(errorMsg), false
+					}
+
+					kind = NUMERIC
+					tokenValue = float64(tokenValueInt)
+					break
+				} else {
+					stream.rewind(1)
+				}
+			}
+
+			tokenString = readTokenUntilFalse(stream, isNumeric)
+			tokenValue, err = strconv.ParseFloat(tokenString, 64)
+
+			if err != nil {
+				errorMsg := fmt.Sprintf("Unable to parse numeric value '%v' to float64\n", tokenString)
+				return ExpressionToken{}, errors.New(errorMsg), false
+			}
+			kind = NUMERIC
+			break
+		}
+
+		// comma, separator
+		if character == ',' {
+
+			tokenValue = ","
+			kind = SEPARATOR
+			break
+		}
+
+		// escaped variable
+		if character == '[' {
+
+			tokenValue, completed = readUntilFalse(stream, true, false, true, isNotClosingBracket)
+			kind = VARIABLE
+
+			if !completed {
+				return ExpressionToken{}, errors.New("Unclosed parameter bracket"), false
+			}
+
+			// above method normally rewinds us to the closing bracket, which we want to skip.
+			stream.rewind(-1)
+			break
+		}
+
+		// regular variable - or function?
+		if unicode.IsLetter(character) {
+
+			tokenString = readTokenUntilFalse(stream, isVariableName)
+
+			tokenValue = tokenString
+			kind = VARIABLE
+
+			// boolean?
+			if tokenValue == "true" {
+
+				kind = BOOLEAN
+				tokenValue = true
+			} else {
+
+				if tokenValue == "false" {
+
+					kind = BOOLEAN
+					tokenValue = false
+				}
+			}
+
+			// textual operator?
+			if tokenValue == "in" || tokenValue == "IN" {
+
+				// force lower case for consistency
+				tokenValue = "in"
+				kind = COMPARATOR
+			}
+
+			// function?
+			function, found = functions[tokenString]
+			if found {
+				kind = FUNCTION
+				tokenValue = function
+			}
+
+			// accessor?
+			accessorIndex := strings.Index(tokenString, ".")
+			if accessorIndex > 0 {
+
+				// check that it doesn't end with a hanging period
+				if tokenString[len(tokenString)-1] == '.' {
+					errorMsg := fmt.Sprintf("Hanging accessor on token '%s'", tokenString)
+					return ExpressionToken{}, errors.New(errorMsg), false
+				}
+
+				kind = ACCESSOR
+				splits := strings.Split(tokenString, ".")
+				tokenValue = splits
+
+				// check that none of them are unexported
+				for i := 1; i < len(splits); i++ {
+
+					firstCharacter := getFirstRune(splits[i])
+
+					if unicode.ToUpper(firstCharacter) != firstCharacter {
+						errorMsg := fmt.Sprintf("Unable to access unexported field '%s' in token '%s'", splits[i], tokenString)
+						return ExpressionToken{}, errors.New(errorMsg), false
+					}
+				}
+			}
+			break
+		}
+
+		if !isNotQuote(character) {
+			tokenValue, completed = readUntilFalse(stream, true, false, true, isNotQuote)
+
+			if !completed {
+				return ExpressionToken{}, errors.New("Unclosed string literal"), false
+			}
+
+			// advance the stream one position, since reading until false assumes the terminator is a real token
+			stream.rewind(-1)
+
+			// check to see if this can be parsed as a time.
+			tokenTime, found = tryParseTime(tokenValue.(string))
+			if found {
+				kind = TIME
+				tokenValue = tokenTime
+			} else {
+				kind = STRING
+			}
+			break
+		}
+
+		if character == '(' {
+			tokenValue = character
+			kind = CLAUSE
+			break
+		}
+
+		if character == ')' {
+			tokenValue = character
+			kind = CLAUSE_CLOSE
+			break
+		}
+
+		// must be a known symbol
+		tokenString = readTokenUntilFalse(stream, isNotAlphanumeric)
+		tokenValue = tokenString
+
+		// quick hack for the case where "-" can mean "prefixed negation" or "minus", which are used
+		// very differently.
+		if state.canTransitionTo(PREFIX) {
+			_, found = prefixSymbols[tokenString]
+			if found {
+
+				kind = PREFIX
+				break
+			}
+		}
+		_, found = modifierSymbols[tokenString]
+		if found {
+
+			kind = MODIFIER
+			break
+		}
+
+		_, found = logicalSymbols[tokenString]
+		if found {
+
+			kind = LOGICALOP
+			break
+		}
+
+		_, found = comparatorSymbols[tokenString]
+		if found {
+
+			kind = COMPARATOR
+			break
+		}
+
+		_, found = ternarySymbols[tokenString]
+		if found {
+
+			kind = TERNARY
+			break
+		}
+
+		errorMessage := fmt.Sprintf("Invalid token: '%s'", tokenString)
+		return ret, errors.New(errorMessage), false
+	}
+
+	ret.Kind = kind
+	ret.Value = tokenValue
+
+	return ret, nil, (kind != UNKNOWN)
+}
+
+func readTokenUntilFalse(stream *lexerStream, condition func(rune) bool) string {
+
+	var ret string
+
+	stream.rewind(1)
+	ret, _ = readUntilFalse(stream, false, true, true, condition)
+	return ret
+}
+
+/*
+	Returns the string that was read until the given [condition] was false, or whitespace was broken.
+	Returns false if the stream ended before whitespace was broken or condition was met.
+*/
+func readUntilFalse(stream *lexerStream, includeWhitespace bool, breakWhitespace bool, allowEscaping bool, condition func(rune) bool) (string, bool) {
+
+	var tokenBuffer bytes.Buffer
+	var character rune
+	var conditioned bool
+
+	conditioned = false
+
+	for stream.canRead() {
+
+		character = stream.readCharacter()
+
+		// Use backslashes to escape anything
+		if allowEscaping && character == '\\' {
+
+			character = stream.readCharacter()
+			tokenBuffer.WriteString(string(character))
+			continue
+		}
+
+		if unicode.IsSpace(character) {
+
+			if breakWhitespace && tokenBuffer.Len() > 0 {
+				conditioned = true
+				break
+			}
+			if !includeWhitespace {
+				continue
+			}
+		}
+
+		if condition(character) {
+			tokenBuffer.WriteString(string(character))
+		} else {
+			conditioned = true
+			stream.rewind(1)
+			break
+		}
+	}
+
+	return tokenBuffer.String(), conditioned
+}
+
+/*
+	Checks to see if any optimizations can be performed on the given [tokens], which form a complete, valid expression.
+	The returns slice will represent the optimized (or unmodified) list of tokens to use.
+*/
+func optimizeTokens(tokens []ExpressionToken) ([]ExpressionToken, error) {
+
+	var token ExpressionToken
+	var symbol OperatorSymbol
+	var err error
+	var index int
+
+	for index, token = range tokens {
+
+		// if we find a regex operator, and the right-hand value is a constant, precompile and replace with a pattern.
+		if token.Kind != COMPARATOR {
+			continue
+		}
+
+		symbol = comparatorSymbols[token.Value.(string)]
+		if symbol != REQ && symbol != NREQ {
+			continue
+		}
+
+		index++
+		token = tokens[index]
+		if token.Kind == STRING {
+
+			token.Kind = PATTERN
+			token.Value, err = regexp.Compile(token.Value.(string))
+
+			if err != nil {
+				return tokens, err
+			}
+
+			tokens[index] = token
+		}
+	}
+	return tokens, nil
+}
+
+/*
+	Checks the balance of tokens which have multiple parts, such as parenthesis.
+*/
+func checkBalance(tokens []ExpressionToken) error {
+
+	var stream *tokenStream
+	var token ExpressionToken
+	var parens int
+
+	stream = newTokenStream(tokens)
+
+	for stream.hasNext() {
+
+		token = stream.next()
+		if token.Kind == CLAUSE {
+			parens++
+			continue
+		}
+		if token.Kind == CLAUSE_CLOSE {
+			parens--
+			continue
+		}
+	}
+
+	if parens != 0 {
+		return errors.New("Unbalanced parenthesis")
+	}
+	return nil
+}
+
+func isDigit(character rune) bool {
+	return unicode.IsDigit(character)
+}
+
+func isHexDigit(character rune) bool {
+
+	character = unicode.ToLower(character)
+
+	return unicode.IsDigit(character) ||
+		character == 'a' ||
+		character == 'b' ||
+		character == 'c' ||
+		character == 'd' ||
+		character == 'e' ||
+		character == 'f'
+}
+
+func isNumeric(character rune) bool {
+
+	return unicode.IsDigit(character) || character == '.'
+}
+
+func isNotQuote(character rune) bool {
+
+	return character != '\'' && character != '"'
+}
+
+func isNotAlphanumeric(character rune) bool {
+
+	return !(unicode.IsDigit(character) ||
+		unicode.IsLetter(character) ||
+		character == '(' ||
+		character == ')' ||
+		character == '[' ||
+		character == ']' || // starting to feel like there needs to be an `isOperation` func (#59)
+		!isNotQuote(character))
+}
+
+func isVariableName(character rune) bool {
+
+	return unicode.IsLetter(character) ||
+		unicode.IsDigit(character) ||
+		character == '_' ||
+		character == '.'
+}
+
+func isNotClosingBracket(character rune) bool {
+
+	return character != ']'
+}
+
+/*
+	Attempts to parse the [candidate] as a Time.
+	Tries a series of standardized date formats, returns the Time if one applies,
+	otherwise returns false through the second return.
+*/
+func tryParseTime(candidate string) (time.Time, bool) {
+
+	var ret time.Time
+	var found bool
+
+	timeFormats := [...]string{
+		time.ANSIC,
+		time.UnixDate,
+		time.RubyDate,
+		time.Kitchen,
+		time.RFC3339,
+		time.RFC3339Nano,
+		"2006-01-02",                         // RFC 3339
+		"2006-01-02 15:04",                   // RFC 3339 with minutes
+		"2006-01-02 15:04:05",                // RFC 3339 with seconds
+		"2006-01-02 15:04:05-07:00",          // RFC 3339 with seconds and timezone
+		"2006-01-02T15Z0700",                 // ISO8601 with hour
+		"2006-01-02T15:04Z0700",              // ISO8601 with minutes
+		"2006-01-02T15:04:05Z0700",           // ISO8601 with seconds
+		"2006-01-02T15:04:05.999999999Z0700", // ISO8601 with nanoseconds
+	}
+
+	for _, format := range timeFormats {
+
+		ret, found = tryParseExactTime(candidate, format)
+		if found {
+			return ret, true
+		}
+	}
+
+	return time.Now(), false
+}
+
+func tryParseExactTime(candidate string, format string) (time.Time, bool) {
+
+	var ret time.Time
+	var err error
+
+	ret, err = time.ParseInLocation(format, candidate, time.Local)
+	if err != nil {
+		return time.Now(), false
+	}
+
+	return ret, true
+}
+
+func getFirstRune(candidate string) rune {
+
+	for _, character := range candidate {
+		return character
+	}
+
+	return 0
+}

data_quality/src/figure/govaluate/parsingFailure_test.go

@@ -0,0 +1,239 @@
+package govaluate
+
+import (
+	"fmt"
+	"regexp/syntax"
+	"strings"
+	"testing"
+)
+
+const (
+	UNEXPECTED_END           string = "Unexpected end of expression"
+	INVALID_TOKEN_TRANSITION        = "Cannot transition token types"
+	INVALID_TOKEN_KIND              = "Invalid token"
+	UNCLOSED_QUOTES                 = "Unclosed string literal"
+	UNCLOSED_BRACKETS               = "Unclosed parameter bracket"
+	UNBALANCED_PARENTHESIS          = "Unbalanced parenthesis"
+	INVALID_NUMERIC                 = "Unable to parse numeric value"
+	UNDEFINED_FUNCTION              = "Undefined function"
+	HANGING_ACCESSOR                = "Hanging accessor on token"
+	UNEXPORTED_ACCESSOR             = "Unable to access unexported"
+	INVALID_HEX                     = "Unable to parse hex value"
+)
+
+/*
+	Represents a test for parsing failures
+*/
+type ParsingFailureTest struct {
+	Name     string
+	Input    string
+	Expected string
+}
+
+func TestParsingFailure(test *testing.T) {
+
+	parsingTests := []ParsingFailureTest{
+
+		ParsingFailureTest{
+
+			Name:     "Invalid equality comparator",
+			Input:    "1 = 1",
+			Expected: INVALID_TOKEN_KIND,
+		},
+		ParsingFailureTest{
+
+			Name:     "Invalid equality comparator",
+			Input:    "1 === 1",
+			Expected: INVALID_TOKEN_KIND,
+		},
+		ParsingFailureTest{
+
+			Name:     "Too many characters for logical operator",
+			Input:    "true &&& false",
+			Expected: INVALID_TOKEN_KIND,
+		},
+		ParsingFailureTest{
+
+			Name:     "Too many characters for logical operator",
+			Input:    "true ||| false",
+			Expected: INVALID_TOKEN_KIND,
+		},
+		ParsingFailureTest{
+
+			Name:     "Premature end to expression, via modifier",
+			Input:    "10 > 5 +",
+			Expected: UNEXPECTED_END,
+		},
+		ParsingFailureTest{
+
+			Name:     "Premature end to expression, via comparator",
+			Input:    "10 + 5 >",
+			Expected: UNEXPECTED_END,
+		},
+		ParsingFailureTest{
+
+			Name:     "Premature end to expression, via logical operator",
+			Input:    "10 > 5 &&",
+			Expected: UNEXPECTED_END,
+		},
+		ParsingFailureTest{
+
+			Name:     "Premature end to expression, via ternary operator",
+			Input:    "true ?",
+			Expected: UNEXPECTED_END,
+		},
+		ParsingFailureTest{
+
+			Name:     "Hanging REQ",
+			Input:    "'wat' =~",
+			Expected: UNEXPECTED_END,
+		},
+		ParsingFailureTest{
+
+			Name:     "Invalid operator change to REQ",
+			Input:    " / =~",
+			Expected: INVALID_TOKEN_TRANSITION,
+		},
+		ParsingFailureTest{
+
+			Name:     "Invalid starting token, comparator",
+			Input:    "> 10",
+			Expected: INVALID_TOKEN_TRANSITION,
+		},
+		ParsingFailureTest{
+
+			Name:     "Invalid starting token, modifier",
+			Input:    "+ 5",
+			Expected: INVALID_TOKEN_TRANSITION,
+		},
+		ParsingFailureTest{
+
+			Name:     "Invalid starting token, logical operator",
+			Input:    "&& 5 < 10",
+			Expected: INVALID_TOKEN_TRANSITION,
+		},
+		ParsingFailureTest{
+
+			Name:     "Invalid NUMERIC transition",
+			Input:    "10 10",
+			Expected: INVALID_TOKEN_TRANSITION,
+		},
+		ParsingFailureTest{
+
+			Name:     "Invalid STRING transition",
+			Input:    "'foo' 'foo'",
+			Expected: INVALID_TOKEN_TRANSITION,
+		},
+		ParsingFailureTest{
+
+			Name:     "Invalid operator transition",
+			Input:    "10 > < 10",
+			Expected: INVALID_TOKEN_TRANSITION,
+		},
+		ParsingFailureTest{
+
+			Name:     "Starting with unbalanced parens",
+			Input:    " ) ( arg2",
+			Expected: INVALID_TOKEN_TRANSITION,
+		},
+		ParsingFailureTest{
+
+			Name:     "Unclosed bracket",
+			Input:    "[foo bar",
+			Expected: UNCLOSED_BRACKETS,
+		},
+		ParsingFailureTest{
+
+			Name:     "Unclosed quote",
+			Input:    "foo == 'responseTime",
+			Expected: UNCLOSED_QUOTES,
+		},
+		ParsingFailureTest{
+
+			Name:     "Constant regex pattern fail to compile",
+			Input:    "foo =~ '[abc'",
+			Expected: string(syntax.ErrMissingBracket),
+		},
+		ParsingFailureTest{
+
+			Name:     "Unbalanced parenthesis",
+			Input:    "10 > (1 + 50",
+			Expected: UNBALANCED_PARENTHESIS,
+		},
+		ParsingFailureTest{
+
+			Name:     "Multiple radix",
+			Input:    "127.0.0.1",
+			Expected: INVALID_NUMERIC,
+		},
+		ParsingFailureTest{
+
+			Name:     "Undefined function",
+			Input:    "foobar()",
+			Expected: UNDEFINED_FUNCTION,
+		},
+		ParsingFailureTest{
+
+			Name:     "Hanging accessor",
+			Input:    "foo.Bar.",
+			Expected: HANGING_ACCESSOR,
+		},
+		ParsingFailureTest{
+
+			// this is expected to change once there are structtags in place that allow aliasing of fields
+			Name:     "Unexported parameter access",
+			Input:    "foo.bar",
+			Expected: UNEXPORTED_ACCESSOR,
+		},
+		ParsingFailureTest{
+			Name:     "Incomplete Hex",
+			Input:    "0x",
+			Expected: INVALID_TOKEN_TRANSITION,
+		},
+		ParsingFailureTest{
+			Name:     "Invalid Hex literal",
+			Input:    "0x > 0",
+			Expected: INVALID_HEX,
+		},
+		ParsingFailureTest{
+			Name:     "Hex float (Unsupported)",
+			Input:    "0x1.1",
+			Expected: INVALID_TOKEN_TRANSITION,
+		},
+		ParsingFailureTest{
+			Name:     "Hex invalid letter",
+			Input:    "0x12g1",
+			Expected: INVALID_TOKEN_TRANSITION,
+		},
+	}
+
+	runParsingFailureTests(parsingTests, test)
+}
+
+func runParsingFailureTests(parsingTests []ParsingFailureTest, test *testing.T) {
+
+	var err error
+
+	fmt.Printf("Running %d parsing test cases...\n", len(parsingTests))
+
+	for _, testCase := range parsingTests {
+
+		_, err = NewEvaluableExpression(testCase.Input)
+
+		if err == nil {
+
+			test.Logf("Test '%s' failed", testCase.Name)
+			test.Logf("Expected a parsing error, found no error.")
+			test.Fail()
+			continue
+		}
+
+		if !strings.Contains(err.Error(), testCase.Expected) {
+
+			test.Logf("Test '%s' failed", testCase.Name)
+			test.Logf("Got error: '%s', expected '%s'", err.Error(), testCase.Expected)
+			test.Fail()
+			continue
+		}
+	}
+}

data_quality/src/figure/govaluate/parsing_test.go

@@ -0,0 +1,1670 @@
+package govaluate
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+	"unicode"
+)
+
+/*
+	Represents a test of parsing all tokens correctly from a string
+*/
+type TokenParsingTest struct {
+	Name      string
+	Input     string
+	Functions map[string]ExpressionFunction
+	Expected  []ExpressionToken
+}
+
+func TestConstantParsing(test *testing.T) {
+
+	tokenParsingTests := []TokenParsingTest{
+
+		TokenParsingTest{
+
+			Name:  "Single numeric",
+			Input: "1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Single two-digit numeric",
+			Input: "50",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 50.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Zero",
+			Input: "0",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 0.0,
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:  "One digit hex",
+			Input: "0x1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:  "Two digit hex",
+			Input: "0x10",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 16.0,
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:  "Hex with lowercase",
+			Input: "0xabcdef",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 11259375.0,
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:  "Hex with uppercase",
+			Input: "0xABCDEF",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 11259375.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Single string",
+			Input: "'foo'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foo",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Single time, RFC3339, only date",
+			Input: "'2014-01-02'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  TIME,
+					Value: time.Date(2014, time.January, 2, 0, 0, 0, 0, time.Local),
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Single time, RFC3339, with hh:mm",
+			Input: "'2014-01-02 14:12'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  TIME,
+					Value: time.Date(2014, time.January, 2, 14, 12, 0, 0, time.Local),
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Single time, RFC3339, with hh:mm:ss",
+			Input: "'2014-01-02 14:12:22'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  TIME,
+					Value: time.Date(2014, time.January, 2, 14, 12, 22, 0, time.Local),
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Single boolean",
+			Input: "true",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: true,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Single large numeric",
+			Input: "1234567890",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1234567890.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Single floating-point",
+			Input: "0.5",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 0.5,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Single large floating point",
+			Input: "3.14567471",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 3.14567471,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Single false boolean",
+			Input: "false",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: false,
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:  "Single internationalized string",
+			Input: "'ÆŦǽഈᚥஇคٸ'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "ÆŦǽഈᚥஇคٸ",
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:  "Single internationalized parameter",
+			Input: "ÆŦǽഈᚥஇคٸ",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "ÆŦǽഈᚥஇคٸ",
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:      "Parameterless function",
+			Input:     "foo()",
+			Functions: map[string]ExpressionFunction{"foo": noop},
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  FUNCTION,
+					Value: noop,
+				},
+				ExpressionToken{
+					Kind: CLAUSE,
+				},
+				ExpressionToken{
+					Kind: CLAUSE_CLOSE,
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:      "Single parameter function",
+			Input:     "foo('bar')",
+			Functions: map[string]ExpressionFunction{"foo": noop},
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  FUNCTION,
+					Value: noop,
+				},
+				ExpressionToken{
+					Kind: CLAUSE,
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "bar",
+				},
+				ExpressionToken{
+					Kind: CLAUSE_CLOSE,
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:      "Multiple parameter function",
+			Input:     "foo('bar', 1.0)",
+			Functions: map[string]ExpressionFunction{"foo": noop},
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  FUNCTION,
+					Value: noop,
+				},
+				ExpressionToken{
+					Kind: CLAUSE,
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "bar",
+				},
+				ExpressionToken{
+					Kind: SEPARATOR,
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind: CLAUSE_CLOSE,
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:      "Nested function",
+			Input:     "foo(foo('bar'), 1.0, foo(2.0))",
+			Functions: map[string]ExpressionFunction{"foo": noop},
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  FUNCTION,
+					Value: noop,
+				},
+				ExpressionToken{
+					Kind: CLAUSE,
+				},
+
+				ExpressionToken{
+					Kind:  FUNCTION,
+					Value: noop,
+				},
+				ExpressionToken{
+					Kind: CLAUSE,
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "bar",
+				},
+				ExpressionToken{
+					Kind: CLAUSE_CLOSE,
+				},
+				ExpressionToken{
+					Kind: SEPARATOR,
+				},
+
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+
+				ExpressionToken{
+					Kind: SEPARATOR,
+				},
+
+				ExpressionToken{
+					Kind:  FUNCTION,
+					Value: noop,
+				},
+				ExpressionToken{
+					Kind: CLAUSE,
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 2.0,
+				},
+				ExpressionToken{
+					Kind: CLAUSE_CLOSE,
+				},
+
+				ExpressionToken{
+					Kind: CLAUSE_CLOSE,
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:      "Function with modifier afterwards (#28)",
+			Input:     "foo() + 1",
+			Functions: map[string]ExpressionFunction{"foo": noop},
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  FUNCTION,
+					Value: noop,
+				},
+				ExpressionToken{
+					Kind: CLAUSE,
+				},
+				ExpressionToken{
+					Kind: CLAUSE_CLOSE,
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: "+",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:      "Function with modifier afterwards and comparator",
+			Input:     "(foo()-1) > 3",
+			Functions: map[string]ExpressionFunction{"foo": noop},
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind: CLAUSE,
+				},
+				ExpressionToken{
+					Kind:  FUNCTION,
+					Value: noop,
+				},
+				ExpressionToken{
+					Kind: CLAUSE,
+				},
+				ExpressionToken{
+					Kind: CLAUSE_CLOSE,
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: "-",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind: CLAUSE_CLOSE,
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: ">",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 3.0,
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:  "Double-quoted string added to square-brackted param (#59)",
+			Input: "\"a\" + [foo]",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "a",
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: "+",
+				},
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "foo",
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:  "Accessor variable",
+			Input: "foo.Var",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  ACCESSOR,
+					Value: []string{"foo", "Var"},
+				},
+			},
+		},
+		TokenParsingTest{
+			Name:  "Accessor function",
+			Input: "foo.Operation()",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  ACCESSOR,
+					Value: []string{"foo", "Operation"},
+				},
+				ExpressionToken{
+					Kind: CLAUSE,
+				},
+				ExpressionToken{
+					Kind: CLAUSE_CLOSE,
+				},
+			},
+		},
+	}
+
+	tokenParsingTests = combineWhitespaceExpressions(tokenParsingTests)
+	runTokenParsingTest(tokenParsingTests, test)
+}
+
+func TestLogicalOperatorParsing(test *testing.T) {
+
+	tokenParsingTests := []TokenParsingTest{
+
+		TokenParsingTest{
+
+			Name:  "Boolean AND",
+			Input: "true && false",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: true,
+				},
+				ExpressionToken{
+					Kind:  LOGICALOP,
+					Value: "&&",
+				},
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: false,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Boolean OR",
+			Input: "true || false",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: true,
+				},
+				ExpressionToken{
+					Kind:  LOGICALOP,
+					Value: "||",
+				},
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: false,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Multiple logical operators",
+			Input: "true || false && true",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: true,
+				},
+				ExpressionToken{
+					Kind:  LOGICALOP,
+					Value: "||",
+				},
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: false,
+				},
+				ExpressionToken{
+					Kind:  LOGICALOP,
+					Value: "&&",
+				},
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: true,
+				},
+			},
+		},
+	}
+
+	tokenParsingTests = combineWhitespaceExpressions(tokenParsingTests)
+	runTokenParsingTest(tokenParsingTests, test)
+}
+
+func TestComparatorParsing(test *testing.T) {
+
+	tokenParsingTests := []TokenParsingTest{
+
+		TokenParsingTest{
+
+			Name:  "Numeric EQ",
+			Input: "1 == 2",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "==",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 2.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric NEQ",
+			Input: "1 != 2",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "!=",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 2.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric GT",
+			Input: "1 > 0",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: ">",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 0.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric LT",
+			Input: "1 < 2",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "<",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 2.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric GTE",
+			Input: "1 >= 2",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: ">=",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 2.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric LTE",
+			Input: "1 <= 2",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "<=",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 2.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "String LT",
+			Input: "'ab.cd' < 'abc.def'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "ab.cd",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "<",
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "abc.def",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "String LTE",
+			Input: "'ab.cd' <= 'abc.def'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "ab.cd",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "<=",
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "abc.def",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "String GT",
+			Input: "'ab.cd' > 'abc.def'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "ab.cd",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: ">",
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "abc.def",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "String GTE",
+			Input: "'ab.cd' >= 'abc.def'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "ab.cd",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: ">=",
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "abc.def",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "String REQ",
+			Input: "'foobar' =~ 'bar'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foobar",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "=~",
+				},
+
+				// it's not particularly clean to test for the contents of a pattern, (since it means modifying the harness below)
+				// so pattern contents are left untested.
+				ExpressionToken{
+					Kind: PATTERN,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "String NREQ",
+			Input: "'foobar' !~ 'bar'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foobar",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "!~",
+				},
+				ExpressionToken{
+					Kind: PATTERN,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Comparator against modifier string additive (#22)",
+			Input: "'foo' == '+'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foo",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "==",
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "+",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Comparator against modifier string multiplicative (#22)",
+			Input: "'foo' == '/'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foo",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "==",
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "/",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Comparator against modifier string exponential (#22)",
+			Input: "'foo' == '**'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foo",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "==",
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "**",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Comparator against modifier string bitwise (#22)",
+			Input: "'foo' == '^'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foo",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "==",
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "^",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Comparator against modifier string shift (#22)",
+			Input: "'foo' == '>>'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foo",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "==",
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: ">>",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Comparator against modifier string ternary (#22)",
+			Input: "'foo' == '?'",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foo",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "==",
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "?",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Array membership lowercase",
+			Input: "'foo' in ('foo', 'bar')",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foo",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "in",
+				},
+				ExpressionToken{
+					Kind: CLAUSE,
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foo",
+				},
+				ExpressionToken{
+					Kind: SEPARATOR,
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "bar",
+				},
+				ExpressionToken{
+					Kind: CLAUSE_CLOSE,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Array membership uppercase",
+			Input: "'foo' IN ('foo', 'bar')",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foo",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "in",
+				},
+				ExpressionToken{
+					Kind: CLAUSE,
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foo",
+				},
+				ExpressionToken{
+					Kind: SEPARATOR,
+				},
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "bar",
+				},
+				ExpressionToken{
+					Kind: CLAUSE_CLOSE,
+				},
+			},
+		},
+	}
+
+	tokenParsingTests = combineWhitespaceExpressions(tokenParsingTests)
+	runTokenParsingTest(tokenParsingTests, test)
+}
+
+func TestModifierParsing(test *testing.T) {
+
+	tokenParsingTests := []TokenParsingTest{
+
+		TokenParsingTest{
+
+			Name:  "Numeric PLUS",
+			Input: "1 + 1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: "+",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric MINUS",
+			Input: "1 - 1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: "-",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric MULTIPLY",
+			Input: "1 * 1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: "*",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric DIVIDE",
+			Input: "1 / 1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: "/",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric MODULUS",
+			Input: "1 % 1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: "%",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric BITWISE_AND",
+			Input: "1 & 1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: "&",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric BITWISE_OR",
+			Input: "1 | 1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: "|",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric BITWISE_XOR",
+			Input: "1 ^ 1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: "^",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric BITWISE_LSHIFT",
+			Input: "1 << 1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: "<<",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Numeric BITWISE_RSHIFT",
+			Input: "1 >> 1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  MODIFIER,
+					Value: ">>",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+	}
+
+	tokenParsingTests = combineWhitespaceExpressions(tokenParsingTests)
+	runTokenParsingTest(tokenParsingTests, test)
+}
+
+func TestPrefixParsing(test *testing.T) {
+
+	testCases := []TokenParsingTest{
+
+		TokenParsingTest{
+
+			Name:  "Sign prefix",
+			Input: "-1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  PREFIX,
+					Value: "-",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Sign prefix on variable",
+			Input: "-foo",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  PREFIX,
+					Value: "-",
+				},
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "foo",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Boolean prefix",
+			Input: "!true",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  PREFIX,
+					Value: "!",
+				},
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: true,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Boolean prefix on variable",
+			Input: "!foo",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  PREFIX,
+					Value: "!",
+				},
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "foo",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Bitwise not prefix",
+			Input: "~1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  PREFIX,
+					Value: "~",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Bitwise not prefix on variable",
+			Input: "~foo",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  PREFIX,
+					Value: "~",
+				},
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "foo",
+				},
+			},
+		},
+	}
+
+	testCases = combineWhitespaceExpressions(testCases)
+	runTokenParsingTest(testCases, test)
+}
+
+func TestEscapedParameters(test *testing.T) {
+
+	testCases := []TokenParsingTest{
+
+		TokenParsingTest{
+
+			Name:  "Single escaped parameter",
+			Input: "[foo]",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "foo",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Single escaped parameter with whitespace",
+			Input: "[foo bar]",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "foo bar",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Single escaped parameter with escaped closing bracket",
+			Input: "[foo[bar\\]]",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "foo[bar]",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Escaped parameters and unescaped parameters",
+			Input: "[foo] > bar",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "foo",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: ">",
+				},
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "bar",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Unescaped parameter with space",
+			Input: "foo\\ bar > bar",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "foo bar",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: ">",
+				},
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "bar",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Unescaped parameter with space",
+			Input: "response\\-time > bar",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "response-time",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: ">",
+				},
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "bar",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Parameters with snake_case",
+			Input: "foo_bar > baz_quux",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "foo_bar",
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: ">",
+				},
+				ExpressionToken{
+					Kind:  VARIABLE,
+					Value: "baz_quux",
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "String literal uses backslash to escape",
+			Input: "\"foo\\'bar\"",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  STRING,
+					Value: "foo'bar",
+				},
+			},
+		},
+	}
+
+	runTokenParsingTest(testCases, test)
+}
+
+func TestTernaryParsing(test *testing.T) {
+	tokenParsingTests := []TokenParsingTest{
+
+		TokenParsingTest{
+
+			Name:  "Ternary after Boolean",
+			Input: "true ? 1",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: true,
+				},
+				ExpressionToken{
+					Kind:  TERNARY,
+					Value: "?",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Ternary after Comperator",
+			Input: "1 == 0 ? true",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  COMPARATOR,
+					Value: "==",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 0.0,
+				},
+				ExpressionToken{
+					Kind:  TERNARY,
+					Value: "?",
+				},
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: true,
+				},
+			},
+		},
+		TokenParsingTest{
+
+			Name:  "Null coalesce left",
+			Input: "1 ?? 2",
+			Expected: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind:  TERNARY,
+					Value: "??",
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 2.0,
+				},
+			},
+		},
+	}
+
+	runTokenParsingTest(tokenParsingTests, test)
+}
+
+/*
+	Tests to make sure that the String() reprsentation of an expression exactly matches what is given to the parse function.
+*/
+func TestOriginalString(test *testing.T) {
+
+	// include all the token types, to be sure there's no shenaniganery going on.
+	expressionString := "2 > 1 &&" +
+		"'something' != 'nothing' || " +
+		"'2014-01-20' < 'Wed Jul  8 23:07:35 MDT 2015' && " +
+		"[escapedVariable name with spaces] <= unescaped\\-variableName &&" +
+		"modifierTest + 1000 / 2 > (80 * 100 % 2) && true ? true : false"
+
+	expression, err := NewEvaluableExpression(expressionString)
+	if err != nil {
+
+		test.Logf("failed to parse original string test: %v", err)
+		test.Fail()
+		return
+	}
+
+	if expression.String() != expressionString {
+		test.Logf("String() did not give the same expression as given to parse")
+		test.Fail()
+	}
+}
+
+/*
+	Tests to make sure that the Vars() reprsentation of an expression identifies all variables contained within the expression.
+*/
+func TestOriginalVars(test *testing.T) {
+
+	// include all the token types, to be sure there's no shenaniganery going on.
+	expressionString := "2 > 1 &&" +
+		"'something' != 'nothing' || " +
+		"'2014-01-20' < 'Wed Jul  8 23:07:35 MDT 2015' && " +
+		"[escapedVariable name with spaces] <= unescaped\\-variableName &&" +
+		"modifierTest + 1000 / 2 > (80 * 100 % 2) && true ? true : false"
+
+	expectedVars := [3]string{"escapedVariable name with spaces",
+		"modifierTest",
+		"unescaped-variableName"}
+
+	expression, err := NewEvaluableExpression(expressionString)
+	if err != nil {
+
+		test.Logf("failed to parse original var test: %v", err)
+		test.Fail()
+		return
+	}
+
+	if len(expression.Vars()) == len(expectedVars) {
+		variableMap := make(map[string]string)
+		for _, v := range expression.Vars() {
+			variableMap[v] = v
+		}
+		for _, v := range expectedVars {
+			if _, ok := variableMap[v]; !ok {
+				test.Logf("Vars() did not correctly identify all variables contained within the expression")
+				test.Fail()
+			}
+		}
+	} else {
+		test.Logf("Vars() did not correctly identify all variables contained within the expression")
+		test.Fail()
+	}
+}
+
+func combineWhitespaceExpressions(testCases []TokenParsingTest) []TokenParsingTest {
+
+	var currentCase, strippedCase TokenParsingTest
+	var caseLength int
+
+	caseLength = len(testCases)
+
+	for i := 0; i < caseLength; i++ {
+
+		currentCase = testCases[i]
+
+		strippedCase = TokenParsingTest{
+
+			Name:      (currentCase.Name + " (without whitespace)"),
+			Input:     stripUnquotedWhitespace(currentCase.Input),
+			Expected:  currentCase.Expected,
+			Functions: currentCase.Functions,
+		}
+
+		testCases = append(testCases, strippedCase, currentCase)
+	}
+
+	return testCases
+}
+
+func stripUnquotedWhitespace(expression string) string {
+
+	var expressionBuffer bytes.Buffer
+	var quoted bool
+
+	for _, character := range expression {
+
+		if !quoted && unicode.IsSpace(character) {
+			continue
+		}
+
+		if character == '\'' {
+			quoted = !quoted
+		}
+
+		expressionBuffer.WriteString(string(character))
+	}
+
+	return expressionBuffer.String()
+}
+
+func runTokenParsingTest(tokenParsingTests []TokenParsingTest, test *testing.T) {
+
+	var parsingTest TokenParsingTest
+	var expression *EvaluableExpression
+	var actualTokens []ExpressionToken
+	var actualToken ExpressionToken
+	var expectedTokenKindString, actualTokenKindString string
+	var expectedTokenLength, actualTokenLength int
+	var err error
+
+	fmt.Printf("Running %d parsing test cases...\n", len(tokenParsingTests))
+	// defer func() {
+	//     if r := recover(); r != nil {
+	//         test.Logf("Panic in test '%s': %v", parsingTest.Name, r)
+	// 		test.Fail()
+	//     }
+	// }()
+
+	// Run the test cases.
+	for _, parsingTest = range tokenParsingTests {
+
+		if parsingTest.Functions != nil {
+			expression, err = NewEvaluableExpressionWithFunctions(parsingTest.Input, parsingTest.Functions)
+		} else {
+			expression, err = NewEvaluableExpression(parsingTest.Input)
+		}
+
+		if err != nil {
+
+			test.Logf("Test '%s' failed to parse: %s", parsingTest.Name, err)
+			test.Logf("Expression: '%s'", parsingTest.Input)
+			test.Fail()
+			continue
+		}
+
+		actualTokens = expression.Tokens()
+
+		expectedTokenLength = len(parsingTest.Expected)
+		actualTokenLength = len(actualTokens)
+
+		if actualTokenLength != expectedTokenLength {
+
+			test.Logf("Test '%s' failed:", parsingTest.Name)
+			test.Logf("Expected %d tokens, actually found %d", expectedTokenLength, actualTokenLength)
+			test.Fail()
+			continue
+		}
+
+		for i, expectedToken := range parsingTest.Expected {
+
+			actualToken = actualTokens[i]
+			if actualToken.Kind != expectedToken.Kind {
+
+				actualTokenKindString = actualToken.Kind.String()
+				expectedTokenKindString = expectedToken.Kind.String()
+
+				test.Logf("Test '%s' failed:", parsingTest.Name)
+				test.Logf("Expected token kind '%v' does not match '%v'", expectedTokenKindString, actualTokenKindString)
+				test.Fail()
+				continue
+			}
+
+			if expectedToken.Value == nil {
+				continue
+			}
+
+			reflectedKind := reflect.TypeOf(expectedToken.Value).Kind()
+			if reflectedKind == reflect.Func {
+				continue
+			}
+
+			// gotta be an accessor
+			if reflectedKind == reflect.Slice {
+
+				if actualToken.Value == nil {
+					test.Logf("Test '%s' failed:", parsingTest.Name)
+					test.Logf("Expected token value '%v' does not match nil", expectedToken.Value)
+					test.Fail()
+				}
+
+				for z, actual := range actualToken.Value.([]string) {
+
+					if actual != expectedToken.Value.([]string)[z] {
+
+						test.Logf("Test '%s' failed:", parsingTest.Name)
+						test.Logf("Expected token value '%v' does not match '%v'", expectedToken.Value, actualToken.Value)
+						test.Fail()
+					}
+				}
+				continue
+			}
+
+			if actualToken.Value != expectedToken.Value {
+
+				test.Logf("Test '%s' failed:", parsingTest.Name)
+				test.Logf("Expected token value '%v' does not match '%v'", expectedToken.Value, actualToken.Value)
+				test.Fail()
+				continue
+			}
+		}
+	}
+}
+
+func noop(arguments ...interface{}) (interface{}, error) {
+	return nil, nil
+}

data_quality/src/figure/govaluate/readme_test.go

@@ -0,0 +1,192 @@
+package govaluate
+
+/*
+  Contains test cases for all the expression examples given in the README.
+  While all of the functionality for these cases should be covered in other tests,
+  this is really just a sanity check.
+*/
+import (
+	"testing"
+)
+
+func TestBasicEvaluation(test *testing.T) {
+
+	expression, err := NewEvaluableExpression("10 > 0")
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	result, err := expression.Evaluate(nil)
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	if result != true {
+		test.Logf("Expected 'true', got '%v'\n", result)
+		test.Fail()
+	}
+}
+
+func TestParameterEvaluation(test *testing.T) {
+
+	expression, err := NewEvaluableExpression("foo > 0")
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	parameters := make(map[string]interface{}, 8)
+	parameters["foo"] = -1
+
+	result, err := expression.Evaluate(parameters)
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	if result != false {
+		test.Logf("Expected 'false', got '%v'\n", result)
+		test.Fail()
+	}
+}
+
+func TestModifierEvaluation(test *testing.T) {
+
+	expression, err := NewEvaluableExpression("(requests_made * requests_succeeded / 100) >= 90")
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	parameters := make(map[string]interface{}, 8)
+	parameters["requests_made"] = 100
+	parameters["requests_succeeded"] = 80
+
+	result, err := expression.Evaluate(parameters)
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	if result != false {
+		test.Logf("Expected 'false', got '%v'\n", result)
+		test.Fail()
+	}
+}
+
+func TestStringEvaluation(test *testing.T) {
+
+	expression, err := NewEvaluableExpression("http_response_body == 'service is ok'")
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	parameters := make(map[string]interface{}, 8)
+	parameters["http_response_body"] = "service is ok"
+
+	result, err := expression.Evaluate(parameters)
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	if result != true {
+		test.Logf("Expected 'false', got '%v'\n", result)
+		test.Fail()
+	}
+}
+
+func TestFloatEvaluation(test *testing.T) {
+
+	expression, err := NewEvaluableExpression("(mem_used / total_mem) * 100")
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	parameters := make(map[string]interface{}, 8)
+	parameters["total_mem"] = 1024
+	parameters["mem_used"] = 512
+
+	result, err := expression.Evaluate(parameters)
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	if result != 50.0 {
+		test.Logf("Expected '50.0', got '%v'\n", result)
+		test.Fail()
+	}
+}
+
+func TestDateComparison(test *testing.T) {
+
+	expression, err := NewEvaluableExpression("'2014-01-02' > '2014-01-01 23:59:59'")
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	result, err := expression.Evaluate(nil)
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	if result != true {
+		test.Logf("Expected 'true', got '%v'\n", result)
+		test.Fail()
+	}
+}
+
+func TestMultipleEvaluation(test *testing.T) {
+	expression, _ := NewEvaluableExpression("response_time <= 100")
+	parameters := make(map[string]interface{}, 8)
+
+	for i := 0; i < 64; i++ {
+		parameters["response_time"] = i
+		result, err := expression.Evaluate(parameters)
+		if err != nil {
+			test.Log(err)
+			test.Fail()
+		}
+
+		if result != true {
+			test.Logf("Expected 'true', got '%v'\n", result)
+			test.Fail()
+			break
+		}
+	}
+}
+
+func TestStrlenFunction(test *testing.T) {
+
+	functions := map[string]ExpressionFunction{
+		"strlen": func(args ...interface{}) (interface{}, error) {
+			length := len(args[0].(string))
+			return (float64)(length), nil
+		},
+	}
+
+	expString := "strlen('someReallyLongInputString') <= 16"
+	expression, err := NewEvaluableExpressionWithFunctions(expString, functions)
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	result, err := expression.Evaluate(nil)
+	if err != nil {
+		test.Log(err)
+		test.Fail()
+	}
+
+	if result != false {
+		test.Logf("Expected 'false', got '%v'\n", result)
+		test.Fail()
+	}
+}

data_quality/src/figure/govaluate/sanitizedParameters.go

@@ -0,0 +1,43 @@
+package govaluate
+
+// sanitizedParameters is a wrapper for Parameters that does sanitization as
+// parameters are accessed.
+type sanitizedParameters struct {
+	orig Parameters
+}
+
+func (p sanitizedParameters) Get(key string) (interface{}, error) {
+	value, err := p.orig.Get(key)
+	if err != nil {
+		return nil, err
+	}
+
+	return castToFloat64(value), nil
+}
+
+func castToFloat64(value interface{}) interface{} {
+	switch value.(type) {
+	case uint8:
+		return float64(value.(uint8))
+	case uint16:
+		return float64(value.(uint16))
+	case uint32:
+		return float64(value.(uint32))
+	case uint64:
+		return float64(value.(uint64))
+	case int8:
+		return float64(value.(int8))
+	case int16:
+		return float64(value.(int16))
+	case int32:
+		return float64(value.(int32))
+	case int64:
+		return float64(value.(int64))
+	case int:
+		return float64(value.(int))
+	case float32:
+		return float64(value.(float32))
+	}
+
+	return value
+}

data_quality/src/figure/govaluate/sql_test.go

@@ -0,0 +1,247 @@
+package govaluate
+
+import (
+	"testing"
+)
+
+/*
+	Represents a test of correctly creating a SQL query string from an expression.
+*/
+type QueryTest struct {
+	Name     string
+	Input    string
+	Expected string
+}
+
+func TestSQLSerialization(test *testing.T) {
+
+	testCases := []QueryTest{
+
+		QueryTest{
+
+			Name:     "Single GT",
+			Input:    "1 > 0",
+			Expected: "1 > 0",
+		},
+		QueryTest{
+
+			Name:     "Single LT",
+			Input:    "0 < 1",
+			Expected: "0 < 1",
+		},
+		QueryTest{
+
+			Name:     "Single GTE",
+			Input:    "1 >= 0",
+			Expected: "1 >= 0",
+		},
+		QueryTest{
+
+			Name:     "Single LTE",
+			Input:    "0 <= 1",
+			Expected: "0 <= 1",
+		},
+		QueryTest{
+
+			Name:     "Single EQ",
+			Input:    "1 == 0",
+			Expected: "1 = 0",
+		},
+		QueryTest{
+
+			Name:     "Single NEQ",
+			Input:    "1 != 0",
+			Expected: "1 <> 0",
+		},
+
+		QueryTest{
+
+			Name:     "Parameter names",
+			Input:    "foo == bar",
+			Expected: "[foo] = [bar]",
+		},
+		QueryTest{
+
+			Name:     "Strings",
+			Input:    "'foo'",
+			Expected: "'foo'",
+		},
+		QueryTest{
+
+			Name:     "Date format",
+			Input:    "'2014-07-04T00:00:00Z'",
+			Expected: "'2014-07-04T00:00:00Z'",
+		},
+		QueryTest{
+
+			Name:     "Single PLUS",
+			Input:    "10 + 10",
+			Expected: "10 + 10",
+		},
+		QueryTest{
+
+			Name:     "Single MINUS",
+			Input:    "10 - 10",
+			Expected: "10 - 10",
+		},
+		QueryTest{
+
+			Name:     "Single MULTIPLY",
+			Input:    "10 * 10",
+			Expected: "10 * 10",
+		},
+		QueryTest{
+
+			Name:     "Single DIVIDE",
+			Input:    "10 / 10",
+			Expected: "10 / 10",
+		},
+		QueryTest{
+
+			Name:     "Single true bool",
+			Input:    "true",
+			Expected: "1",
+		},
+		QueryTest{
+
+			Name:     "Single false bool",
+			Input:    "false",
+			Expected: "0",
+		},
+		QueryTest{
+
+			Name:     "Single AND",
+			Input:    "true && true",
+			Expected: "1 AND 1",
+		},
+		QueryTest{
+
+			Name:     "Single OR",
+			Input:    "true || true",
+			Expected: "1 OR 1",
+		},
+		QueryTest{
+
+			Name:     "Clauses",
+			Input:    "10 + (foo + bar)",
+			Expected: "10 + ( [foo] + [bar] )",
+		},
+		QueryTest{
+
+			Name:     "Negate prefix",
+			Input:    "foo < -1",
+			Expected: "[foo] < -1",
+		},
+		QueryTest{
+
+			Name:     "Invert prefix",
+			Input:    "!(foo > 1)",
+			Expected: "NOT ( [foo] > 1 )",
+		},
+		QueryTest{
+
+			Name:     "Exponent",
+			Input:    "1 ** 2",
+			Expected: "POW(1, 2)",
+		},
+		QueryTest{
+
+			Name:     "Modulus",
+			Input:    "10 % 2",
+			Expected: "MOD(10, 2)",
+		},
+		QueryTest{
+
+			Name:     "Membership operator",
+			Input:    "foo IN (1, 2, 3)",
+			Expected: "[foo] in ( 1 , 2 , 3 )",
+		},
+		QueryTest{
+
+			Name:     "Null coalescence",
+			Input:    "foo ?? bar",
+			Expected: "COALESCE([foo], [bar])",
+		},
+		/*
+			// Ternaries don't work yet, because the outputter is not yet sophisticated enough to produce them.
+			QueryTest{
+
+				Name:     "Full ternary",
+				Input:    "[foo] == 5 ? 1 : 2",
+				Expected: "IF([foo] = 5, 1, 2)",
+			},
+			QueryTest{
+
+				Name:     "Half ternary",
+				Input:    "[foo] == 5 ? 1",
+				Expected: "IF([foo] = 5, 1)",
+			},
+			QueryTest{
+
+				Name:     "Full ternary with implicit bool",
+				Input:    "[foo] ? 1 : 2",
+				Expected: "IF([foo] = 0, 1, 2)",
+			},
+			QueryTest{
+
+				Name:     "Half ternary with implicit bool",
+				Input:    "[foo] ? 1",
+				Expected: "IF([foo] = 0, 1)",
+			},*/
+		QueryTest{
+
+			Name:     "Regex equals",
+			Input:    "'foo' =~ '[fF][oO]+'",
+			Expected: "'foo' RLIKE '[fF][oO]+'",
+		},
+		QueryTest{
+
+			Name:     "Regex not-equals",
+			Input:    "'foo' !~ '[fF][oO]+'",
+			Expected: "'foo' NOT RLIKE '[fF][oO]+'",
+		},
+	}
+
+	runQueryTests(testCases, test)
+}
+
+func runQueryTests(testCases []QueryTest, test *testing.T) {
+
+	var expression *EvaluableExpression
+	var actualQuery string
+	var err error
+
+	test.Logf("Running %d SQL translation test cases", len(testCases))
+
+	// Run the test cases.
+	for _, testCase := range testCases {
+
+		expression, err = NewEvaluableExpression(testCase.Input)
+
+		if err != nil {
+
+			test.Logf("Test '%s' failed to parse: %s", testCase.Name, err)
+			test.Logf("Expression: '%s'", testCase.Input)
+			test.Fail()
+			continue
+		}
+
+		actualQuery, err = expression.ToSQLQuery()
+
+		if err != nil {
+
+			test.Logf("Test '%s' failed to create query: %s", testCase.Name, err)
+			test.Logf("Expression: '%s'", testCase.Input)
+			test.Fail()
+			continue
+		}
+
+		if actualQuery != testCase.Expected {
+
+			test.Logf("Test '%s' did not create expected query.", testCase.Name)
+			test.Logf("Actual: '%s', expected '%s'", actualQuery, testCase.Expected)
+			test.Fail()
+			continue
+		}
+	}
+}

data_quality/src/figure/govaluate/stagePlanner.go

@@ -0,0 +1,724 @@
+package govaluate
+
+import (
+	"errors"
+	"fmt"
+	"time"
+)
+
+var stageSymbolMap = map[OperatorSymbol]evaluationOperator{
+	EQ:             equalStage,
+	NEQ:            notEqualStage,
+	GT:             gtStage,
+	LT:             ltStage,
+	GTE:            gteStage,
+	LTE:            lteStage,
+	REQ:            regexStage,
+	NREQ:           notRegexStage,
+	AND:            andStage,
+	OR:             orStage,
+	IN:             inStage,
+	BITWISE_OR:     bitwiseOrStage,
+	BITWISE_AND:    bitwiseAndStage,
+	BITWISE_XOR:    bitwiseXORStage,
+	BITWISE_LSHIFT: leftShiftStage,
+	BITWISE_RSHIFT: rightShiftStage,
+	PLUS:           addStage,
+	MINUS:          subtractStage,
+	MULTIPLY:       multiplyStage,
+	DIVIDE:         divideStage,
+	MODULUS:        modulusStage,
+	EXPONENT:       exponentStage,
+	NEGATE:         negateStage,
+	INVERT:         invertStage,
+	BITWISE_NOT:    bitwiseNotStage,
+	TERNARY_TRUE:   ternaryIfStage,
+	TERNARY_FALSE:  ternaryElseStage,
+	COALESCE:       ternaryElseStage,
+	SEPARATE:       separatorStage,
+}
+
+/*
+	A "precedent" is a function which will recursively parse new evaluateionStages from a given stream of tokens.
+	It's called a `precedent` because it is expected to handle exactly what precedence of operator,
+	and defer to other `precedent`s for other operators.
+*/
+type precedent func(stream *tokenStream) (*evaluationStage, error)
+
+/*
+	A convenience function for specifying the behavior of a `precedent`.
+	Most `precedent` functions can be described by the same function, just with different type checks, symbols, and error formats.
+	This struct is passed to `makePrecedentFromPlanner` to create a `precedent` function.
+*/
+type precedencePlanner struct {
+	validSymbols map[string]OperatorSymbol
+	validKinds   []TokenKind
+
+	typeErrorFormat string
+
+	next      precedent
+	nextRight precedent
+}
+
+var planPrefix precedent
+var planExponential precedent
+var planMultiplicative precedent
+var planAdditive precedent
+var planBitwise precedent
+var planShift precedent
+var planComparator precedent
+var planLogicalAnd precedent
+var planLogicalOr precedent
+var planTernary precedent
+var planSeparator precedent
+
+func init() {
+
+	// all these stages can use the same code (in `planPrecedenceLevel`) to execute,
+	// they simply need different type checks, symbols, and recursive precedents.
+	// While not all precedent phases are listed here, most can be represented this way.
+	planPrefix = makePrecedentFromPlanner(&precedencePlanner{
+		validSymbols:    prefixSymbols,
+		validKinds:      []TokenKind{PREFIX},
+		typeErrorFormat: prefixErrorFormat,
+		nextRight:       planFunction,
+	})
+	planExponential = makePrecedentFromPlanner(&precedencePlanner{
+		validSymbols:    exponentialSymbolsS,
+		validKinds:      []TokenKind{MODIFIER},
+		typeErrorFormat: modifierErrorFormat,
+		next:            planFunction,
+	})
+	planMultiplicative = makePrecedentFromPlanner(&precedencePlanner{
+		validSymbols:    multiplicativeSymbols,
+		validKinds:      []TokenKind{MODIFIER},
+		typeErrorFormat: modifierErrorFormat,
+		next:            planExponential,
+	})
+	planAdditive = makePrecedentFromPlanner(&precedencePlanner{
+		validSymbols:    additiveSymbols,
+		validKinds:      []TokenKind{MODIFIER},
+		typeErrorFormat: modifierErrorFormat,
+		next:            planMultiplicative,
+	})
+	planShift = makePrecedentFromPlanner(&precedencePlanner{
+		validSymbols:    bitwiseShiftSymbols,
+		validKinds:      []TokenKind{MODIFIER},
+		typeErrorFormat: modifierErrorFormat,
+		next:            planAdditive,
+	})
+	planBitwise = makePrecedentFromPlanner(&precedencePlanner{
+		validSymbols:    bitwiseSymbols,
+		validKinds:      []TokenKind{MODIFIER},
+		typeErrorFormat: modifierErrorFormat,
+		next:            planShift,
+	})
+	planComparator = makePrecedentFromPlanner(&precedencePlanner{
+		validSymbols:    comparatorSymbols,
+		validKinds:      []TokenKind{COMPARATOR},
+		typeErrorFormat: comparatorErrorFormat,
+		next:            planBitwise,
+	})
+	planLogicalAnd = makePrecedentFromPlanner(&precedencePlanner{
+		validSymbols:    map[string]OperatorSymbol{"&&": AND},
+		validKinds:      []TokenKind{LOGICALOP},
+		typeErrorFormat: logicalErrorFormat,
+		next:            planComparator,
+	})
+	planLogicalOr = makePrecedentFromPlanner(&precedencePlanner{
+		validSymbols:    map[string]OperatorSymbol{"||": OR},
+		validKinds:      []TokenKind{LOGICALOP},
+		typeErrorFormat: logicalErrorFormat,
+		next:            planLogicalAnd,
+	})
+	planTernary = makePrecedentFromPlanner(&precedencePlanner{
+		validSymbols:    ternarySymbols,
+		validKinds:      []TokenKind{TERNARY},
+		typeErrorFormat: ternaryErrorFormat,
+		next:            planLogicalOr,
+	})
+	planSeparator = makePrecedentFromPlanner(&precedencePlanner{
+		validSymbols: separatorSymbols,
+		validKinds:   []TokenKind{SEPARATOR},
+		next:         planTernary,
+	})
+}
+
+/*
+	Given a planner, creates a function which will evaluate a specific precedence level of operators,
+	and link it to other `precedent`s which recurse to parse other precedence levels.
+*/
+func makePrecedentFromPlanner(planner *precedencePlanner) precedent {
+
+	var generated precedent
+	var nextRight precedent
+
+	generated = func(stream *tokenStream) (*evaluationStage, error) {
+		return planPrecedenceLevel(
+			stream,
+			planner.typeErrorFormat,
+			planner.validSymbols,
+			planner.validKinds,
+			nextRight,
+			planner.next,
+		)
+	}
+
+	if planner.nextRight != nil {
+		nextRight = planner.nextRight
+	} else {
+		nextRight = generated
+	}
+
+	return generated
+}
+
+/*
+	Creates a `evaluationStageList` object which represents an execution plan (or tree)
+	which is used to completely evaluate a set of tokens at evaluation-time.
+	The three stages of evaluation can be thought of as parsing strings to tokens, then tokens to a stage list, then evaluation with parameters.
+*/
+func planStages(tokens []ExpressionToken) (*evaluationStage, error) {
+
+	stream := newTokenStream(tokens)
+
+	stage, err := planTokens(stream)
+	if err != nil {
+		return nil, err
+	}
+
+	// while we're now fully-planned, we now need to re-order same-precedence operators.
+	// this could probably be avoided with a different planning method
+	reorderStages(stage)
+
+	stage = elideLiterals(stage)
+	return stage, nil
+}
+
+func planTokens(stream *tokenStream) (*evaluationStage, error) {
+
+	if !stream.hasNext() {
+		return nil, nil
+	}
+
+	return planSeparator(stream)
+}
+
+/*
+	The most usual method of parsing an evaluation stage for a given precedence.
+	Most stages use the same logic
+*/
+func planPrecedenceLevel(
+	stream *tokenStream,
+	typeErrorFormat string,
+	validSymbols map[string]OperatorSymbol,
+	validKinds []TokenKind,
+	rightPrecedent precedent,
+	leftPrecedent precedent) (*evaluationStage, error) {
+
+	var token ExpressionToken
+	var symbol OperatorSymbol
+	var leftStage, rightStage *evaluationStage
+	var checks typeChecks
+	var err error
+	var keyFound bool
+
+	if leftPrecedent != nil {
+
+		leftStage, err = leftPrecedent(stream)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	for stream.hasNext() {
+
+		token = stream.next()
+
+		if len(validKinds) > 0 {
+
+			keyFound = false
+			for _, kind := range validKinds {
+				if kind == token.Kind {
+					keyFound = true
+					break
+				}
+			}
+
+			if !keyFound {
+				break
+			}
+		}
+
+		if validSymbols != nil {
+
+			if !isString(token.Value) {
+				break
+			}
+
+			symbol, keyFound = validSymbols[token.Value.(string)]
+			if !keyFound {
+				break
+			}
+		}
+
+		if rightPrecedent != nil {
+			rightStage, err = rightPrecedent(stream)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		checks = findTypeChecks(symbol)
+
+		return &evaluationStage{
+
+			symbol:     symbol,
+			leftStage:  leftStage,
+			rightStage: rightStage,
+			operator:   stageSymbolMap[symbol],
+
+			leftTypeCheck:   checks.left,
+			rightTypeCheck:  checks.right,
+			typeCheck:       checks.combined,
+			typeErrorFormat: typeErrorFormat,
+		}, nil
+	}
+
+	stream.rewind()
+	return leftStage, nil
+}
+
+/*
+	A special case where functions need to be of higher precedence than values, and need a special wrapped execution stage operator.
+*/
+func planFunction(stream *tokenStream) (*evaluationStage, error) {
+
+	var token ExpressionToken
+	var rightStage *evaluationStage
+	var err error
+
+	token = stream.next()
+
+	if token.Kind != FUNCTION {
+		stream.rewind()
+		return planAccessor(stream)
+	}
+
+	rightStage, err = planAccessor(stream)
+	if err != nil {
+		return nil, err
+	}
+
+	return &evaluationStage{
+
+		symbol:          FUNCTIONAL,
+		rightStage:      rightStage,
+		operator:        makeFunctionStage(token.Value.(ExpressionFunction)),
+		typeErrorFormat: "Unable to run function '%v': %v",
+	}, nil
+}
+
+func planAccessor(stream *tokenStream) (*evaluationStage, error) {
+
+	var token, otherToken ExpressionToken
+	var rightStage *evaluationStage
+	var err error
+
+	if !stream.hasNext() {
+		return nil, nil
+	}
+
+	token = stream.next()
+
+	if token.Kind != ACCESSOR {
+		stream.rewind()
+		return planValue(stream)
+	}
+
+	// check if this is meant to be a function or a field.
+	// fields have a clause next to them, functions do not.
+	// if it's a function, parse the arguments. Otherwise leave the right stage null.
+	if stream.hasNext() {
+
+		otherToken = stream.next()
+		if otherToken.Kind == CLAUSE {
+
+			stream.rewind()
+
+			rightStage, err = planTokens(stream)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			stream.rewind()
+		}
+	}
+
+	return &evaluationStage{
+
+		symbol:          ACCESS,
+		rightStage:      rightStage,
+		operator:        makeAccessorStage(token.Value.([]string)),
+		typeErrorFormat: "Unable to access parameter field or method '%v': %v",
+	}, nil
+}
+
+/*
+	A truly special precedence function, this handles all the "lowest-case" errata of the process, including literals, parmeters,
+	clauses, and prefixes.
+*/
+func planValue(stream *tokenStream) (*evaluationStage, error) {
+
+	var token ExpressionToken
+	var symbol OperatorSymbol
+	var ret *evaluationStage
+	var operator evaluationOperator
+	var err error
+
+	if !stream.hasNext() {
+		return nil, nil
+	}
+
+	token = stream.next()
+
+	switch token.Kind {
+
+	case CLAUSE:
+
+		ret, err = planTokens(stream)
+		if err != nil {
+			return nil, err
+		}
+
+		// advance past the CLAUSE_CLOSE token. We know that it's a CLAUSE_CLOSE, because at parse-time we check for unbalanced parens.
+		stream.next()
+
+		// the stage we got represents all of the logic contained within the parens
+		// but for technical reasons, we need to wrap this stage in a "noop" stage which breaks long chains of precedence.
+		// see github #33.
+		ret = &evaluationStage{
+			rightStage: ret,
+			operator:   noopStageRight,
+			symbol:     NOOP,
+		}
+
+		return ret, nil
+
+	case CLAUSE_CLOSE:
+
+		// when functions have empty params, this will be hit. In this case, we don't have any evaluation stage to do,
+		// so we just return nil so that the stage planner continues on its way.
+		stream.rewind()
+		return nil, nil
+
+	case VARIABLE:
+		operator = makeParameterStage(token.Value.(string))
+
+	case NUMERIC:
+		fallthrough
+	case STRING:
+		fallthrough
+	case PATTERN:
+		fallthrough
+	case BOOLEAN:
+		symbol = LITERAL
+		operator = makeLiteralStage(token.Value)
+	case TIME:
+		symbol = LITERAL
+		operator = makeLiteralStage(float64(token.Value.(time.Time).Unix()))
+
+	case PREFIX:
+		stream.rewind()
+		return planPrefix(stream)
+	}
+
+	if operator == nil {
+		errorMsg := fmt.Sprintf("Unable to plan token kind: '%s', value: '%v'", token.Kind.String(), token.Value)
+		return nil, errors.New(errorMsg)
+	}
+
+	return &evaluationStage{
+		symbol:   symbol,
+		operator: operator,
+	}, nil
+}
+
+/*
+	Convenience function to pass a triplet of typechecks between `findTypeChecks` and `planPrecedenceLevel`.
+	Each of these members may be nil, which indicates that type does not matter for that value.
+*/
+type typeChecks struct {
+	left     stageTypeCheck
+	right    stageTypeCheck
+	combined stageCombinedTypeCheck
+}
+
+/*
+	Maps a given [symbol] to a set of typechecks to be used during runtime.
+*/
+func findTypeChecks(symbol OperatorSymbol) typeChecks {
+
+	switch symbol {
+	case GT:
+		fallthrough
+	case LT:
+		fallthrough
+	case GTE:
+		fallthrough
+	case LTE:
+		return typeChecks{
+			combined: comparatorTypeCheck,
+		}
+	case REQ:
+		fallthrough
+	case NREQ:
+		return typeChecks{
+			left:  isString,
+			right: isRegexOrString,
+		}
+	case AND:
+		fallthrough
+	case OR:
+		return typeChecks{
+			left:  isBool,
+			right: isBool,
+		}
+	case IN:
+		return typeChecks{
+			right: isArray,
+		}
+	case BITWISE_LSHIFT:
+		fallthrough
+	case BITWISE_RSHIFT:
+		fallthrough
+	case BITWISE_OR:
+		fallthrough
+	case BITWISE_AND:
+		fallthrough
+	case BITWISE_XOR:
+		return typeChecks{
+			left:  isFloat64,
+			right: isFloat64,
+		}
+	case PLUS:
+		return typeChecks{
+			combined: additionTypeCheck,
+		}
+	case MINUS:
+		fallthrough
+	case MULTIPLY:
+		fallthrough
+	case DIVIDE:
+		fallthrough
+	case MODULUS:
+		fallthrough
+	case EXPONENT:
+		return typeChecks{
+			left:  isFloat64,
+			right: isFloat64,
+		}
+	case NEGATE:
+		return typeChecks{
+			right: isFloat64,
+		}
+	case INVERT:
+		return typeChecks{
+			right: isBool,
+		}
+	case BITWISE_NOT:
+		return typeChecks{
+			right: isFloat64,
+		}
+	case TERNARY_TRUE:
+		return typeChecks{
+			left: isBool,
+		}
+
+	// unchecked cases
+	case EQ:
+		fallthrough
+	case NEQ:
+		return typeChecks{}
+	case TERNARY_FALSE:
+		fallthrough
+	case COALESCE:
+		fallthrough
+	default:
+		return typeChecks{}
+	}
+}
+
+/*
+	During stage planning, stages of equal precedence are parsed such that they'll be evaluated in reverse order.
+	For commutative operators like "+" or "-", it's no big deal. But for order-specific operators, it ruins the expected result.
+*/
+func reorderStages(rootStage *evaluationStage) {
+
+	// traverse every rightStage until we find multiples in a row of the same precedence.
+	var identicalPrecedences []*evaluationStage
+	var currentStage, nextStage *evaluationStage
+	var precedence, currentPrecedence operatorPrecedence
+
+	nextStage = rootStage
+	precedence = findOperatorPrecedenceForSymbol(rootStage.symbol)
+
+	for nextStage != nil {
+
+		currentStage = nextStage
+		nextStage = currentStage.rightStage
+
+		// left depth first, since this entire method only looks for precedences down the right side of the tree
+		if currentStage.leftStage != nil {
+			reorderStages(currentStage.leftStage)
+		}
+
+		currentPrecedence = findOperatorPrecedenceForSymbol(currentStage.symbol)
+
+		if currentPrecedence == precedence {
+			identicalPrecedences = append(identicalPrecedences, currentStage)
+			continue
+		}
+
+		// precedence break.
+		// See how many in a row we had, and reorder if there's more than one.
+		if len(identicalPrecedences) > 1 {
+			mirrorStageSubtree(identicalPrecedences)
+		}
+
+		identicalPrecedences = []*evaluationStage{currentStage}
+		precedence = currentPrecedence
+	}
+
+	if len(identicalPrecedences) > 1 {
+		mirrorStageSubtree(identicalPrecedences)
+	}
+}
+
+/*
+	Performs a "mirror" on a subtree of stages.
+	This mirror functionally inverts the order of execution for all members of the [stages] list.
+	That list is assumed to be a root-to-leaf (ordered) list of evaluation stages, where each is a right-hand stage of the last.
+*/
+func mirrorStageSubtree(stages []*evaluationStage) {
+
+	var rootStage, inverseStage, carryStage, frontStage *evaluationStage
+
+	stagesLength := len(stages)
+
+	// reverse all right/left
+	for _, frontStage = range stages {
+
+		carryStage = frontStage.rightStage
+		frontStage.rightStage = frontStage.leftStage
+		frontStage.leftStage = carryStage
+	}
+
+	// end left swaps with root right
+	rootStage = stages[0]
+	frontStage = stages[stagesLength-1]
+
+	carryStage = frontStage.leftStage
+	frontStage.leftStage = rootStage.rightStage
+	rootStage.rightStage = carryStage
+
+	// for all non-root non-end stages, right is swapped with inverse stage right in list
+	for i := 0; i < (stagesLength-2)/2+1; i++ {
+
+		frontStage = stages[i+1]
+		inverseStage = stages[stagesLength-i-1]
+
+		carryStage = frontStage.rightStage
+		frontStage.rightStage = inverseStage.rightStage
+		inverseStage.rightStage = carryStage
+	}
+
+	// swap all other information with inverse stages
+	for i := 0; i < stagesLength/2; i++ {
+
+		frontStage = stages[i]
+		inverseStage = stages[stagesLength-i-1]
+		frontStage.swapWith(inverseStage)
+	}
+}
+
+/*
+	Recurses through all operators in the entire tree, eliding operators where both sides are literals.
+*/
+func elideLiterals(root *evaluationStage) *evaluationStage {
+
+	if root.leftStage != nil {
+		root.leftStage = elideLiterals(root.leftStage)
+	}
+
+	if root.rightStage != nil {
+		root.rightStage = elideLiterals(root.rightStage)
+	}
+
+	return elideStage(root)
+}
+
+/*
+	Elides a specific stage, if possible.
+	Returns the unmodified [root] stage if it cannot or should not be elided.
+	Otherwise, returns a new stage representing the condensed value from the elided stages.
+*/
+func elideStage(root *evaluationStage) *evaluationStage {
+
+	var leftValue, rightValue, result interface{}
+	var err error
+
+	// right side must be a non-nil value. Left side must be nil or a value.
+	if root.rightStage == nil ||
+		root.rightStage.symbol != LITERAL ||
+		root.leftStage == nil ||
+		root.leftStage.symbol != LITERAL {
+		return root
+	}
+
+	// don't elide some operators
+	switch root.symbol {
+	case SEPARATE:
+		fallthrough
+	case IN:
+		return root
+	}
+
+	// both sides are values, get their actual values.
+	// errors should be near-impossible here. If we encounter them, just abort this optimization.
+	leftValue, err = root.leftStage.operator(nil, nil, nil)
+	if err != nil {
+		return root
+	}
+
+	rightValue, err = root.rightStage.operator(nil, nil, nil)
+	if err != nil {
+		return root
+	}
+
+	// typcheck, since the grammar checker is a bit loose with which operator symbols go together.
+	err = typeCheck(root.leftTypeCheck, leftValue, root.symbol, root.typeErrorFormat)
+	if err != nil {
+		return root
+	}
+
+	err = typeCheck(root.rightTypeCheck, rightValue, root.symbol, root.typeErrorFormat)
+	if err != nil {
+		return root
+	}
+
+	if root.typeCheck != nil && !root.typeCheck(leftValue, rightValue) {
+		return root
+	}
+
+	// pre-calculate, and return a new stage representing the result.
+	result, err = root.operator(leftValue, rightValue, nil)
+	if err != nil {
+		return root
+	}
+
+	return &evaluationStage{
+		symbol:   LITERAL,
+		operator: makeLiteralStage(result),
+	}
+}

data_quality/src/figure/govaluate/test.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Script that runs tests, code coverage, and benchmarks all at once.
+# Builds a symlink in /tmp, mostly to avoid messing with GOPATH at the user's shell level.
+
+TEMPORARY_PATH="/tmp/govaluate_test"
+SRC_PATH="${TEMPORARY_PATH}/src"
+FULL_PATH="${TEMPORARY_PATH}/src/govaluate"
+
+# set up temporary directory
+rm -rf "${FULL_PATH}"
+mkdir -p "${SRC_PATH}"
+
+ln -s $(pwd) "${FULL_PATH}"
+export GOPATH="${TEMPORARY_PATH}"
+
+pushd "${TEMPORARY_PATH}/src/govaluate"
+
+# run the actual tests.
+export GOVALUATE_TORTURE_TEST="true"
+go test -bench=. -benchmem #-coverprofile coverage.out
+status=$?
+
+if [ "${status}" != 0 ];
+then
+	exit $status
+fi
+
+# coverage
+# disabled because travis go1.4 seems not to support it suddenly?
+#go tool cover -func=coverage.out
+
+popd

data_quality/src/figure/govaluate/tokenExpressionFailure_test.go

@@ -0,0 +1,204 @@
+package govaluate
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+)
+
+const (
+	EXPERR_NIL_VALUE string = "cannot have a nil value"
+)
+
+/*
+	Contains a single test case for the EvaluableExpression.NewEvaluableExpressionFromTokens() method.
+
+	These tests, and the ones in `tokenExpressionFailure_test` will be fairly incomplete.
+	Creating an expression from a string and from tokens _must_ both perform the same syntax checks.
+	So all the checks in `parsing_test` will follow the same logic as the ones here.
+
+	These tests check some corner cases - such as tokens having nil values when they must have something.
+	Cases that cannot occur through the normal parser, but may occur in other parsers.
+*/
+type ExpressionTokenSyntaxTest struct {
+	Name     string
+	Input    []ExpressionToken
+	Expected string
+}
+
+func TestNilValues(test *testing.T) {
+
+	cases := []ExpressionTokenSyntaxTest{
+		ExpressionTokenSyntaxTest{
+			Name: "Nil numeric",
+			Input: []ExpressionToken{
+				ExpressionToken{
+					Kind: NUMERIC,
+				},
+			},
+			Expected: EXPERR_NIL_VALUE,
+		},
+		ExpressionTokenSyntaxTest{
+			Name: "Nil string",
+			Input: []ExpressionToken{
+				ExpressionToken{
+					Kind: STRING,
+				},
+			},
+			Expected: EXPERR_NIL_VALUE,
+		},
+		ExpressionTokenSyntaxTest{
+			Name: "Nil bool",
+			Input: []ExpressionToken{
+				ExpressionToken{
+					Kind: BOOLEAN,
+				},
+			},
+			Expected: EXPERR_NIL_VALUE,
+		},
+		ExpressionTokenSyntaxTest{
+			Name: "Nil time",
+			Input: []ExpressionToken{
+				ExpressionToken{
+					Kind: TIME,
+				},
+			},
+			Expected: EXPERR_NIL_VALUE,
+		},
+		ExpressionTokenSyntaxTest{
+			Name: "Nil pattern",
+			Input: []ExpressionToken{
+				ExpressionToken{
+					Kind: PATTERN,
+				},
+			},
+			Expected: EXPERR_NIL_VALUE,
+		},
+		ExpressionTokenSyntaxTest{
+			Name: "Nil variable",
+			Input: []ExpressionToken{
+				ExpressionToken{
+					Kind: VARIABLE,
+				},
+			},
+			Expected: EXPERR_NIL_VALUE,
+		},
+		ExpressionTokenSyntaxTest{
+			Name: "Nil prefix",
+			Input: []ExpressionToken{
+				ExpressionToken{
+					Kind: PREFIX,
+				},
+			},
+			Expected: EXPERR_NIL_VALUE,
+		},
+		ExpressionTokenSyntaxTest{
+			Name: "Nil comparator",
+			Input: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind: COMPARATOR,
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+			Expected: EXPERR_NIL_VALUE,
+		},
+		ExpressionTokenSyntaxTest{
+			Name: "Nil logicalop",
+			Input: []ExpressionToken{
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: true,
+				},
+				ExpressionToken{
+					Kind: LOGICALOP,
+				},
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: true,
+				},
+			},
+			Expected: EXPERR_NIL_VALUE,
+		},
+		ExpressionTokenSyntaxTest{
+			Name: "Nil modifer",
+			Input: []ExpressionToken{
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+				ExpressionToken{
+					Kind: MODIFIER,
+				},
+				ExpressionToken{
+					Kind:  NUMERIC,
+					Value: 1.0,
+				},
+			},
+			Expected: EXPERR_NIL_VALUE,
+		},
+		ExpressionTokenSyntaxTest{
+			Name: "Nil ternary",
+			Input: []ExpressionToken{
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: true,
+				},
+				ExpressionToken{
+					Kind: TERNARY,
+				},
+				ExpressionToken{
+					Kind:  BOOLEAN,
+					Value: true,
+				},
+			},
+			Expected: EXPERR_NIL_VALUE,
+		},
+	}
+
+	runExpressionFromTokenTests(cases, true, test)
+}
+
+func runExpressionFromTokenTests(cases []ExpressionTokenSyntaxTest, expectFail bool, test *testing.T) {
+
+	var err error
+
+	fmt.Printf("Running %d expression from expression token tests...\n", len(cases))
+
+	for _, testCase := range cases {
+
+		_, err = NewEvaluableExpressionFromTokens(testCase.Input)
+
+		if err != nil {
+			if expectFail {
+
+				if !strings.Contains(err.Error(), testCase.Expected) {
+
+					test.Logf("Test '%s' failed", testCase.Name)
+					test.Logf("Got error: '%s', expected '%s'", err.Error(), testCase.Expected)
+					test.Fail()
+				}
+				continue
+			}
+
+			test.Logf("Test '%s' failed", testCase.Name)
+			test.Logf("Got error: '%s'", err)
+			test.Fail()
+			continue
+		} else {
+			if expectFail {
+
+				test.Logf("Test '%s' failed", testCase.Name)
+				test.Logf("Expected error, found none\n")
+				test.Fail()
+				continue
+			}
+		}
+	}
+}

data_quality/src/figure/govaluate/tokenKind_test.go

@@ -0,0 +1,47 @@
+package govaluate
+
+import (
+	"testing"
+)
+
+/*
+	Tests to make sure that all the different token kinds have different string representations
+	Gotta get that 95% code coverage yall. That's why tests like this get written; over-reliance on bad metrics.
+*/
+func TestTokenKindStrings(test *testing.T) {
+
+	var kindStrings []string
+	var kindString string
+
+	kinds := []TokenKind{
+		UNKNOWN,
+		PREFIX,
+		NUMERIC,
+		BOOLEAN,
+		STRING,
+		PATTERN,
+		TIME,
+		VARIABLE,
+		COMPARATOR,
+		LOGICALOP,
+		MODIFIER,
+		CLAUSE,
+		CLAUSE_CLOSE,
+		TERNARY,
+	}
+
+	for _, kind := range kinds {
+
+		kindString = kind.String()
+
+		for _, extantKind := range kindStrings {
+			if extantKind == kindString {
+				test.Logf("Token kind test found duplicate string for token kind %v ('%v')\n", kind, kindString)
+				test.Fail()
+				return
+			}
+		}
+
+		kindStrings = append(kindStrings, kindString)
+	}
+}

data_quality/src/figure/govaluate/tokenStream.go

@@ -0,0 +1,36 @@
+package govaluate
+
+type tokenStream struct {
+	tokens      []ExpressionToken
+	index       int
+	tokenLength int
+}
+
+func newTokenStream(tokens []ExpressionToken) *tokenStream {
+
+	var ret *tokenStream
+
+	ret = new(tokenStream)
+	ret.tokens = tokens
+	ret.tokenLength = len(tokens)
+	return ret
+}
+
+func (this *tokenStream) rewind() {
+	this.index -= 1
+}
+
+func (this *tokenStream) next() ExpressionToken {
+
+	var token ExpressionToken
+
+	token = this.tokens[this.index]
+
+	this.index += 1
+	return token
+}
+
+func (this tokenStream) hasNext() bool {
+
+	return this.index < this.tokenLength
+}

data_quality/src/figure/govaluate/torture_test.go

@@ -0,0 +1,176 @@
+package govaluate
+
+/*
+	Courtesy of abrander
+	ref: https://gist.github.com/abrander/fa05ae9b181b48ffe7afb12c961b6e90
+*/
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+)
+
+var (
+	hello  = "hello"
+	empty  struct{}
+	empty2 *string
+	empty3 *int
+
+	values = []interface{}{
+		-1,
+		0,
+		12,
+		13,
+		"",
+		"hello",
+		&hello,
+		nil,
+		"nil",
+		empty,
+		empty2,
+		true,
+		false,
+		time.Now(),
+		rune('r'),
+		int64(34),
+		time.Duration(0),
+		"true",
+		"false",
+		"\ntrue\n",
+		"\nfalse\n",
+		"12",
+		"nil",
+		"arg1",
+		"arg2",
+		int(12),
+		int32(12),
+		int64(12),
+		complex(1.0, 1.0),
+		[]byte{0, 0, 0},
+		[]int{0, 0, 0},
+		[]string{},
+		"[]",
+		"{}",
+		"\"\"",
+		"\"12\"",
+		"\"hello\"",
+		".*",
+		"==",
+		"!=",
+		">",
+		">=",
+		"<",
+		"<=",
+		"=~",
+		"!~",
+		"in",
+		"&&",
+		"||",
+		"^",
+		"&",
+		"|",
+		">>",
+		"<<",
+		"+",
+		"-",
+		"*",
+		"/",
+		"%",
+		"**",
+		"-",
+		"!",
+		"~",
+		"?",
+		":",
+		"??",
+		"+",
+		"-",
+		"*",
+		"/",
+		"%",
+		"**",
+		"&",
+		"|",
+		"^",
+		">>",
+		"<<",
+		",",
+		"(",
+		")",
+		"[",
+		"]",
+		"\n",
+		"\000",
+	}
+
+	panics = 0
+)
+
+const (
+	ITERATIONS = 10000000
+	SEED       = 1487873697990155515
+)
+
+func init() {
+	rand.Seed(SEED)
+}
+
+func TestPanics(test *testing.T) {
+
+	if os.Getenv("GOVALUATE_TORTURE_TEST") == "" {
+		test.Logf("'GOVALUATE_TORTURE_TEST' env var not set - skipping torture test.")
+		test.Skip()
+		return
+	}
+
+	fmt.Printf("Running %d torture test cases...\n", ITERATIONS)
+
+	for i := 0; i < ITERATIONS; i++ {
+
+		num := rand.Intn(3) + 2
+		expression := ""
+
+		for n := 0; n < num; n++ {
+			expression += fmt.Sprintf(" %s", getRandom(values))
+		}
+
+		checkPanic(expression, test)
+	}
+
+	test.Logf("Done. %d/%d panics.\n", panics, ITERATIONS)
+	if panics > 0 {
+		test.Fail()
+	}
+}
+
+func checkPanic(expression string, test *testing.T) {
+
+	parameters := make(map[string]interface{})
+
+	defer func() {
+		if r := recover(); r != nil {
+			test.Logf("Panic: \"%s\". Expression: \"%s\". Parameters: %+v\n", r, expression, parameters)
+			panics++
+		}
+	}()
+
+	eval, _ := NewEvaluableExpression(expression)
+	if eval == nil {
+		return
+	}
+
+	vars := eval.Vars()
+	for _, v := range vars {
+		parameters[v] = getRandom(values)
+	}
+
+	eval.Evaluate(parameters)
+}
+
+func getRandom(haystack []interface{}) interface{} {
+
+	i := rand.Intn(len(haystack))
+	return haystack[i]
+}

data_quality/src/figure/govalueate.go

@@ -1,8 +1,36 @@
 package figure
 
+import (
+	"figure/govaluate"
+	log "github.com/donnie4w/go-logger/logger"
+)
+
 func GoValueateTest() {
+	// 支持多个逻辑表达式
+	expr, err := govaluate.NewEvaluableExpression("(10 > 0) && (2.1 == 2.1) && 'service is ok' == 'service is ok'" +
+		" && 1 in (1,2) && 'code2' in ('code3','code2',1)")
+	if err != nil {
+		log.Fatal("syntax error:", err)
+	}
+	result, err := expr.Evaluate(nil)
+	if err != nil {
+		log.Fatal("evaluate error:", err)
+	}
+	log.Debug(result)
+
+	// 逻辑表达式包含变量
+	expression, err := govaluate.NewEvaluableExpression("http_response_body == 'service is ok'")
+	parameters := make(map[string]interface{}, 8)
+	parameters["http_response_body"] = "service is ok"
+	res, _ := expression.Evaluate(parameters)
+	log.Debug(res)
 
-	//expr, err := govaluate.NewEvaluableExpression("(10 > 0) && (2.1 == 2.1) && 'service is ok' == 'service is ok'" +
-	//	" && 1 in (1,2) && 'code1' in ('code3','code2',1)")
+	// 算数表达式包含变量
+	expression1, _ := govaluate.NewEvaluableExpression("requests_made + requests_succeeded")
+	parameters1 := make(map[string]interface{}, 8)
+	parameters1["requests_made"] = 100
+	parameters1["requests_succeeded"] = 80
+	result1, _ := expression1.Evaluate(parameters1)
+	log.Debug(result1)
 
 }

data_quality/src/main.go

@@ -1,12 +1,11 @@
 package main
 
 import (
-	"figure"
 	u "ul"
 )
 
 func init() {
-	//u.IsLocal = true
+	u.IsLocal = true
 	u.InitClass()
 }
 
@@ -16,8 +15,9 @@ func main() {
 	//figure.FigurelQualityLevelSecond()
 	//figure.FigurelSourceInfoQuantity()
 
-	figure.FigureUnLevelScreenInfo()
-	figure.FigureUnLevelScreenNumber()
+	//figure.FigureUnLevelScreenInfo()
+	//figure.FigureUnLevelScreenNumber()
+	//figure.FigureGlobalScoreInfo()
 
 	//保持活性
 	lock := make(chan bool)

data_ylqx/src/bidding/bidding.go

@@ -9,8 +9,8 @@ import (
 )
 
 var (
-	YlPurBaseInfo    = "zktest_dwd_f_yl_purchasing_baseinfo"
-	YlPurWinBaseInfo = "zktest_dwd_f_yl_purchasing_win_baseinfo"
+	YlPurBaseInfo    = "dwd_f_yl_purchasing_baseinfo"
+	YlPurWinBaseInfo = "dwd_f_yl_purchasing_win_baseinfo"
 )
 
 //对比标的物名称
@@ -52,7 +52,7 @@ func confrimTargetMedicalClass(name string) (string, map[string]interface{}) {
 	}
 	new_code := calculateSimilarityScore(indexDocs, itemArr)
 	if new_code != "" {
-		catalog := ul.CodeCatalog[med_code]
+		catalog := ul.CodeCatalog[new_code]
 		info = map[string]interface{}{
 			"class_1": catalog["class_1"],
 			"class_2": catalog["class_2"],

data_ylqx/src/bidding/purchasing.go

@@ -9,8 +9,9 @@ import (
 	ul "util"
 )
 
+//存量处理...
 func StartDealWithPurashingInfo() {
-	importPurchasingBaseInfo("147caf2aa1840869018d5fac", "647caf2aa1840869018d5fac")
+	importPurchasingBaseInfo("100000000000000000000000", "900000000000000000000000")
 }
 func importPurchasingBaseInfo(gtid string, lteid string) {
 	sess := ul.Ext_Mgo.GetMgoConn()
@@ -34,6 +35,7 @@ func importPurchasingBaseInfo(gtid string, lteid string) {
 				ul.Wg_Mgo.Done()
 			}()
 			createPurchasingBaseInfo(tmp)
+			createWinnerBaseInfo(tmp)
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
@@ -190,6 +192,8 @@ func ConvertStatusType(bidstatus string, bidtype string, baseInfo *map[string]in
 func AlculatePurInfo(purInfo map[string]interface{}, baseInfo *map[string]interface{}) {
 	if itemname := qu.ObjToString(purInfo["itemname"]); itemname != "" {
 		(*baseInfo)["itemname"] = itemname
+		//标的物可信度评级-reliability
+		(*baseInfo)["reliability"] = ul.PostReliability(itemname)
 		//计算标的物分类
 		med_code, med_info := ComparisonItemNameInfo(itemname)
 		if class_1 := qu.ObjToString(med_info["class_1"]); class_1 != "" {

data_ylqx/src/bidding_add/bidding_add.go

@@ -1,40 +1,60 @@
 package bidding_add
 
 import (
-	"github.com/cron"
 	log "github.com/donnie4w/go-logger/logger"
+	"relation_add"
 	"time"
+	ul "util"
 )
 
-//定时任务~
-func BiddingAddTimedTask() {
-	c := cron.New()
-	c.AddFunc("0 0 3 ? * *", func() {
-		getQueryTimeSql()
-	})
-	c.Start()
-	//以下临时执行
-	q_time := 1685592000
-	query := map[string]interface{}{
+var mod_buyer, mod_winner []string
+
+// 定时任务~
+func BiddingRelationAddTask() {
+	initBiddingRelationUdpMsg() //监听
+}
+
+// 获取查询语句
+func GetBiddingIncrementalInfo(pici int64) {
+	mod_buyer, mod_buyer = []string{}, []string{}
+	if pici <= int64(0) {
+		log.Debug("错误...", pici)
+		return
+	}
+	query, total := map[string]interface{}{
 		"pici": map[string]interface{}{
-			"$gte": q_time,
+			"$gte": pici,
 		},
+	}, 0
+	log.Debug("当前获取待更新的query...", query)
+	sess := ul.Ext_Mgo.GetMgoConn()
+	defer ul.Ext_Mgo.DestoryMongoConn(sess)
+	it := sess.DB(ul.Ext_Mgo.DbName).C("projectset_medical").Find(&query).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%1000 == 0 {
+			log.Debug("curent index ", total)
+		}
+		incrementalPursInfo(tmp)
+		incrementalWinsInfo(tmp)
+		tmp = make(map[string]interface{})
 	}
-	log.Debug("临时~执行一次~~~", query)
-	increasePurchasingBaseInfo(query)
-	increaseWinnerBaseInfo(query)
+	log.Debug("is over ... ", total)
+	//根据待修改-重新构建
+	mod_buyer = resetRepeatModInfo(mod_buyer)
+	mod_winner = resetRepeatModInfo(mod_winner)
+	log.Debug("待修改...", len(mod_buyer), "~", len(mod_winner))
+	time.Sleep(time.Second * 2)
+	relation_add.GetRelationIncrementalInfo(mod_buyer, mod_winner)
 }
 
-//获取查询语句
-func getQueryTimeSql() {
-	now := time.Now()
-	q_time := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
-	query := map[string]interface{}{
-		"pici": map[string]interface{}{
-			"$gte": q_time,
-		},
+func resetRepeatModInfo(modArr []string) []string {
+	arr := []string{}
+	unique := map[string]string{}
+	for _, v := range modArr {
+		if v != "" && unique[v] == "" {
+			unique[v] = v
+			arr = append(arr, v)
+		}
 	}
-	log.Debug("获取当天凌晨的sql语句~~~", q_time)
-	increasePurchasingBaseInfo(query)
-	increaseWinnerBaseInfo(query)
+	return arr
 }

data_ylqx/src/bidding_add/bidding_udp.go

@@ -0,0 +1,50 @@
+package bidding_add
+
+import (
+	"encoding/json"
+	log "github.com/donnie4w/go-logger/logger"
+	mu "mfw/util"
+	"net"
+	qu "qfw/util"
+	"sync"
+)
+
+var (
+	udpclient mu.UdpClient
+	udpLock   sync.Mutex
+)
+
+func initBiddingRelationUdpMsg() {
+	port := ":1584"
+	udpclient = mu.UdpClient{Local: port, BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	log.Debug("监听~", port)
+}
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case mu.OP_TYPE_DATA:
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		if err != nil {
+			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			stype := qu.ObjToString(mapInfo["stype"])
+			if stype == "monitor" {
+				log.Debug("收到监测......")
+				udpclient.WriteUdp([]byte(qu.ObjToString(mapInfo["key"])), mu.OP_NOOP, ra)
+				break
+			}
+			pici := qu.Int64All(mapInfo["pici"])
+			if pici <= int64(0) {
+				log.Debug("接收异常...", mapInfo)
+			} else {
+				udpclient.WriteUdp([]byte(qu.ObjToString(mapInfo["key"])), mu.OP_NOOP, ra)
+				udpLock.Lock()
+				GetBiddingIncrementalInfo(pici)
+				udpLock.Unlock()
+			}
+		}
+	case mu.OP_NOOP: //下个节点回应
+		log.Debug("接收~", string(data))
+	}
+}

data_ylqx/src/bidding_add/purchasing_add.go

@@ -4,52 +4,36 @@ import (
 	"bidding"
 	"fmt"
 	log "github.com/donnie4w/go-logger/logger"
-	"go.mongodb.org/mongo-driver/bson"
 	qu "qfw/util"
 	"time"
 	"unicode/utf8"
 	ul "util"
 )
 
-//增加标的物信息
-func increasePurchasingBaseInfo(query map[string]interface{}) {
-	sess := ul.Ext_Mgo.GetMgoConn()
-	defer ul.Ext_Mgo.DestoryMongoConn(sess)
-	total := 0
-	it := sess.DB(ul.Ext_Mgo.DbName).C("projectset_medical").Find(&query).Iter()
-	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%1000 == 0 {
-			log.Debug("curent index ", total)
-		}
-		dealWithIncrementalPursInfo(tmp)
-		tmp = make(map[string]interface{})
-	}
-	log.Debug("is over ... ", total)
-	time.Sleep(time.Second * 2)
-}
-
 //处理是否新增标的物信息
-func dealWithIncrementalPursInfo(tmp map[string]interface{}) {
-	jg_plist := ul.IsMarkInterfaceMap(tmp["jg_plist"])
-	if len(jg_plist) == 0 {
-		return
-	}
+func incrementalPursInfo(tmp map[string]interface{}) {
 	projectid, infoid := ul.BsonTOStringId(tmp["_id"]), qu.ObjToString(tmp["sourceinfoid"])
-	bidstatus, bidtype := qu.ObjToString(tmp["bidstatus"]), qu.ObjToString(tmp["bidtype"])
 	if infoid == "" || projectid == "" {
 		log.Debug("项目信息异常~~~", projectid)
 		return
 	}
-	isExists := false
-	purInfos := ul.MysqlSubjectTool.Find(bidding.YlPurBaseInfo, bson.M{"projectid": projectid}, "", "", -1, -1)
-	if purInfos != nil {
-		if len(*purInfos) > 0 {
-			isExists = true
+	//先删除旧版项目标的物信息
+	data := ul.MysqlSubjectTool.FindOne(bidding.YlPurBaseInfo, map[string]interface{}{"projectid": projectid}, "buyer", "")
+	if data != nil {
+		ul.MysqlSubjectTool.Delete(bidding.YlPurBaseInfo, map[string]interface{}{"projectid": projectid})
+		if buyer := qu.ObjToString((*data)["buyer"]); buyer != "" {
+			mod_buyer = append(mod_buyer, buyer)
+		}
+		if buyer := qu.ObjToString(tmp["buyer"]); buyer != "" {
+			mod_buyer = append(mod_buyer, buyer)
 		}
 	}
-	if isExists {
-		jg_plist = confrimIsIncreasePurInfo(jg_plist, *purInfos)
+	//判断标的物是否存在以及合理等
+	jg_plist := ul.IsMarkInterfaceMap(tmp["jg_plist"])
+	if len(jg_plist) == 0 {
+		return
 	}
+	bidstatus, bidtype := qu.ObjToString(tmp["bidstatus"]), qu.ObjToString(tmp["bidtype"])
 	jyhref := fmt.Sprintf(ul.Url, qu.CommonEncodeArticle("content", infoid))
 	area_code, city_code, district_code := ul.DealWithRegionCode(qu.ObjToString(tmp["area"]), qu.ObjToString(tmp["city"]), qu.ObjToString(tmp["district"]))
 	for _, v := range jg_plist {
@@ -73,8 +57,8 @@ func dealWithIncrementalPursInfo(tmp map[string]interface{}) {
 		if tmp["projectname"] != nil {
 			projectname := qu.ObjToString(tmp["projectname"])
 			baseInfo["projectname"] = projectname
-			if utf8.RuneCountInString(projectname) > 300 {
-				baseInfo["projectname"] = len(projectname[:300])
+			if utf8.RuneCountInString(projectname) > 200 {
+				baseInfo["projectname"] = len(projectname[:200])
 			}
 		}
 		if tmp["projectcode"] != nil {

data_ylqx/src/bidding_add/winner_add.go

@@ -3,32 +3,29 @@ package bidding_add
 import (
 	"bidding"
 	log "github.com/donnie4w/go-logger/logger"
-	"go.mongodb.org/mongo-driver/bson"
 	qu "qfw/util"
 	"time"
-	"unicode/utf8"
 	ul "util"
 )
 
-//增加标的物信息
-func increaseWinnerBaseInfo(query map[string]interface{}) {
-	sess := ul.Ext_Mgo.GetMgoConn()
-	defer ul.Ext_Mgo.DestoryMongoConn(sess)
-	total := 0
-	it := sess.DB(ul.Ext_Mgo.DbName).C("projectset_medical").Find(&query).Iter()
-	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%1000 == 0 {
-			log.Debug("curent index ", total)
+//处理是否新增中标信息
+func incrementalWinsInfo(tmp map[string]interface{}) {
+	projectid, infoid := ul.BsonTOStringId(tmp["_id"]), qu.ObjToString(tmp["sourceinfoid"])
+	if infoid == "" || projectid == "" {
+		log.Debug("项目信息异常~~~", projectid)
+		return
+	}
+	//先删除旧版项目中标信息
+	dataArr := ul.MysqlSubjectTool.Find(bidding.YlPurWinBaseInfo, map[string]interface{}{"projectid": projectid}, "winner", "", -1, -1)
+	if dataArr != nil {
+		ul.MysqlSubjectTool.Delete(bidding.YlPurWinBaseInfo, map[string]interface{}{"projectid": projectid})
+		for _, v := range *dataArr {
+			if winner := qu.ObjToString(v["winner"]); winner != "" {
+				mod_winner = append(mod_winner, winner)
+			}
 		}
-		dealWithIncrementalWinsInfo(tmp)
-		tmp = make(map[string]interface{})
 	}
-	log.Debug("is over ... ", total)
-	time.Sleep(time.Second * 2)
-}
-
-//处理是否新增中标信息
-func dealWithIncrementalWinsInfo(tmp map[string]interface{}) {
+	//判断标的物是否存在以及合理等
 	jg_plist := ul.IsMarkInterfaceMap(tmp["jg_plist"])
 	if len(jg_plist) == 0 {
 		return
@@ -38,22 +35,8 @@ func dealWithIncrementalWinsInfo(tmp map[string]interface{}) {
 	if winner == "" && len(winnerorder) == 0 {
 		return
 	}
-	projectid, infoid := ul.BsonTOStringId(tmp["_id"]), qu.ObjToString(tmp["sourceinfoid"])
-	if infoid == "" || projectid == "" {
-		log.Debug("项目信息异常~~~", projectid)
-		return
-	}
-	isExists := false
-	winInfos := ul.MysqlSubjectTool.Find(bidding.YlPurWinBaseInfo, bson.M{"projectid": projectid}, "", "", -1, -1)
-	if winInfos != nil {
-		if len(*winInfos) > 0 {
-			isExists = true
-		}
-	}
-	new_wins, is_wins := []string{}, []int{}
-	if isExists {
-		new_wins, is_wins = confrimIsIncreaseWinInfo(winner, winnerorder, *winInfos)
-	}
+	new_wins, is_wins := bidding.MergeWinsInfo(winner, winnerorder)
+	mod_winner = append(mod_winner, new_wins...)
 	contact_name, contact_tel := qu.ObjToString(tmp["winnerperson"]), qu.ObjToString(tmp["winnertel"])
 	bidamount := qu.Float64All(tmp["bidamount"])
 	jgtime_str := ""
@@ -62,9 +45,6 @@ func dealWithIncrementalWinsInfo(tmp map[string]interface{}) {
 	}
 	//以上为通用信息
 	for k, v := range new_wins {
-		if utf8.RuneCountInString(v) > 50 {
-			continue
-		}
 		info := map[string]interface{}{}
 		info["projectid"] = projectid
 		info["infoid"] = infoid

data_ylqx/src/dimension/dimension.go

@@ -42,8 +42,8 @@ func StartBuildDealerDimensionGroupInfo() {
 	defer ul.Bid_Mgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{
 		"_id": map[string]interface{}{
-			"$gte": ul.StringTOBsonId("1fedf5800000000000000000"),
-			"$lte": ul.StringTOBsonId("94776f000000000000000000"),
+			"$gt":  ul.StringTOBsonId("100000000000000000000000"),
+			"$lte": ul.StringTOBsonId("900000000000000000000000"),
 		},
 	}
 	it := sess.DB(ul.Bid_Mgo.DbName).C("bidding_medical").Find(&q).Sort("_id").Select(map[string]interface{}{
@@ -59,16 +59,6 @@ func StartBuildDealerDimensionGroupInfo() {
 		if total%10000 == 0 {
 			log.Debug("cur index ", total, "~", tmp["_id"])
 		}
-		if total%1000000 == 0 {
-			isok := 0
-			for _, v := range dealer_datas {
-				if len(v) > 0 {
-					isok++
-				}
-			}
-			log.Debug("数据模型此时~", isok)
-		}
-
 		ul.Pool_Mgo <- true
 		ul.Wg_Mgo.Add(1)
 		go func(tmp map[string]interface{}) {
@@ -122,20 +112,24 @@ func dealWithDimensionInfo(tmp map[string]interface{}) {
 	dataLock.Lock()
 	for _, winner := range win_arr {
 		info := dealer_datas[winner]
-		if winner != "" && info != nil {
-			//进行数据组装
-			area_data := info["area"]
-			area_data = append(area_data, area)
-			info["area"] = area_data
-
+		if winner != "" && info != nil { //进行数据组装
+			isCover := false
 			for _, v := range p_list {
-				itemname := qu.ObjToString(v["itemname"])
-				if itemname != "" {
-					p_list_datas := info["p_list"]
-					p_list_datas = append(p_list_datas, itemname)
-					info["p_list"] = p_list_datas
+				if itemname := qu.ObjToString(v["itemname"]); itemname != "" {
+					if ul.PostReliability(itemname) >= ul.S_Reality {
+						isCover = true
+						p_list_datas := info["p_list"]
+						p_list_datas = append(p_list_datas, itemname)
+						info["p_list"] = p_list_datas
+					}
 				}
 			}
+			if isCover {
+				area_data := info["area"]
+				area_data = append(area_data, area)
+				info["area"] = area_data
+			}
+			dealer_datas[winner] = info
 		}
 	}
 	dataLock.Unlock()

data_ylqx/src/dimension/dimension_add.go

@@ -1,10 +1,8 @@
 package dimension
 
 import (
-	"github.com/cron"
 	log "github.com/donnie4w/go-logger/logger"
 	qu "qfw/util"
-	"strconv"
 	"strings"
 	"time"
 	"unicode/utf8"
@@ -53,44 +51,27 @@ func initIncreaseDepartCode() {
 	log.Debug("设备科室~", len(increase_depart_codes))
 }
 
-//增量标签处理
-func DimensionAddTimedTask() {
+// 增量标签处理
+func DimensionAddTask() {
+	//增加udp控制监测与接收处理数据
 	initIncreaseDealerInfo() //加载经销商信息
 	initIncreaseAreaCode()
 	initIncreaseDepartCode()
+	initDimensionUdpMsg() //监听
 	if len(increase_dealer_id) <= 0 || len(increase_area_codes) <= 0 || len(increase_depart_codes) <= 0 {
-		log.Debug("数据准备异常... ...")
+		log.Debug("数据准备异常......")
 		return
 	}
-	//查标讯信息
-	c := cron.New()
-	c.AddFunc("0 0 2 ? * *", func() {
-		getQueryTimeSql()
-	})
-	c.Start()
 }
 
-//获取查询语句
-func getQueryTimeSql() {
-	now := time.Now()
-	start := time.Date(now.Year(), now.Month(), now.Day()-2, 0, 0, 0, 0, time.Local).Unix()
-	end := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local).Unix()
-	if end > 0 && start > 0 {
-		gteid := strconv.FormatInt(start, 16) + "0000000000000000"
-		ltid := strconv.FormatInt(end, 16) + "0000000000000000"
-		dealWithIncreaseCoverDepartProTag(gteid, ltid)
-	} else {
-		log.Debug("时间转换异常......")
-	}
-}
-func dealWithIncreaseCoverDepartProTag(gteid string, ltid string) {
+func dealWithIncreaseCoverDepartProTag(gtid string, lteid string) {
 	increase_datas = map[string]map[string][]string{}
 	sess := ul.Bid_Mgo.GetMgoConn()
 	defer ul.Bid_Mgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{
 		"_id": map[string]interface{}{
-			"$gte": ul.StringTOBsonId(gteid),
-			"$lt":  ul.StringTOBsonId(ltid),
+			"$gt":  ul.StringTOBsonId(gtid),
+			"$lte": ul.StringTOBsonId(lteid),
 		},
 	}
 	log.Debug("查询语句~", q)
@@ -133,7 +114,7 @@ func dealWithIncreaseCoverDepartProTag(gteid string, ltid string) {
 	time.Sleep(time.Second * 2)
 }
 
-//处理经销商~招标信息
+// 处理经销商~招标信息
 func buildIncreaseDimensionInfo(tmp map[string]interface{}) {
 	s_winner := qu.ObjToString(tmp["s_winner"])
 	area := qu.ObjToString(tmp["area"])
@@ -145,23 +126,28 @@ func buildIncreaseDimensionInfo(tmp map[string]interface{}) {
 			if info == nil {
 				info = map[string][]string{}
 			}
-			area_data := info["area"]
-			area_data = append(area_data, area)
-			info["area"] = area_data
+			isCover := false
 			for _, v := range p_list {
-				itemname := qu.ObjToString(v["itemname"])
-				if itemname != "" {
-					p_list_datas := info["p_list"]
-					p_list_datas = append(p_list_datas, itemname)
-					info["p_list"] = p_list_datas
+				if itemname := qu.ObjToString(v["itemname"]); itemname != "" {
+					if ul.PostReliability(itemname) >= ul.S_Reality {
+						isCover = true
+						p_list_datas := info["p_list"]
+						p_list_datas = append(p_list_datas, itemname)
+						info["p_list"] = p_list_datas
+					}
 				}
 			}
+			if isCover {
+				area_data := info["area"]
+				area_data = append(area_data, area)
+				info["area"] = area_data
+			}
 			increase_datas[winner] = info
 		}
 	}
 }
 
-//增量-地域标签
+// 增量-地域标签
 func createIncreaseAreaTag(dealer_id string, area_list []string) {
 	unique := map[string]string{}
 	for _, v := range area_list {
@@ -179,7 +165,7 @@ func createIncreaseAreaTag(dealer_id string, area_list []string) {
 	}
 }
 
-//增量-产品标签
+// 增量-产品标签
 func createIncreaseNameTag(dealer_id string, p_list []string) {
 	unique := map[string]string{}
 	for _, v := range p_list {
@@ -198,7 +184,7 @@ func createIncreaseNameTag(dealer_id string, p_list []string) {
 	}
 }
 
-//增量-科室标签
+// 增量-科室标签
 func createIncreaseDepartTag(dealer_id string, p_list []string) {
 	unique := map[string]string{}
 	for _, v := range p_list {
@@ -219,7 +205,7 @@ func createIncreaseDepartTag(dealer_id string, p_list []string) {
 	}
 }
 
-//去重-标签信息
+// 去重-标签信息
 func calculateIncreaseRepeatTagInfo(dealer_id string, coll_name string, unique map[string]string) []string {
 	new_list := []string{}
 	datas := ul.MysqlSubjectTool.Find(coll_name, map[string]interface{}{"dealer_id": dealer_id}, "labelvalues", "", -1, -1)

data_ylqx/src/dimension/dimension_tag.go

@@ -67,12 +67,10 @@ func initDepartCode() {
 }
 
 func StartDealWithDealerDimensionTagInfo() {
-
 	//获取经销商id
 	initDealerNameId()
 	initAreaCode()
 	initDepartCode()
-
 	sess := ul.Save_Mgo.GetMgoConn()
 	defer ul.Save_Mgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{}

data_ylqx/src/dimension/dimension_udp.go

@@ -0,0 +1,54 @@
+package dimension
+
+import (
+	"encoding/json"
+	log "github.com/donnie4w/go-logger/logger"
+	mu "mfw/util"
+	"net"
+	qu "qfw/util"
+	"sync"
+)
+
+var (
+	udpclient mu.UdpClient
+	udpLock   sync.Mutex
+)
+
+func initDimensionUdpMsg() {
+	port := ":1583"
+	udpclient = mu.UdpClient{Local: port, BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	log.Debug("监听~", port)
+}
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case mu.OP_TYPE_DATA:
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		if err != nil {
+			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			sid, eid := qu.ObjToString(mapInfo["gtid"]), qu.ObjToString(mapInfo["lteid"])
+			stype := qu.ObjToString(mapInfo["stype"])
+			if stype == "monitor" {
+				log.Debug("收到监测......")
+				key := qu.ObjToString(mapInfo["key"])
+				udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
+				return
+			}
+			if sid == "" || eid == "" {
+				log.Debug("异常~", sid, "~", eid)
+				
+			} else {
+				key := sid + "-" + eid + "-" + qu.ObjToString(mapInfo["stype"])
+				udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
+				udpLock.Lock()
+				dealWithIncreaseCoverDepartProTag(sid, eid)
+				udpLock.Unlock()
+			}
+		}
+	case mu.OP_NOOP: //下个节点回应
+		log.Debug("接收~", string(data))
+	}
+}

data_ylqx/src/main.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"figure"
+	"dimension"
 	"fmt"
 	log "github.com/donnie4w/go-logger/logger"
 	"go.mongodb.org/mongo-driver/bson"
@@ -12,8 +12,9 @@ import (
 func init() {
 	//临时测试tidb
 	//ul.IsDev = true
-	ul.IsLocal = true
+	//ul.IsLocal = true
 	ul.InitClass()
+	//log.Debug("测试接口...", ul.PostReliability("布展工程"))
 }
 
 func main() {
@@ -27,21 +28,25 @@ func main() {
 	//dimension.StartBuildDealerDimensionGroupInfo()
 	//dimension.StartSupplyDealerDimensionProductInfo()
 	//dimension.StartDealWithDealerDimensionTagInfo()
-	//dimension.DimensionAddTimedTask()
 
-	//标的物中标信息
+	//标的物与中标信息
 	//bidding.StartDealWithPurashingInfo()
-	//bidding.StartDealWithWinnerInfo()
-	//bidding_add.BiddingAddTimedTask()
-
 	//关系型数据
 	//relation.StartDealWithRelationBuyerWinnerInfo()
-	//relation.StartDealWithRelationWinnerInfo()
-	//relation_add.RelationAddTimedTask()
 
-	//T.RepairPurchasingJyHref()
+	//标的物与中标信息与关系型-增量部署
+	//bidding_add.BiddingRelationAddTask()
+	//经销商相关联标签-增量部署
+	dimension.DimensionAddTask()
+
+	//code, info := bidding.ComparisonItemNameInfo("宫腔检查镜")
+	//log.Debug(code, info)
 
-	figure.GoValueateTest()
+	//导出疑似问题数据-
+	//test2.ExportLabelErrBidInfo()
+	//test2.ExportWarningBidInfo()
+
+	//relation.StartDealWithRelationBuyerWinnerInfo()
 
 	//保持活性
 	lock := make(chan bool)

data_ylqx/src/relation/relation.go

@@ -18,15 +18,17 @@ var (
 	YlWinnerStatistics      = "dws_f_yl_winner_statistics"
 )
 
-//采购单位~中标单位关系表
+// 采购单位~中标单位关系表
 func StartDealWithRelationBuyerWinnerInfo() {
-	//项目标的物信息~中标信息
+	//设备代码，经销商信息
+	initEquipmentCode()
 	initDealerInfos()
+	//项目标的物信息~中标信息
 	initProjectInfos()
 	initWinnerInfos()
+	//构建唯一性...
 	initBuyerWinnerInfos()
-	initEquipmentDepartCode()
-	log.Debug("待处理数据~", len(buyer_winner_index), "条")
+	time.Sleep(2 * time.Second)
 	for k, v := range buyer_winner_index {
 		if k%1000 == 0 {
 			log.Debug("cur index ", k)
@@ -53,14 +55,34 @@ func StartDealWithRelationBuyerWinnerInfo() {
 			info["cooperat_productclass"] = p_codes
 			info["cooperat_depart"] = p_depart_codes
 			info["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+			info["updatetime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
 			ul.InsertMysqlSubjectData(YlBuyerWinnerRelation, info, "")
 		}
 	}
 	log.Debug("is over ...", len(buyer_winner_index))
 	time.Sleep(time.Second * 2)
+
+	log.Debug("待处理中标中标数据~", len(winnerIds), "条")
+	for k, v := range winnerIds {
+		if k%1000 == 0 {
+			log.Debug("cur index ", k)
+		}
+		win_infos := winnerInfos[v] //is_winner 有1也有2
+		if len(win_infos) > 0 {
+			//重构中标信息组
+			new_win_infos := refactorWinnersInfo(win_infos)
+			if len(new_win_infos) > 0 {
+				final_win_infos := createWinnerInfo(new_win_infos, v)
+				ul.InsertMysqlSubjectData(YlWinnerStatistics, final_win_infos, "")
+			}
+		}
+	}
+	log.Debug("is over ...", len(winnerIds))
+	time.Sleep(time.Second * 2)
+
 }
 
-//查询标的物相关信息
+// 查询标的物相关信息
 func queryPurchasingAllInfos(winner_id string, buyer_id string) (string, string, string) {
 	win_infos := winnerInfos[winner_id]
 	project_ids := []string{}
@@ -83,7 +105,7 @@ func queryPurchasingAllInfos(winner_id string, buyer_id string) (string, string,
 					new_buyer_id := qu.ObjToString(v1["buyer_id"])
 					if new_buyer_id == buyer_id && new_buyer_id != "" {
 						itemname := qu.ObjToString(v1["itemname"])
-						if utf8.RuneCountInString(itemname) > 30 {
+						if utf8.RuneCountInString(itemname) > 50 || qu.Float64All(v1["reliability"]) < ul.S_Reality {
 							continue
 						}
 						sdequipment_code := qu.ObjToString(v1["sdequipment_code"])
@@ -128,7 +150,7 @@ func queryPurchasingAllInfos(winner_id string, buyer_id string) (string, string,
 	return new_p_names, new_p_codes, new_depart_codes
 }
 
-//查询~最后中标时间~以及合作次数
+// 查询~最后中标时间~以及合作次数
 func queryLastTimeAndCooperatNum(buyer_id string, winner_id string) (string, int) {
 	win_infos := winnerInfos[winner_id]
 	project_ids := []string{}

data_ylqx/src/relation/relation_init.go

@@ -34,6 +34,7 @@ var (
 )
 
 func initDealerInfos() {
+	dealerInfos = map[string]string{}
 	datas := *ul.MysqlSubjectTool.Find(YlDealerBaseInfo, nil,
 		"dealer_name", "id", -1, -1)
 	for _, v := range datas {
@@ -48,7 +49,7 @@ func initDealerInfos() {
 //构建项目信息
 func initProjectInfos() {
 	datas := *ul.MysqlSubjectTool.Find(YlPurchasingBaseInfo, nil,
-		"buyer,buyer_id,projectid,itemname,sdequipment_code,mi_level_code,jgtime", "id", -1, -1)
+		"", "id", -1, -1)
 	for _, v := range datas {
 		jgtime := qu.ObjToString(v["jgtime"])
 		if jgtime == "" {
@@ -75,7 +76,6 @@ func initProjectInfos() {
 		}
 	}
 	log.Debug("最终项目数量~医院等级~", len(datas), "~", len(projectIds), "~", len(medicalLevel))
-
 }
 
 //中标信息
@@ -117,9 +117,13 @@ func initWinnerInfos() {
 
 //buyer-winner唯一
 func initBuyerWinnerInfos() {
-	log.Debug("构建buyer与winner的唯一性")
-	for _, v := range projectIds {
-		infos := projectInfos[v]
+	log.Debug("构建buyer与winner的唯一性...")
+	for _, p_id := range projectIds {
+		infos := projectInfos[p_id]
+		//计算是否含有效标的物
+		if !IsReliability(infos) {
+			continue
+		}
 		buyer, buyer_id := "", ""
 		for _, info := range infos {
 			if qu.ObjToString(info["buyer_id"]) != "" {
@@ -131,7 +135,7 @@ func initBuyerWinnerInfos() {
 		if buyer_id == "" {
 			continue
 		}
-		winners := projectWins[v]
+		winners := projectWins[p_id]
 		for _, win_value := range winners {
 			winner := qu.ObjToString(win_value["winner"])
 			winner_id := qu.ObjToString(win_value["winner_id"])
@@ -151,7 +155,7 @@ func initBuyerWinnerInfos() {
 			}
 		}
 	}
-	log.Debug("唯一性构建完毕~", len(buyer_winner_index))
+	log.Debug("唯一性构建完毕...", len(buyer_winner_index))
 }
 
 //医院对应等级代码
@@ -167,7 +171,7 @@ func initBuyerWinnerInfos() {
 //}
 
 //设备对应科室代码
-func initEquipmentDepartCode() {
+func initEquipmentCode() {
 	datas := *ul.MysqlSubjectTool.Find(YlSddepartProClass, nil,
 		"equipment,sddepartname_code", "id", -1, -1)
 	for _, v := range datas {
@@ -180,3 +184,14 @@ func initEquipmentDepartCode() {
 	}
 	log.Debug("设备对应的科室代码~", len(equipmentDeparts))
 }
+
+//计算此项目是否有效
+func IsReliability(infos []map[string]interface{}) bool {
+	for _, v := range infos {
+		reliability := qu.Float64All(v["reliability"])
+		if reliability >= ul.S_Reality {
+			return true
+		}
+	}
+	return false
+}

data_ylqx/src/relation/relation_winner.go

@@ -1,48 +1,28 @@
 package relation
 
 import (
-	log "github.com/donnie4w/go-logger/logger"
 	qu "qfw/util"
 	"time"
 	ul "util"
 )
 
-func StartDealWithRelationWinnerInfo() {
-	initDealerInfos()
-	initProjectInfos()
-	initWinnerInfos()
-	log.Debug("待处理数据~", len(winnerIds), "条")
-	for k, v := range winnerIds {
-		if k%1000 == 0 {
-			log.Debug("cur index ", k)
-		}
-		infos := winnerInfos[v]
-		if len(infos) > 0 {
-			//重构中标信息组
-			new_infos := refactorWinnersInfo(infos)
-			if len(new_infos) > 0 {
-				win_info := createWinnerInfo(new_infos, v)
-				ul.InsertMysqlSubjectData(YlWinnerStatistics, win_info, "")
-			}
-		}
-	}
-	log.Debug("is over ...", len(winnerIds))
-	time.Sleep(time.Second * 2)
-}
-
 //重构~中标信息组
-func refactorWinnersInfo(infos []map[string]interface{}) []map[string]interface{} {
+func refactorWinnersInfo(win_infos []map[string]interface{}) []map[string]interface{} {
 	project_unique := map[string]string{}
 	new_infos := []map[string]interface{}{}
-	for _, v := range infos {
+	for _, v := range win_infos {
 		is_winner := qu.IntAll(v["is_winner"])
 		projectid := qu.ObjToString(v["projectid"])
+		//计算该项目是否合理有效
+		if !IsReliability(projectInfos[projectid]) {
+			continue
+		}
 		if project_unique[projectid] == "" {
 			if is_winner == 1 {
 				new_infos = append(new_infos, v)
 			} else { //遍历数据~优先级 is_winner==1
 				isExists := false
-				for _, n_v := range infos {
+				for _, n_v := range win_infos {
 					n_is_winner := qu.IntAll(n_v["is_winner"])
 					n_projectid := qu.ObjToString(n_v["projectid"])
 					if projectid == n_projectid && n_is_winner == 1 {
@@ -118,6 +98,7 @@ func createWinnerInfo(infos []map[string]interface{}, winner_id string) map[stri
 		win_info["lastwintime"] = lastwintime
 	}
 	win_info["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+	win_info["updatetime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
 
 	return win_info
 }

data_ylqx/src/relation_add/relation_add.go

@@ -1,7 +1,6 @@
 package relation_add
 
 import (
-	"github.com/cron"
 	log "github.com/donnie4w/go-logger/logger"
 	qu "qfw/util"
 	"relation"
@@ -10,38 +9,20 @@ import (
 )
 
 var (
-	up_winner_ids        []string
-	up_winner_infos      map[string][]map[string]interface{}
-	up_project_ids       []string
-	up_project_infos     map[string][]map[string]interface{}
+	//构建的信息
+	bw_win_ids, bw_pro_ids                       []string
+	bw_win_infos, bw_pro_infos, bw_pro_win_infos map[string][]map[string]interface{}
+
+	nw_win_ids, nw_pro_ids     []string
+	nw_win_infos, nw_pro_infos map[string][]map[string]interface{}
+
 	up_med_level         map[string]string
 	up_equipment_departs map[string][]string
-
 	//经销商信息
 	up_dealerInfos = map[string]string{}
 )
 
-//定时任务~
-func RelationAddTimedTask() {
-	c := cron.New()
-	c.AddFunc("0 0 4 ? * *", func() {
-		getQueryTimeSql()
-	})
-	c.Start()
-}
-
-//获取查询语句
-func getQueryTimeSql() {
-	now := time.Now()
-	sql_time := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Format(ul.TimeLayout)
-	log.Debug("获取前一天凌晨的sql语句~~~", sql_time)
-	query := map[string]interface{}{
-		"updatetime": map[string]string{
-			"gte": sql_time,
-		},
-	}
-	RelationIncreaseInfo(query)
-}
+// 经销商信息
 func initDealerInfos() {
 	datas := *ul.MysqlSubjectTool.Find(relation.YlDealerBaseInfo, nil,
 		"dealer_name", "id", -1, -1)
@@ -54,7 +35,7 @@ func initDealerInfos() {
 	log.Debug("经销商数量~", len(up_dealerInfos))
 }
 
-//科室设备对应代码
+// 科室设备对应代码
 func initEquipmentDepartCode() {
 	datas := *ul.MysqlSubjectTool.Find(relation.YlSddepartProClass, nil,
 		"equipment,sddepartname_code", "id", -1, -1)
@@ -68,133 +49,83 @@ func initEquipmentDepartCode() {
 	log.Debug("设备对应的科室代码~", len(up_equipment_departs))
 }
 func initResetVar() {
-	up_winner_ids = []string{}
-	up_winner_infos = map[string][]map[string]interface{}{}
-	up_project_ids = []string{}
-	up_project_infos = map[string][]map[string]interface{}{}
+	bw_win_ids, bw_pro_ids = []string{}, []string{}
+	bw_win_infos, bw_pro_infos, bw_pro_win_infos = map[string][]map[string]interface{}{}, map[string][]map[string]interface{}{}, map[string][]map[string]interface{}{}
+
+	nw_win_ids, nw_pro_ids = []string{}, []string{}
+	nw_win_infos, nw_pro_infos = map[string][]map[string]interface{}{}, map[string][]map[string]interface{}{}
+
 	up_med_level = map[string]string{}
 	up_equipment_departs = map[string][]string{}
 	up_dealerInfos = map[string]string{}
 }
 
-//增量统计更新关系表
-func RelationIncreaseInfo(query map[string]interface{}) {
+// 增量统计更新关系表
+func GetRelationIncrementalInfo(mod_buyer []string, mod_winner []string) {
 	initResetVar()
 	initEquipmentDepartCode()
 	initDealerInfos()
-
-	//取项目相关信息
-	project_ids := findUpdateProjectIds(query)
-	project_infos := findUpdateProjectInfos(project_ids)
-	findUpdateWinnerIds(project_ids)
-	p_win_infos := findUpdateWinnerInfos()
-	log.Debug("关联中标标识~", len(up_winner_ids), "~信息~", len(up_winner_infos))
-	up_bw_index, up_bw_infos := findBuyerAndWinnerUniqueness(project_ids, project_infos, p_win_infos)
-	log.Debug("关系唯一性~", len(up_bw_index), "~等级~", len(up_med_level), "~临时项目~", len(p_win_infos))
-	findUpdateAllProjectInfos()
-	log.Debug("关联项目标识~", len(up_project_ids), "~信息~", len(up_project_infos))
-
+	//根据修改的buyer,先删除所有b_w关系数据,在构建b_w的唯一性
+	constructModBueryInfos(mod_buyer)
+	bw_index, bw_infos := constructModBueryUniqueness()
+	log.Debug("b_w ... ", len(bw_index), "~", len(bw_infos))
 	//更新操作buyer_winner_relation
-	updateBuyerWinnerInfoRelation(up_bw_index, up_bw_infos)
+	updateBuyerWinnerRelationInfo(bw_index, bw_infos)
+	//根据修改的winner,先删除所有win的关系数据，在构建信息的
+	constructModWinnerInfos(mod_winner)
+	log.Debug("n_w ... ", len(nw_win_ids))
 	//更新操作winner_relation
-	updateWinnerInfoRelation()
-	log.Debug("更新完毕~", len(up_bw_index), "~", len(up_winner_ids))
-
-	log.Debug("处理完毕释放...")
+	updateWinnerRelationInfo()
+	//释放内存
 	initResetVar()
 }
 
-//找项目标识
-func findUpdateProjectIds(query map[string]interface{}) []string {
-	up_ids := []string{}
-	log.Debug("查询语句~", query)
-	datas := *ul.MysqlSubjectTool.Find(relation.YlPurchasingWinBaseInfo, query,
-		"projectid,jgtime", "id", -1, -1)
-	unique_id := map[string]string{}
-	for _, v := range datas {
-		jgtime := qu.ObjToString(v["jgtime"])
-		if jgtime == "" {
-			continue
-		}
-		t, _ := time.ParseInLocation(ul.TimeLayout, jgtime, time.Local)
-		if t.Unix() < int64(1514736000) {
-			continue
-		}
-		projectid := qu.ObjToString(v["projectid"])
-		if unique_id[projectid] == "" && projectid != "" {
-			unique_id[projectid] = projectid
-			up_ids = append(up_ids, projectid)
-		}
+// 构建 B_W 的基础信息
+func constructModBueryInfos(mod_buyer []string) {
+	//删除buyer有关的关系数据
+	for _, v := range mod_buyer {
+		ul.MysqlSubjectTool.Delete(relation.YlBuyerWinnerRelation, map[string]interface{}{"buyer": v})
 	}
-	return up_ids
-}
-
-//找项目信息
-func findUpdateProjectInfos(up_ids []string) map[string][]map[string]interface{} {
-	up_infos := map[string][]map[string]interface{}{}
-	for _, p_id := range up_ids {
-		datas := *ul.MysqlSubjectTool.Find(relation.YlPurchasingBaseInfo, map[string]interface{}{
-			"projectid": p_id,
-		}, "buyer,buyer_id,mi_level_code,jgtime", "id", -1, -1)
-		for _, v := range datas {
-			//jgtime := qu.ObjToString(v["jgtime"])
-			//if jgtime == "" {
-			//	continue
-			//}
-			//t, _ := time.ParseInLocation(ul.TimeLayout, jgtime, time.Local)
-			//if t.Unix() < int64(1514736000) {
-			//	continue
-			//}
-			infoArr := []map[string]interface{}{}
-			if up_infos[p_id] != nil {
-				infoArr = up_infos[p_id]
-			}
-			infoArr = append(infoArr, v)
-			up_infos[p_id] = infoArr
-			buyer_id := qu.ObjToString(v["buyer_id"])
-			mi_level_code := qu.ObjToString(v["mi_level_code"])
-			if buyer_id != "" {
-				up_med_level[buyer_id] = mi_level_code
+	un_p_ids := map[string]string{}
+	for _, v := range mod_buyer {
+		datas := ul.MysqlSubjectTool.Find(relation.YlPurchasingBaseInfo, map[string]interface{}{
+			"buyer": v,
+		}, "", "id", -1, -1)
+		if datas != nil {
+			for _, v1 := range *datas {
+				jgtime := qu.ObjToString(v1["jgtime"])
+				if jgtime == "" {
+					continue
+				}
+				t, _ := time.ParseInLocation(ul.TimeLayout, jgtime, time.Local)
+				if t.Unix() < int64(1514736000) {
+					continue
+				}
+				//项目标识
+				p_id := qu.ObjToString(v1["projectid"])
+				if un_p_ids[p_id] == "" {
+					un_p_ids[p_id] = p_id
+					bw_pro_ids = append(bw_pro_ids, p_id)
+				}
+				//项目标识-项目信息
+				infoArr := []map[string]interface{}{}
+				if bw_pro_infos[p_id] != nil {
+					infoArr = bw_pro_infos[p_id]
+				}
+				infoArr = append(infoArr, v1)
+				bw_pro_infos[p_id] = infoArr
+				//等级代码
+				if buyer_id := qu.ObjToString(v1["buyer_id"]); buyer_id != "" {
+					up_med_level[buyer_id] = qu.ObjToString(v1["mi_level_code"])
+				}
 			}
 		}
 	}
-	return up_infos
-}
-
-//找中标标识
-func findUpdateWinnerIds(up_ids []string) {
-	for _, p_id := range up_ids {
+	un_w_ids := map[string]string{}
+	for _, p_id := range bw_pro_ids {
 		//找所有项目下的中标单位
 		datas := *ul.MysqlSubjectTool.Find(relation.YlPurchasingWinBaseInfo, map[string]interface{}{
 			"projectid": p_id,
-		}, "winner_id,jgtime,winner", "id", -1, -1)
-		unique_id := map[string]string{}
-		for _, v := range datas {
-			jgtime := qu.ObjToString(v["jgtime"])
-			if jgtime == "" {
-				continue
-			}
-			t, _ := time.ParseInLocation(ul.TimeLayout, jgtime, time.Local)
-			if t.Unix() < int64(1514736000) {
-				continue
-			}
-			winner_id := qu.ObjToString(v["winner_id"])
-			winner := qu.ObjToString(v["winner"])
-			if unique_id[winner_id] == "" && winner_id != "" && up_dealerInfos[winner] != "" {
-				unique_id[winner_id] = winner_id
-				up_winner_ids = append(up_winner_ids, winner_id)
-			}
-		}
-	}
-}
-
-//找中标标识,中标信息,对应项目信息
-func findUpdateWinnerInfos() map[string][]map[string]interface{} {
-	p_win_infos := map[string][]map[string]interface{}{}
-	for _, w_id := range up_winner_ids {
-		//中标标识~对应所有的中标单位
-		datas := *ul.MysqlSubjectTool.Find(relation.YlPurchasingWinBaseInfo, map[string]interface{}{
-			"winner_id": w_id,
 		}, "", "id", -1, -1)
 		for _, v := range datas {
 			jgtime := qu.ObjToString(v["jgtime"])
@@ -205,44 +136,49 @@ func findUpdateWinnerInfos() map[string][]map[string]interface{} {
 			if t.Unix() < int64(1514736000) {
 				continue
 			}
-			winner_id := qu.ObjToString(v["winner_id"])
 			winner := qu.ObjToString(v["winner"])
+			winner_id := qu.ObjToString(v["winner_id"])
 			projectid := qu.ObjToString(v["projectid"])
 			if winner_id != "" && winner != "" {
-				//中标标识~对应的中标信息
+				//中标标识
+				if un_w_ids[winner_id] == "" && winner_id != "" && up_dealerInfos[winner] != "" {
+					un_w_ids[winner_id] = winner_id
+					bw_win_ids = append(bw_win_ids, winner_id)
+				}
+				//中标标识-中标信息
 				w_infos := []map[string]interface{}{}
-				if up_winner_infos[winner_id] != nil {
-					w_infos = up_winner_infos[winner_id]
+				if bw_win_infos[winner_id] != nil {
+					w_infos = bw_win_infos[winner_id]
 				}
 				w_infos = append(w_infos, v)
-				up_winner_infos[winner_id] = w_infos
-
-				//项目标识~对应的中标信息
-				if projectid != "" {
-					p_infos := []map[string]interface{}{}
-					if p_win_infos[projectid] != nil {
-						p_infos = p_win_infos[projectid]
-					}
-					p_infos = append(p_infos, v)
-					p_win_infos[projectid] = p_infos
+				bw_win_infos[winner_id] = w_infos
+				//项目标识-中标信息
+				p_infos := []map[string]interface{}{}
+				if bw_pro_win_infos[projectid] != nil {
+					p_infos = bw_pro_win_infos[projectid]
 				}
+				p_infos = append(p_infos, v)
+				bw_pro_win_infos[projectid] = p_infos
 			}
 		}
 	}
-	return p_win_infos
 }
 
-//找唯一性信息
-func findBuyerAndWinnerUniqueness(up_ids []string, up_infos map[string][]map[string]interface{}, p_win_infos map[string][]map[string]interface{}) ([]string, map[string]map[string]string) {
+// 构建 B_W 唯一性信息
+func constructModBueryUniqueness() ([]string, map[string]map[string]string) {
 	up_bw_index := []string{}
 	up_bw_infos := map[string]map[string]string{}
-	for _, v := range up_ids {
-		infos := up_infos[v]
-		buyer, buyer_id, b := findBuyerInfo(infos)
+	for _, v := range bw_pro_ids {
+		infos := bw_pro_infos[v]
+		//计算是否含有效标的物
+		if !relation.IsReliability(infos) {
+			continue
+		}
+		buyer, buyer_id, b := findOneBuyerInfo(infos)
 		if !b {
 			continue
 		}
-		winners := p_win_infos[v]
+		winners := bw_pro_win_infos[v]
 		for _, win_value := range winners {
 			winner := qu.ObjToString(win_value["winner"])
 			winner_id := qu.ObjToString(win_value["winner_id"])
@@ -265,8 +201,8 @@ func findBuyerAndWinnerUniqueness(up_ids []string, up_infos map[string][]map[str
 	return up_bw_index, up_bw_infos
 }
 
-//找采购单位
-func findBuyerInfo(infos []map[string]interface{}) (string, string, bool) {
+// 找采购单位
+func findOneBuyerInfo(infos []map[string]interface{}) (string, string, bool) {
 	buyer, buyer_id := "", ""
 	for _, info := range infos {
 		if qu.ObjToString(info["buyer_id"]) != "" {
@@ -282,23 +218,57 @@ func findBuyerInfo(infos []map[string]interface{}) (string, string, bool) {
 	return buyer, buyer_id, b
 }
 
-//找所有相关联~项目的信息
-func findUpdateAllProjectInfos() {
-	project_unique := map[string]string{}
-	for _, winner_id := range up_winner_ids {
-		win_infos := up_winner_infos[winner_id]
-		for _, v := range win_infos {
-			projectid := qu.ObjToString(v["projectid"])
-			if project_unique[projectid] == "" && projectid != "" {
-				up_project_ids = append(up_project_ids, projectid)
-				project_unique[projectid] = projectid
+// 找所有相关联~项目的信息
+func constructModWinnerInfos(mod_winner []string) {
+	//删除winner有关的关系数据
+	for _, v := range mod_winner {
+		ul.MysqlSubjectTool.Delete(relation.YlWinnerStatistics, map[string]interface{}{"winner": v})
+	}
+	//找所有中标信息
+	un_w_ids, un_p_ids := map[string]string{}, map[string]string{}
+	for _, v := range mod_winner {
+		//找所有项目下的中标单位
+		datas := *ul.MysqlSubjectTool.Find(relation.YlPurchasingWinBaseInfo, map[string]interface{}{
+			"winner": v,
+		}, "", "id", -1, -1)
+		for _, v1 := range datas {
+			jgtime := qu.ObjToString(v1["jgtime"])
+			if jgtime == "" {
+				continue
+			}
+			t, _ := time.ParseInLocation(ul.TimeLayout, jgtime, time.Local)
+			if t.Unix() < int64(1514736000) {
+				continue
+			}
+			winner := qu.ObjToString(v1["winner"])
+			winner_id := qu.ObjToString(v1["winner_id"])
+			projectid := qu.ObjToString(v1["projectid"])
+			if winner_id != "" && winner != "" {
+				//中标标识
+				if un_w_ids[winner_id] == "" && winner_id != "" && up_dealerInfos[winner] != "" {
+					un_w_ids[winner_id] = winner_id
+					nw_win_ids = append(nw_win_ids, winner_id)
+				}
+				//项目标识
+				if un_p_ids[projectid] == "" && projectid != "" && up_dealerInfos[winner] != "" {
+					un_p_ids[projectid] = projectid
+					nw_pro_ids = append(nw_pro_ids, projectid)
+				}
+				//中标标识-中标信息
+				w_infos := []map[string]interface{}{}
+				if nw_win_infos[winner_id] != nil && len(nw_win_infos[winner_id]) > 0 {
+					w_infos = nw_win_infos[winner_id]
+				}
+				w_infos = append(w_infos, v1)
+				nw_win_infos[winner_id] = w_infos
 			}
 		}
 	}
-	for _, p_id := range up_project_ids {
+	log.Debug("is nw_pro_ids ", len(nw_pro_ids))
+	for _, p_id := range nw_pro_ids {
 		p_infos := *ul.MysqlSubjectTool.Find(relation.YlPurchasingBaseInfo, map[string]interface{}{
 			"projectid": p_id,
-		}, "buyer,buyer_id,projectid,itemname,sdequipment_code,mi_level_code,jgtime", "id", -1, -1)
+		}, "", "id", -1, -1)
 		new_p_infos := []map[string]interface{}{}
 		for _, p_v := range p_infos {
 			jgtime := qu.ObjToString(p_v["jgtime"])
@@ -311,6 +281,7 @@ func findUpdateAllProjectInfos() {
 			}
 			new_p_infos = append(new_p_infos, p_v)
 		}
-		up_project_infos[p_id] = new_p_infos
+		nw_pro_infos[p_id] = new_p_infos
 	}
+
 }

data_ylqx/src/relation_add/relation_method.go

@@ -9,9 +9,9 @@ import (
 	ul "util"
 )
 
-//最后中标时间~以及合作次数
+// 最后中标时间~以及合作次数
 func updateLastTimeAndCooperatNum(buyer_id string, winner_id string) (string, int) {
-	win_infos := up_winner_infos[winner_id]
+	win_infos := bw_win_infos[winner_id]
 	project_ids := []string{}
 	project_times := []string{}
 	project_timestamp := []int64{}
@@ -36,7 +36,7 @@ func updateLastTimeAndCooperatNum(buyer_id string, winner_id string) (string, in
 	cooperat_unique := map[string]string{}
 	if len(project_ids) > 0 {
 		for k, p_id := range project_ids {
-			p_infos := up_project_infos[p_id]
+			p_infos := bw_pro_infos[p_id]
 			if len(p_infos) > 0 {
 				new_buyer_id := ""
 				for _, p_v := range p_infos {
@@ -60,9 +60,9 @@ func updateLastTimeAndCooperatNum(buyer_id string, winner_id string) (string, in
 	return lastwintime, len(cooperat_unique)
 }
 
-//标的物相关信息
+// 标的物相关信息
 func updatePurchasingAllInfos(winner_id string, buyer_id string) (string, string, string) {
-	win_infos := up_winner_infos[winner_id]
+	win_infos := bw_win_infos[winner_id]
 	project_ids := []string{}
 	for _, v := range win_infos {
 		is_winner := qu.IntAll(v["is_winner"])
@@ -77,13 +77,13 @@ func updatePurchasingAllInfos(winner_id string, buyer_id string) (string, string
 	p_codes_unique := map[string]string{}
 	if len(project_ids) > 0 {
 		for _, p_id := range project_ids {
-			p_infos := up_project_infos[p_id]
+			p_infos := bw_pro_infos[p_id]
 			if len(p_infos) > 0 {
 				for _, p_v := range p_infos {
 					new_buyer_id := qu.ObjToString(p_v["buyer_id"])
 					if new_buyer_id == buyer_id && new_buyer_id != "" {
 						itemname := qu.ObjToString(p_v["itemname"])
-						if utf8.RuneCountInString(itemname) > 30 {
+						if utf8.RuneCountInString(itemname) > 50 || qu.Float64All(p_v["reliability"]) < ul.S_Reality {
 							continue
 						}
 						sdequipment_code := qu.ObjToString(p_v["sdequipment_code"])
@@ -128,7 +128,7 @@ func updatePurchasingAllInfos(winner_id string, buyer_id string) (string, string
 	return new_p_names, new_p_codes, new_depart_codes
 }
 
-//重构~中标信息组
+// 重构~中标信息组
 func updateRefactorWinnersInfo(infos []map[string]interface{}) []map[string]interface{} {
 	project_unique := map[string]string{}
 	new_infos := []map[string]interface{}{}
@@ -159,7 +159,103 @@ func updateRefactorWinnersInfo(infos []map[string]interface{}) []map[string]inte
 	return new_infos
 }
 
-//查询是否更新
+// 重构~中标信息组
+func refactorWinnersInfo(win_infos []map[string]interface{}) []map[string]interface{} {
+	project_unique := map[string]string{}
+	new_infos := []map[string]interface{}{}
+	for _, v := range win_infos {
+		is_winner := qu.IntAll(v["is_winner"])
+		projectid := qu.ObjToString(v["projectid"])
+		//计算该项目是否合理有效
+		//if !relation.IsReliability(nw_pro_infos[projectid]) {
+		//	continue
+		//}
+		if project_unique[projectid] == "" {
+			if is_winner == 1 {
+				new_infos = append(new_infos, v)
+			} else { //遍历数据~优先级 is_winner==1
+				isExists := false
+				for _, n_v := range win_infos {
+					n_is_winner := qu.IntAll(n_v["is_winner"])
+					n_projectid := qu.ObjToString(n_v["projectid"])
+					if projectid == n_projectid && n_is_winner == 1 {
+						isExists = true
+						new_infos = append(new_infos, v)
+						break
+					}
+				}
+				if !isExists {
+					new_infos = append(new_infos, v)
+				}
+			}
+			project_unique[projectid] = projectid
+		}
+	}
+	return new_infos
+}
+
+// 构建中标关系信息
+func createWinnerInfo(infos []map[string]interface{}, winner_id string) map[string]interface{} {
+	win_info := map[string]interface{}{}
+	win_info["winner"] = infos[0]["winner"]
+	win_info["winner_id"] = winner_id
+	join_count, win_count := 0, 0
+	contact_num := 0
+	bidamount := float64(0)
+	lastwintime := ""
+	cooperat_ins_num := 0
+	contact_unique := map[string]string{}
+	buyer_id_unique := map[string]string{}
+	for _, info := range infos {
+		join_count++
+		if qu.IntAll(info["is_winner"]) == 1 {
+			win_count++
+		}
+		per := qu.ObjToString(info["contact_name"])
+		tel := qu.ObjToString(info["contact_tel"])
+		if per != "" && tel != "" {
+			contact_key := per + "~" + tel
+			if contact_unique[contact_key] == "" {
+				contact_unique[contact_key] = contact_key
+				contact_num++
+			}
+		}
+		bidamount += qu.Float64All(info["bidamount"])
+		if qu.ObjToString(info["jgtime"]) != "" {
+			lastwintime = qu.ObjToString(info["jgtime"])
+		}
+		projectid := qu.ObjToString(info["projectid"])
+		pro_infos := nw_pro_infos[projectid]
+		if len(pro_infos) > 0 {
+			buyer_id := ""
+			for _, pro_info := range pro_infos {
+				if qu.ObjToString(pro_info["buyer_id"]) != "" {
+					buyer_id = qu.ObjToString(pro_info["buyer_id"])
+					break
+				}
+			}
+			if buyer_id != "" {
+				if buyer_id_unique[buyer_id] == "" {
+					buyer_id_unique[buyer_id] = buyer_id
+					cooperat_ins_num++
+				}
+			}
+		}
+	}
+	win_info["join_count"] = join_count
+	win_info["win_count"] = win_count
+	win_info["contact_num"] = contact_num
+	win_info["cooperat_ins_num"] = cooperat_ins_num
+	win_info["bidamount"] = bidamount
+	if lastwintime != "" {
+		win_info["lastwintime"] = lastwintime
+	}
+	win_info["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+
+	return win_info
+}
+
+// 查询是否更新-暂时弃用
 func queryUpdateRelationId(query map[string]interface{}, tableName string) (bool, string) {
 	tmpid := ""
 	info := ul.MysqlSubjectTool.FindOne(tableName, query, "id", "id")

data_ylqx/src/relation_add/relation_update.go

@@ -8,10 +8,10 @@ import (
 	ul "util"
 )
 
-//更新buyer_winner 关系数据
-func updateBuyerWinnerInfoRelation(bw_index []string, bw_infos map[string]map[string]string) {
+// 更新 B_W 关系数据
+func updateBuyerWinnerRelationInfo(bw_index []string, bw_infos map[string]map[string]string) {
 	for k, v := range bw_index {
-		if k%10 == 0 {
+		if k%100 == 0 {
 			log.Debug("cur index ", k)
 		}
 		tmp := bw_infos[v]
@@ -20,19 +20,10 @@ func updateBuyerWinnerInfoRelation(bw_index []string, bw_infos map[string]map[st
 		winner := qu.ObjToString(tmp["winner"])
 		winner_id := qu.ObjToString(tmp["winner_id"])
 		info := map[string]interface{}{}
-		is_update := false
-		//查询是否更新~新增
-		is_update, update_id := queryUpdateRelationId(map[string]interface{}{
-			"buyer_id":  buyer_id,
-			"winner_id": winner_id,
-		}, relation.YlBuyerWinnerRelation)
-		//更新~指定信息~新增信息
-		if !is_update {
-			info["buyer"] = buyer
-			info["buyer_id"] = buyer_id
-			info["winner"] = winner
-			info["winner_id"] = winner_id
-		}
+		info["buyer"] = buyer
+		info["buyer_id"] = buyer_id
+		info["winner"] = winner
+		info["winner_id"] = winner_id
 		info["mi_level_code"] = qu.ObjToString(up_med_level[buyer_id])
 		//查询lastwintime最新中标时间 合作次数
 		lastwintime, cooperat_num := updateLastTimeAndCooperatNum(buyer_id, winner_id)
@@ -45,101 +36,36 @@ func updateBuyerWinnerInfoRelation(bw_index []string, bw_infos map[string]map[st
 			info["cooperat_productclass"] = p_codes
 			info["cooperat_depart"] = p_depart_codes
 		}
-		//更新操作
-		if is_update && update_id != "" {
-			info["updatetime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
-			ul.MysqlSubjectTool.Update(relation.YlBuyerWinnerRelation, map[string]interface{}{"id": update_id}, info)
-		} else {
-			info["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
-			ul.InsertMysqlSubjectData(relation.YlBuyerWinnerRelation, info, "")
-		}
-
+		info["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+		ul.InsertMysqlSubjectData(relation.YlBuyerWinnerRelation, info, "")
 	}
-	log.Debug("is b_w update over ", len(bw_index))
+	log.Debug("is b_w over ... ", len(bw_index))
 }
 
+/*
+*******************************
+***************分***************
+***************割***************
+***************线***************
+********************************
+ */
 //更新winner 关系数据
-func updateWinnerInfoRelation() {
-	for k, w_id := range up_winner_ids {
-		if k%10 == 0 {
+func updateWinnerRelationInfo() {
+	for k, v := range nw_win_ids {
+		if k%100 == 0 {
 			log.Debug("cur index ", k)
 		}
-		old_infos := up_winner_infos[w_id]
-		if len(old_infos) > 0 {
+		win_infos := nw_win_infos[v] //is_winner 有1也有2
+		if len(win_infos) > 0 {
 			//重构中标信息组
-			new_infos := updateRefactorWinnersInfo(old_infos)
-			if len(new_infos) > 0 {
-				win_info := map[string]interface{}{}
-				is_update := false
-				//是否更新操作
-				is_update, update_id := queryUpdateRelationId(map[string]interface{}{
-					"winner_id": w_id,
-				}, relation.YlWinnerStatistics)
-				if !is_update {
-					win_info["winner"] = new_infos[0]["winner"]
-					win_info["winner_id"] = w_id
-				}
-				join_count, win_count := 0, 0
-				contact_num := 0
-				bidamount := float64(0)
-				lastwintime := ""
-				cooperat_ins_num := 0
-				contact_unique := map[string]string{}
-				buyer_id_unique := map[string]string{}
-				for _, info := range new_infos {
-					join_count++
-					if qu.IntAll(info["is_winner"]) == 1 {
-						win_count++
-					}
-					per := qu.ObjToString(info["contact_name"])
-					tel := qu.ObjToString(info["contact_tel"])
-					if per != "" && tel != "" {
-						contact_key := per + "~" + tel
-						if contact_unique[contact_key] == "" {
-							contact_unique[contact_key] = contact_key
-							contact_num++
-						}
-					}
-					bidamount += qu.Float64All(info["bidamount"])
-					if qu.ObjToString(info["jgtime"]) != "" {
-						lastwintime = qu.ObjToString(info["jgtime"])
-					}
-					//取项目信息
-					projectid := qu.ObjToString(info["projectid"])
-					p_infos := up_project_infos[projectid]
-					if len(p_infos) > 0 {
-						buyer_id := ""
-						for _, p_info := range p_infos {
-							if qu.ObjToString(p_info["buyer_id"]) != "" {
-								buyer_id = qu.ObjToString(p_info["buyer_id"])
-								break
-							}
-						}
-						if buyer_id != "" {
-							if buyer_id_unique[buyer_id] == "" {
-								buyer_id_unique[buyer_id] = buyer_id
-								cooperat_ins_num++
-							}
-						}
-					}
-				}
-				win_info["join_count"] = join_count
-				win_info["win_count"] = win_count
-				win_info["contact_num"] = contact_num
-				win_info["cooperat_ins_num"] = cooperat_ins_num
-				win_info["bidamount"] = bidamount
-				if lastwintime != "" {
-					win_info["lastwintime"] = lastwintime
-				}
-				if is_update && update_id != "" {
-					win_info["updatetime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
-					ul.MysqlSubjectTool.Update(relation.YlWinnerStatistics, map[string]interface{}{"id": update_id}, win_info)
-				} else {
-					win_info["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
-					ul.InsertMysqlSubjectData(relation.YlWinnerStatistics, win_info, win_info)
-				}
+			new_win_infos := refactorWinnersInfo(win_infos)
+			if len(new_win_infos) > 0 {
+				final_win_infos := createWinnerInfo(new_win_infos, v)
+				ul.InsertMysqlSubjectData(relation.YlWinnerStatistics, final_win_infos, "")
 			}
 		}
 	}
-	log.Debug("is over ", len(up_winner_ids))
+	log.Debug("is n_w over ... ", len(nw_win_ids))
+	time.Sleep(time.Second * 2)
+	log.Debug("is n_w over ... ", len(nw_win_infos))
 }

data_ylqx/src/test/other.go

@@ -3,10 +3,174 @@ package test
 import (
 	log "github.com/donnie4w/go-logger/logger"
 	"github.com/tealeg/xlsx"
+	qu "qfw/util"
 	"strings"
 	ul "util"
 )
 
+//导出错误标注数据与看板数据
+func ExportLabelErrBidInfo() {
+	arr1 := []string{
+		"647daf58a1840869018fe768",
+		"647d32eceb01e8efa6318a77",
+		"64802ef6d6ab496933e66b76",
+		"6481b2618aea8786d11851ab",
+		"647fecc9cc93eebb8568132c",
+		"647ffc14cc93eebb856854b0",
+		"647e9704a18408690191ec0c",
+		"647eda978aea8786d1157272",
+		"648207aed6ab496933ecc251",
+		"647fb7e8d6ab496933e48ba4",
+		"647ec1e3eb01e8efa6351245",
+		"647edcb3a18408690193cae5",
+		"6481b714d6ab496933eb6465",
+		"647dac5d8aea8786d113e9ab",
+		"64812c138aea8786d11783dc",
+		"647d9a4aa1840869018f8709",
+		"647eedcfa1840869019416cb",
+		"647d4c60a1840869018e7586",
+		"64804d158aea8786d1170cd1",
+		"647ef96ea184086901946a7e",
+		"647eed8d8aea8786d1159a97",
+		"647e905aa18408690191b2e1",
+		"647ecbfba184086901935d7e",
+		"647d41f2a1840869018e4e51",
+		"647f2a5c8aea8786d1162132",
+		"6480553bd6ab496933e73136",
+		"64804d178aea8786d1170d2c",
+		"64804631cc93eebb85693e4b",
+	}
+	arr2 := []string{
+		"62d059db2f27cc9ca37bd90e",
+		"62d129292f27cc9ca37c6deb",
+		"62d0b3af2f27cc9ca37bf1fd",
+		"62d09b022f27cc9ca37bebef",
+		"62d070cd2f27cc9ca37bdf7d",
+		"62d0729c2f27cc9ca37bdfc3",
+		"62d06adf2f27cc9ca37bddf5",
+		"62d06b942f27cc9ca37bde0a",
+		"62d059db2f27cc9ca37bd90e",
+		"62d035a22f27cc9ca37bcd62",
+		"62d029432f27cc9ca37bc67f",
+		"62cf355a2f27cc9ca37ae761",
+		"62cf37202f27cc9ca37ae7d0",
+		"62e2cd682f27cc9ca3865c41",
+		"62e217682f27cc9ca385c8e4",
+		"62d930622f27cc9ca380cb46",
+		"62d8a8202f27cc9ca3802d61",
+		"62d43ac02f27cc9ca37d4b59",
+		"62d3ed4a2f27cc9ca37d3fa9",
+		"62d3efa42f27cc9ca37d4014",
+		"62d1f0262f27cc9ca37ceef7",
+		"6476c47feb01e8efa62a295c",
+		"62d8a8202f27cc9ca3802d61",
+		"6433b4448aea8786d1e1d1a5",
+		"63087e1036c823627b4e1326",
+		"6438f42b8aea8786d1e7774c",
+		"62e39d2f2f27cc9ca386db75",
+		"630362142f27cc9ca397146f",
+		"62d8dcde2f27cc9ca3806e7b",
+		"6461e3888aea8786d1003c4c",
+		"63086269c994e88c25e765e7",
+		"649142f1bc7765fecd23d81b",
+		"641ad72d779467cff19cc0e2",
+		"62724be7923488e172581fd2",
+		"649b9961bc7765fecd33682e",
+	}
+	arr := append(arr1, arr2...)
+	isok1 := 0
+	for _, v := range arr {
+		data := ul.Bid_Mgo.FindById("bidding", v)
+		if data != nil && len(data) > 2 {
+			ul.Bid_Mgo.Save("zktest_zhongbiao_info", data)
+			isok1++
+		} else {
+			data = ul.Bid_Mgo.FindById("bidding_back", v)
+			if data != nil && len(data) > 2 {
+				ul.Bid_Mgo.Save("zktest_zhongbiao_info", data)
+				isok1++
+			} else {
+				log.Debug(v)
+			}
+		}
+	}
+	log.Debug("is over ... ", len(arr), "~", isok1)
+
+	dataArr, _ := ul.Lab_Mgo.Find("f_sourceinfo_extract_0525_moban", nil, nil, nil)
+	log.Debug("查询数量...", len(dataArr))
+	isok2 := 0
+	for _, v := range dataArr {
+		i_ckdata := qu.IntAll(v["i_ckdata"])
+		if i_ckdata == 2 {
+			tmpid := qu.ObjToString(v["id"])
+			v_taginfo := *qu.ObjToMap(v["v_taginfo"])
+			for k1, v1 := range v_taginfo {
+				if k1 == "s_winner" || k1 == "bidamount" {
+					if qu.IntAll(v1) == 2 {
+						data := ul.Bid_Mgo.FindById("bidding", tmpid)
+						if len(data) > 0 {
+							ul.Bid_Mgo.Save("zktest_zhongbiao_info", data)
+							isok2++
+							break
+						} else {
+							data = ul.Bid_Mgo.FindById("bidding_back", tmpid)
+							if len(data) > 0 {
+								ul.Bid_Mgo.Save("zktest_zhongbiao_info", data)
+								isok2++
+								break
+							} else {
+								log.Debug(tmpid)
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	log.Debug("is over ... ", isok2)
+
+}
+
+//导出疑似异常数据
+func ExportWarningBidInfo() {
+	arr, _ := ul.Bid_Mgo.Find("zktest_zhongbiao_info", nil, nil, nil)
+	log.Debug("此时数据...", len(arr))
+	//7月1号之前的数据
+	q, total := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$lte": ul.StringTOBsonId("649efc000000000000000000"),
+		},
+	}, 0
+	sess := ul.Bid_Mgo.GetMgoConn()
+	defer ul.Bid_Mgo.DestoryMongoConn(sess)
+	isok, temp := 0, map[string]string{}
+	isok = len(arr)
+	it := sess.DB(ul.Bid_Mgo.DbName).C("bidding").Find(&q).Sort("-_id").Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%1000 == 0 {
+			log.Debug("curent index ", total, "~", isok)
+		}
+		subtype := qu.ObjToString(tmp["subtype"])
+		site := qu.ObjToString(tmp["site"])
+		pkg := *qu.ObjToMap(tmp["package"])
+		s_winner := qu.ObjToString(tmp["s_winner"])
+		bidamount := qu.Float64All(tmp["bidamount"])
+		if (subtype == "中标" || subtype == "成交" || subtype == "合同") &&
+			len(pkg) > 0 && (s_winner == "" || bidamount <= float64(0)) {
+			if temp[site] == "" {
+				temp[site] = site
+				isok++
+				ul.Bid_Mgo.Save("zktest_zhongbiao_info", tmp)
+			}
+		}
+		if isok >= 450 {
+			break
+		}
+		tmp = make(map[string]interface{})
+	}
+	log.Debug("is over ", total, "~", isok)
+}
+
 func ExportSSKXlsxData() {
 	ff, _ := xlsx.OpenFile("res/1.xlsx")
 	for _, sheet := range ff.Sheets {
@@ -40,3 +204,33 @@ func ExportSSKXlsxData() {
 	}
 	log.Debug("is over ......")
 }
+
+//......
+func ExportWarningDateInfo() {
+	query, total, isok := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gte": ul.StringTOBsonId("64776f000000000000000000"),
+			"$lt":  ul.StringTOBsonId("649d561f9f3f6a78fc3c149f"),
+		},
+	}, 0, 0
+	log.Debug("当前获取待更新的query...", query)
+	sess := ul.Bid_Mgo.GetMgoConn()
+	defer ul.Bid_Mgo.DestoryMongoConn(sess)
+	it := sess.DB(ul.Bid_Mgo.DbName).C("bidding").Find(&query).Sort("_id").Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%10000 == 0 {
+			log.Debug("cur index ", total, "~", isok, "~", tmp["_id"])
+		}
+		toptype := qu.ObjToString(tmp["toptype"])
+		if toptype == "拟建" {
+			project_completedate := qu.ObjToString(tmp["project_completedate"])
+			project_startdate := qu.ObjToString(tmp["project_startdate"])
+			if project_completedate != "" || project_startdate != "" {
+				isok++
+				ul.Bid_Mgo.Save("zktest_repair_0630", tmp)
+			}
+		}
+		tmp = make(map[string]interface{})
+	}
+	log.Debug("is over ... ", total, "~", isok)
+}

data_ylqx/src/util/initcfg.go

@@ -1,7 +1,12 @@
 package util
 
 import (
+	"encoding/json"
 	log "github.com/donnie4w/go-logger/logger"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	qu "qfw/util"
 	"sync"
 )
 
@@ -9,12 +14,13 @@ var (
 	Py_Mgo            *MongodbSim
 	Bid_Mgo, Save_Mgo *MongodbSim
 	Spi_Mgo, Qyxy_Mgo *MongodbSim
-	Ext_Mgo           *MongodbSim
+	Ext_Mgo, Lab_Mgo  *MongodbSim
 	MysqlGlobalTool   *Mysql
 	MysqlSubjectTool  *Mysql
 	IsLocal, IsDev    bool
 	TimeLayout        = "2006-01-02 15:04:05"
 	TimeTmeplate      = "2006-01-02"
+	S_Reality         = 0.8
 )
 var Pool_Mgo = make(chan bool, 10)
 var Wg_Mgo = &sync.WaitGroup{}
@@ -22,10 +28,10 @@ var Wg_Mgo = &sync.WaitGroup{}
 func InitClass() {
 	initMgoInfo()
 	initMysqlInfo()
-	//initCodeInfo()
+	initCodeInfo()
 }
 
-//初始化mgo
+// 初始化mgo
 func initMgoInfo() {
 	//Old_Mgo = &MongodbSim{
 	//	MongodbAddr: "127.0.0.1:12003",
@@ -36,11 +42,10 @@ func initMgoInfo() {
 	//}
 	//Old_Mgo.InitPool()
 	//return
-
 	if IsLocal {
 		Py_Mgo = &MongodbSim{
 			MongodbAddr: "127.0.0.1:27017",
-			DbName:      "medical",
+			DbName:      "zhengkun",
 			Size:        10,
 			UserName:    "",
 			Password:    "",
@@ -49,7 +54,7 @@ func initMgoInfo() {
 
 		Bid_Mgo = &MongodbSim{
 			MongodbAddr: "127.0.0.1:27017",
-			DbName:      "medical",
+			DbName:      "zhengkun",
 			Size:        10,
 			UserName:    "",
 			Password:    "",
@@ -58,7 +63,7 @@ func initMgoInfo() {
 
 		Save_Mgo = &MongodbSim{
 			MongodbAddr: "127.0.0.1:27017",
-			DbName:      "medical",
+			DbName:      "zhengkun",
 			Size:        10,
 			UserName:    "",
 			Password:    "",
@@ -67,7 +72,7 @@ func initMgoInfo() {
 
 		Spi_Mgo = &MongodbSim{
 			MongodbAddr: "127.0.0.1:27017",
-			DbName:      "medical",
+			DbName:      "zhengkun",
 			Size:        10,
 			UserName:    "",
 			Password:    "",
@@ -76,7 +81,7 @@ func initMgoInfo() {
 
 		Ext_Mgo = &MongodbSim{
 			MongodbAddr: "127.0.0.1:27017",
-			DbName:      "medical",
+			DbName:      "zhengkun",
 			Size:        10,
 			UserName:    "",
 			Password:    "",
@@ -85,12 +90,21 @@ func initMgoInfo() {
 
 		Qyxy_Mgo = &MongodbSim{
 			MongodbAddr: "127.0.0.1:27017",
-			DbName:      "medical",
+			DbName:      "zhengkun",
 			Size:        10,
 			UserName:    "",
 			Password:    "",
 		}
 		Qyxy_Mgo.InitPool()
+
+		Lab_Mgo = &MongodbSim{
+			MongodbAddr: "127.0.0.1:27017",
+			DbName:      "zhengkun",
+			Size:        10,
+			UserName:    "",
+			Password:    "",
+		}
+		Lab_Mgo.InitPool()
 	} else {
 		Bid_Mgo = &MongodbSim{
 			MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
@@ -145,6 +159,15 @@ func initMgoInfo() {
 			Password:    "zk@123123",
 		}
 		Qyxy_Mgo.InitPool()
+
+		Lab_Mgo = &MongodbSim{
+			MongodbAddr: "172.17.4.86:27080",
+			DbName:      "jyqykhfw_test",
+			Size:        10,
+			UserName:    "",
+			Password:    "",
+		}
+		Lab_Mgo.InitPool()
 	}
 }
 func initMysqlInfo() {
@@ -176,7 +199,7 @@ func initMysqlInfo() {
 
 }
 
-//插入数据
+// 插入数据
 func InsertMysqlGlobalData(name string, data map[string]interface{}, mark interface{}) int64 {
 	inb := MysqlGlobalTool.Insert(name, data)
 	if inb == -1 {
@@ -185,7 +208,7 @@ func InsertMysqlGlobalData(name string, data map[string]interface{}, mark interf
 	return inb
 }
 
-//插入数据
+// 插入数据
 func InsertMysqlSubjectData(name string, data map[string]interface{}, mark interface{}) int64 {
 	inb := MysqlSubjectTool.Insert(name, data)
 	if inb == -1 {
@@ -193,3 +216,20 @@ func InsertMysqlSubjectData(name string, data map[string]interface{}, mark inter
 	}
 	return inb
 }
+
+// http请求post
+func PostReliability(name string) float64 {
+	urlAddr := "http://172.17.4.188:20615/search/"
+	if IsLocal {
+		urlAddr = "http://192.168.3.109:20615/search/"
+	}
+	resp, _ := http.PostForm(urlAddr, url.Values{"word": {name}})
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return float64(0)
+	}
+	res := map[string]interface{}{}
+	json.Unmarshal(body, &res)
+	return qu.Float64All(res["result"])
+}

listen_data/src/main.go

@@ -19,6 +19,7 @@ var (
 	save_c_name, site_c_name, python_c_name, xlsx_name, site_unique_name string
 	prepareData                                                          []map[string]interface{}
 	startRun, startSend                                                  string
+	MysqlGlobalTool                                                      *Mysql
 )
 
 func initMgo() {
@@ -70,6 +71,14 @@ func init() {
 
 func main() {
 
+	//MysqlGlobalTool = &Mysql{
+	//	Address:  "192.168.3.217:4000",
+	//	UserName: "root",
+	//	PassWord: "=PDT49#80Z!RVv52_z",
+	//	DBName:   "global_common_data",
+	//}
+	//MysqlGlobalTool.Init()
+
 	//qy_mgo = &MongodbSim{
 	//	MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
 	//	DbName:      "mixdata",
@@ -79,50 +88,41 @@ func main() {
 	//}
 	//qy_mgo.InitPool()
 
-	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "172.17.4.87:27080",
-	//	DbName:      "editor",
-	//	Size:        10,
-	//	UserName:    "",
-	//	Password:    "",
-	//}
-	//save_mgo.InitPool()
-
 	//python_mgo = &MongodbSim{
-	//	MongodbAddr: "172.17.4.85:27080",
+	//	MongodbAddr: "172.17.4.87:27080",
 	//	DbName:      "qfw",
 	//	Size:        10,
 	//	UserName:    "",
 	//	Password:    "",
 	//}
 	//python_mgo.InitPool()
-
-	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "172.17.4.87:27080",
-	//	DbName:      "zk_regions",
-	//	Size:        10,
-	//	UserName:    "",
-	//	Password:    "",
-	//}
-	//save_mgo.InitPool()
-
-	//save_mgo = &MongodbSim{
+	//
+	//ext_mgo = &MongodbSim{
 	//	MongodbAddr: "172.17.4.85:27080",
 	//	DbName:      "qfw",
 	//	Size:        10,
 	//	UserName:    "",
 	//	Password:    "",
 	//}
-	//save_mgo.InitPool()
+	//ext_mgo.InitPool()
+
+	save_mgo = &MongodbSim{
+		MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
+		DbName:      "qfw",
+		Size:        10,
+		UserName:    "zhengkun",
+		Password:    "zk@123123",
+	}
+	save_mgo.InitPool()
 
 	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
+	//	MongodbAddr: "127.0.0.1:12005",
 	//	DbName:      "qfw",
 	//	Size:        10,
 	//	UserName:    "zhengkun",
 	//	Password:    "zk@123123",
 	//}
-	//save_mgo.InitPool()
+	//save_mgo.InitPoolDirect()
 
 	//save_mgo = &MongodbSim{
 	//	MongodbAddr: "127.0.0.1:27017",
@@ -133,33 +133,6 @@ func main() {
 	//}
 	//save_mgo.InitPool()
 
-	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "172.17.4.87:27080",
-	//	DbName:      "datamark1",
-	//	Size:        10,
-	//	UserName:    "",
-	//	Password:    "",
-	//}
-	//save_mgo.InitPool()
-
-	ext_mgo = &MongodbSim{
-		MongodbAddr: "172.17.4.85:27080",
-		DbName:      "qfw",
-		Size:        10,
-		UserName:    "",
-		Password:    "",
-	}
-	ext_mgo.InitPool()
-
-	//decodeJyUrl("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")
-	//decodeSEUrl("QltDeTokbwUCBA0WV0V6anBgCwIGW0VZ")
-	//decodeFeedBackInfo()
-	//export0427wininfo()
-
-	//decodeSEUrl("QltFK2gnbAIIUFwVV0dybyZpVAkEWkNb")
-	//decodeSEUrl("QltFK2h2OQMIUFwVV0dybyZpVAkFWEdY")
-	//exportErrInfo0515()
-
 	return
 
 	/*
@@ -290,8 +263,7 @@ func dealWithSiteData() {
 	}
 
 	log.Println("is python over :", total)
-
-	os.Remove(xlsx_name)
+	os.RemoveAll(xlsx_name)
 	f := xlsx.NewFile()
 
 	for _, tmp := range prepareData {
@@ -364,9 +336,8 @@ func dealWithSiteData() {
 	log.Println("结束......")
 }
 
-//准备模板站点数据
+// 准备模板站点数据
 func prepareXlsxSiteData() {
-
 	prepareData = make([]map[string]interface{}, 0)
 	sess := save_mgo.GetMgoConn()
 	defer save_mgo.DestoryMongoConn(sess)
@@ -381,7 +352,7 @@ func prepareXlsxSiteData() {
 	log.Println("准备完毕... ...")
 }
 
-//准备站点模型数据
+// 准备站点模型数据
 func prepareTableData() {
 
 	log.Println("准备数据...")
@@ -447,7 +418,6 @@ func prepareTableData() {
 			"data": arr,
 		})
 	}
-
 	log.Println("is save over")
 
 }

listen_data/src/mgo.go

@@ -142,17 +142,48 @@ func (m *MongodbSim) DestoryMongoConn(ms *MgoSess) {
 	ms = nil
 }
 
+func (m *MongodbSim) InitPoolDirect() {
+	opts := options.Client()
+	opts.SetConnectTimeout(3 * time.Second)
+	opts.ApplyURI("mongodb://" + m.MongodbAddr)
+	opts.SetMaxPoolSize(uint64(m.Size))
+	opts.SetDirect(true)
+	m.pool = make(chan bool, m.Size)
+
+	if m.UserName != "" && m.Password != "" {
+		cre := options.Credential{
+			Username:   m.UserName,
+			Password:   m.Password,
+			AuthSource: "admin",
+		}
+		opts.SetAuth(cre)
+	}
+
+	opts.SetMaxConnIdleTime(2 * time.Hour)
+	m.Ctx, _ = context.WithTimeout(context.Background(), 99999*time.Hour)
+	m.ShortCtx, _ = context.WithTimeout(context.Background(), 1*time.Minute)
+	client, err := mongo.Connect(m.ShortCtx, opts)
+	if err != nil {
+		log.Println("mgo init error:", err.Error())
+	} else {
+		m.C = client
+		log.Println("init success")
+	}
+}
+
 func (m *MongodbSim) InitPool() {
 	opts := options.Client()
 	opts.SetConnectTimeout(3 * time.Second)
 	opts.ApplyURI("mongodb://" + m.MongodbAddr)
 	opts.SetMaxPoolSize(uint64(m.Size))
+	//opts.SetDirect(true)
 	m.pool = make(chan bool, m.Size)
 
 	if m.UserName != "" && m.Password != "" {
 		cre := options.Credential{
-			Username: m.UserName,
-			Password: m.Password,
+			Username:   m.UserName,
+			Password:   m.Password,
+			AuthSource: "admin",
 		}
 		opts.SetAuth(cre)
 	}
@@ -176,7 +207,7 @@ func (m *MongodbSim) Close() {
 	<-m.pool
 }
 
-//新建表并生成索引
+// 新建表并生成索引
 func (m *MongodbSim) CreateIndex(c string, models []mongo.IndexModel) bool {
 	m.Open()
 	defer m.Close()
@@ -190,7 +221,7 @@ func (m *MongodbSim) CreateIndex(c string, models []mongo.IndexModel) bool {
 	}
 }
 
-//批量插入
+// 批量插入
 func (m *MongodbSim) UpSertBulk(c string, doc ...[]map[string]interface{}) (map[int64]interface{}, bool) {
 	m.Open()
 	defer m.Close()
@@ -217,7 +248,7 @@ func (m *MongodbSim) UpSertBulk(c string, doc ...[]map[string]interface{}) (map[
 	return r.UpsertedIDs, true
 }
 
-//批量插入
+// 批量插入
 func (m *MongodbSim) SaveBulk(c string, doc ...map[string]interface{}) bool {
 	m.Open()
 	defer m.Close()
@@ -236,7 +267,7 @@ func (m *MongodbSim) SaveBulk(c string, doc ...map[string]interface{}) bool {
 	return true
 }
 
-//保存
+// 保存
 func (m *MongodbSim) Save(c string, doc map[string]interface{}) interface{} {
 	m.Open()
 	defer m.Close()
@@ -248,7 +279,7 @@ func (m *MongodbSim) Save(c string, doc map[string]interface{}) interface{} {
 	return r.InsertedID
 }
 
-//更新by Id
+// 更新by Id
 func (m *MongodbSim) UpdateById(c, id string, doc map[string]interface{}) bool {
 	m.Open()
 	defer m.Close()
@@ -282,7 +313,7 @@ func (m *MongodbSim) UpdateQueryData(c string, query map[string]interface{}, doc
 	return true
 }
 
-//删除by id
+// 删除by id
 func (m *MongodbSim) DeleteById(c, id string) int64 {
 	m.Open()
 	defer m.Close()
@@ -294,7 +325,7 @@ func (m *MongodbSim) DeleteById(c, id string) int64 {
 	return r.DeletedCount
 }
 
-//通过条件删除
+// 通过条件删除
 func (m *MongodbSim) Delete(c string, query map[string]interface{}) int64 {
 	m.Open()
 	defer m.Close()
@@ -306,7 +337,7 @@ func (m *MongodbSim) Delete(c string, query map[string]interface{}) int64 {
 	return r.DeletedCount
 }
 
-//findbyid
+// findbyid
 func (m *MongodbSim) FindById(c, id string) map[string]interface{} {
 	m.Open()
 	defer m.Close()
@@ -317,7 +348,7 @@ func (m *MongodbSim) FindById(c, id string) map[string]interface{} {
 	return v
 }
 
-//findone
+// findone
 func (m *MongodbSim) FindOne(c string, query map[string]interface{}) map[string]interface{} {
 	m.Open()
 	defer m.Close()
@@ -328,7 +359,7 @@ func (m *MongodbSim) FindOne(c string, query map[string]interface{}) map[string]
 	return v
 }
 
-//find
+// find
 func (m *MongodbSim) Find(c string, query map[string]interface{}, sort, fields interface{}) ([]map[string]interface{}, error) {
 	m.Open()
 	defer m.Close()
@@ -348,7 +379,7 @@ func (m *MongodbSim) Find(c string, query map[string]interface{}, sort, fields i
 	return results, nil
 }
 
-//find
+// find
 func (m *MongodbSim) FindLimit(c string, query map[string]interface{}, sort, fields interface{}, limit int64) ([]map[string]interface{}, error) {
 	m.Open()
 	defer m.Close()
@@ -368,7 +399,7 @@ func (m *MongodbSim) FindLimit(c string, query map[string]interface{}, sort, fie
 	return results, nil
 }
 
-//创建_id
+// 创建_id
 func NewObjectId() primitive.ObjectID {
 	return primitive.NewObjectID()
 }

listen_data/src/mysql.go

@@ -29,7 +29,7 @@ func (m *Mysql) Init() {
 	if m.MaxIdleConns <= 0 {
 		m.MaxIdleConns = 20
 	}
-	var err error  //utf8mb4
+	var err error //utf8mb4
 	m.DB, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4", m.UserName, m.PassWord, m.Address, m.DBName))
 	log.Println(fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4", m.UserName, m.PassWord, m.Address, m.DBName))
 	if err != nil {
@@ -351,7 +351,7 @@ func (m *Mysql) UpdateByTx(tx *sql.Tx, tableName string, query, update map[strin
 		values = append(values, v)
 	}
 	q := fmt.Sprintf("update %s set %s where %s", tableName, strings.Join(q_fs, ","), strings.Join(u_fs, " and "))
-	log.Println(q, values)
+	//log.Println(q, values)
 	return m.UpdateOrDeleteBySqlByTx(tx, q, values...) >= 0
 }