analytable.go 109 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561
  1. package pretreated
  2. import (
  3. "fmt"
  4. u "jy/util"
  5. qutil "qfw/util"
  6. "regexp"
  7. "strings"
  8. "github.com/PuerkitoBio/goquery"
  9. )
  10. /**
  11. 全局变量,主要是一堆判断正则
  12. **/
  13. var (
  14. //清理品目中数字
  15. numclear = regexp.MustCompile("^[\\d一二三四五六七八九十.]+")
  16. //清理表格title中的不需要的内容
  17. tabletitleclear = regexp.MustCompile("[\\s\u3000\u2003\u00a0\\n、.,.。、_/((人民币万元件个公斤))]")
  18. //清理表格中是key中包含的空格或数字等
  19. tablekeyclear = regexp.MustCompile("[\\s\u3000\u2003\u00a0\\n、.,.。、_/]+|^[\\d一二三四五六七八九十]+[、.]*|[((【\\[].*?[))】\\]]")
  20. //清理表格td中的符号
  21. tabletdclear = regexp.MustCompile("[\\s\u3000\u2003\u00a0\\n、,。、_??;;~\\-#\\\\()(){}【】\\[\\]<>《》{}〔〕¥$]*")
  22. //判断key是金额,对万元的处理
  23. moneyreg = regexp.MustCompile("(预算|费|价|额|规模|投资)")
  24. //根据表格的内容判断是不是表头,如果含有金额则不是表头
  25. MoneyReg = regexp.MustCompile("^[\\s  ::0-9.万元()()人民币¥$]+$")
  26. //判断分包时
  27. moneyNum = regexp.MustCompile("[元整¥万]")
  28. //对隐藏表格的判断
  29. display = regexp.MustCompile("(?i).*?display\\s?[:]\\s?none.*")
  30. //---------------
  31. //求是分包的概率
  32. //根据表格的标签对分包进行打分
  33. TableMultiPackageReg_4 = regexp.MustCompile("(标段|分包|包段|划分|子包|标包|合同段)")
  34. TableMultiPackageReg_2 = regexp.MustCompile("(概况|范围|情况|内容|详细|结果|信息)")
  35. //在判断分包打分前过虑表格key
  36. FilterKey_2 = regexp.MustCompile("招标|投标|项目")
  37. //根据表格的key进行分包打分
  38. FindKey_2 = regexp.MustCompile("([分子][包标](号)?|标[号项段包](划分)?|包件?[号段名数])")
  39. //对值进行分包判断
  40. FindVal_1 = regexp.MustCompile("[第]?([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)((子|合同|分|施工|监理)?(标段?|包|合同段|标包))|((子|合同|分|施工|监理)?(标|包)(段|号)?)[  \u3000\u2003\u00a0]*([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)")
  41. FindVal2_1 = regexp.MustCompile("([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+)|^(设计|施工|监理|验收)[分子]?[标包]?[段号]?$")
  42. //判断分包前排除
  43. excludeKey = regexp.MustCompile("(涉及包号|包件号?|项目标号|规格|型号|招标范围|业绩|废标)|(^编号$)|([^包段标]编号)") //编号|划分
  44. //-------------
  45. cut = u.NewCut()
  46. //清理表格标签正则
  47. ClearTagReg = regexp.MustCompile("<[^>]*?>|[\\s\\n\\r]*$")
  48. //查找表格标签正则
  49. ttagreg = regexp.MustCompile("(?s)([^\\n::。,;\\s\u3000\u2003\u00a0]{2,30})[::]?[^::。;!\\n]{0,35}[\\s\\n]*$")
  50. //判断表格是表头的概率
  51. checkval = float32(0.6)
  52. //tdval_reg = regexp.MustCompile(`([\p{Han}][\p{Han}\s、()\\(\\)]{1,9})[::]([^::\\n。]{5,60})(?:[;;,,.。\\n\\t\\s])?`)
  53. //空格替换
  54. repSpace = regexp.MustCompile("[\\s\u3000\u2003\u00a0::]+|\\\\t+")
  55. //对表格kv的处理
  56. //对不能标准化的key做批识
  57. filter_tag_zb = regexp.MustCompile("(中标|成交|投标)[\\p{Han}]{0,6}(情况|结果|信息|明细)")
  58. //中标金额
  59. //包含以下字眼做标准化处理
  60. filter_zbje_k = regexp.MustCompile("(中标|成交|总|拦标|合同|供[应货]商|报)[\\p{Han}、]{0,6}(价|额|[大小]写|[万亿]?元).{0,4}$")
  61. //简单判断金额
  62. filter_zbje_jd = regexp.MustCompile("^[^(售|保证)]{0,4}(价|额).{0,4}$")
  63. //且排队以下字眼的key
  64. filter_zbje_kn = regexp.MustCompile("得分|打分|时间|业绩|须知|分|要求$")
  65. //且值包含以下字眼
  66. filter_zbje_v = regexp.MustCompile("[¥$$0-9一二三四五六七八九十,,〇零点..壹贰叁肆伍陆柒捌玖拾百佰千仟万亿億元圆角分整正()::大小写]{2,16}")
  67. //中标单位的处理
  68. //包含以下字眼的Key标准化
  69. filter_zbdw_ky = regexp.MustCompile("(中标|成交|拦标|合同|选中|投标|拟|预|最终)[\\p{Han}、]{0,6}(供[应货]商|企业|单位|人|机构)(名称)?.{0,4}$")
  70. //简单判断
  71. filter_zbdw_jd = regexp.MustCompile("(投标|成交|中标|合同)(供应商|单位|人|名称).{0,4}$")
  72. //且不包含以下字眼
  73. filter_zbdw_kn = regexp.MustCompile("第[2二3三4四5五]|得分|地址")
  74. //且值包含以下字眼
  75. filter_zbdw_v = regexp.MustCompile("(公司|集团|研究院|设计院|局|厂|部|站|城|店|市|所|处)$|([^购]中心|办公|用品)")
  76. //且值包含以下字眼
  77. filter_zbdw_v2 = regexp.MustCompile("(公司|集团|研究院|设计院|局|厂|部|站|城|店|市|所|处)$")
  78. //Tg = map[string]interface{}{}
  79. //一些表格没有表头,是空的,对值是排序的做处理对应 NullTxBid
  80. NullTdReg = regexp.MustCompile("(首选|第[一二三四五1-5])(中标|成交)?(名(称)?|(候选|排序)?(人|单位|供应商))")
  81. NullTxtBid = "成交供应商排名"
  82. projectnameReg = regexp.MustCompile("((公开)?招标)*[((第]*[一二三四五六七八九十a-zA-Z0-9]+(标段|包|标|段)[))]*$")
  83. MhSpilt = regexp.MustCompile("[::]")
  84. //识别采购单位联系人、联系电话、代理机构联系人、联系电话
  85. ContactInfoVagueReg = regexp.MustCompile("邮政编码|邮编|名称|(征求意见|报名审核购买)?((联系人?(及|和)?|办公|单位)?((电话([//]传真|及手机)?|手机)(号码)?|邮箱(地址)?|(地(址|点)))|(联系|收料)(人(姓名)?|方式)|传真|电子邮件|(主要负责|项目(负责|联系)|经办)人)|采购方代表")
  86. ContactInfoMustReg = regexp.MustCompile("^(" + ContactInfoVagueReg.String() + ")$")
  87. ContactType = map[string]*regexp.Regexp{
  88. "采购单位": regexp.MustCompile("(采购(项目.{2}|服务)?|比选|询价|招标(服务)?|甲|建设|委托|发包|业主|使用|谈判|本招标项目经办|征求意见联系|项目实施)(人|单位|部门|机构|机关|(执行)?方$)|(项目|建(库|设))单位|招标人信息|采购中心地址|业主|收料人|采购部"),
  89. "代理机构": regexp.MustCompile("(代理|受托|集中采购).{0,2}(人|方|单位|公司|机构)|招标机构|采购代理"),
  90. }
  91. ContactBuyerPersonFilterReg = regexp.MustCompile("(管理局)$")
  92. MultipleValueSplitReg = regexp.MustCompile("[,,、\\s\u3000\u2003\u00a0]")
  93. BuyerContacts = []string{"采购单位联系人", "采购单位联系电话", "采购单位联系地址"}
  94. FilterSerial = regexp.MustCompile(".+[、..::,]")
  95. underline = regexp.MustCompile("_+$")
  96. iswinnertabletag = regexp.MustCompile("(中标|候选人|成交|结果)")
  97. nswinnertabletag = regexp.MustCompile("[评得分估]+|标的|班子成员")
  98. jsonReg = regexp.MustCompile(`\{.+:[^}]*\} `) // \{".*\":\".+\"}
  99. regHz = regexp.MustCompile("[\u4e00-\u9fa5]")
  100. winnerOrderAndBidResult = regexp.MustCompile("((中标)?候选人|(中标|评标)结果)")
  101. )
  102. //在解析时,判断表格元素是否隐藏
  103. func IsHide(g *goquery.Selection) (b bool) {
  104. style, exists := g.Attr("style")
  105. if exists {
  106. b = display.MatchString(style)
  107. }
  108. return
  109. }
  110. //对表格的key进行标准化处理,多个k相同时,出现覆盖问题
  111. //待扩展,暂不支持正则标签库
  112. func CommonDataAnaly(k, tabletag, tabledesc string, v interface{},isSite bool,codeSite string) (kvTags map[string][]*u.Tag, returntag string) {
  113. kvTags = map[string][]*u.Tag{}
  114. v1 := ""
  115. if sv, sok := v.(string); sok { //取KV
  116. v1 = sv
  117. } else if sv, sok := v.([]string); sok { //是数组先默认取第一个
  118. if len(sv) >= 1 {
  119. v1 = sv[0]
  120. }
  121. }
  122. //对值单位的处理 (预算|费|价|额|规模|投资)
  123. if moneyreg.MatchString(k) {
  124. v1 += GetMoneyUnit(k, v1)
  125. }
  126. //先清理key
  127. //u.Debug(1, k, v1)
  128. k1 := ClearKey(k, 2)
  129. //u.Debug(2, k)
  130. //取标准key
  131. res := u.GetTags(k1,isSite,codeSite)
  132. if len(res) == 0 && k1 != k {
  133. res = u.GetTags(k,isSite,codeSite)
  134. k1 = k
  135. }
  136. //log.Println(k, res)
  137. // if len(res) == 0 {
  138. // go u.AddtoNoMatchMap(tk)
  139. // }
  140. //当取到标准化值时,放入数组
  141. if len(res) > 0 {
  142. for _, t1 := range res {
  143. //降低冒号值的权重
  144. if MhSpilt.MatchString(v1) {
  145. t1.Weight -= 50
  146. }
  147. if winnerOrderAndBidResult.MatchString(tabletag) && t1.Value == "采购单位联系人" { //处理table中项目负责人
  148. kvTags[k] = append(kvTags[k], &u.Tag{Key: k, Value: v1, IsInvalid: true})
  149. } else if regexp.MustCompile("(中标候选人|名单及其排序|排序)").MatchString(tabletag) && t1.Value == "采购单位" {
  150. kvTags[t1.Value] = append(kvTags[t1.Value], &u.Tag{Key: k1, Value: v1, Weight: t1.Weight - 100})
  151. } else {
  152. kvTags[t1.Value] = append(kvTags[t1.Value], &u.Tag{Key: k1, Value: v1, Weight: t1.Weight})
  153. }
  154. }
  155. //k1 = res[0].Value
  156. } else {
  157. kvTags[k] = append(kvTags[k], &u.Tag{Key: k, Value: v1, IsInvalid: true})
  158. //没有取到标准化key时,对中标金额和中标单位的逻辑处理
  159. if filter_zbje_k.MatchString(k) && !filter_zbje_kn.MatchString(k) && filter_zbje_v.MatchString(v1) {
  160. if tabletag == "" {
  161. returntag = "中标情况"
  162. }
  163. kvTags["中标金额"] = append(kvTags["中标金额"], &u.Tag{Key: k, Value: v1, Weight: -100})
  164. } else if filter_zbdw_ky.MatchString(k) && !filter_zbdw_kn.MatchString(k) &&
  165. filter_zbdw_v.MatchString(v1) {
  166. kvTags["中标单位"] = append(kvTags["中标单位"], &u.Tag{Key: k, Value: v1, Weight: -100})
  167. if tabletag == "" {
  168. returntag = "中标情况"
  169. }
  170. } else {
  171. //对上一步没有取到标准化key的进一步处理
  172. if tabletag == "" {
  173. }
  174. if filter_tag_zb.MatchString(tabletag) || filter_tag_zb.MatchString(tabledesc) {
  175. //u.Debug(v1, k, "-----", filter_zbdw_jd.MatchString(k), filter_zbdw_v.MatchString(v1))
  176. if filter_zbje_jd.MatchString(k) && !filter_zbje_kn.MatchString(k) && filter_zbje_v.MatchString(v1) {
  177. kvTags["中标金额"] = append(kvTags["中标金额"], &u.Tag{Key: k, Value: v1, Weight: -100})
  178. } /*else if filter_zbdw_jd.MatchString(k) && filter_zbdw_v.MatchString(v1) {
  179. k1 = append(k1, "中标单位")
  180. weight = append(weight, -100)
  181. b = true
  182. }*/
  183. }
  184. }
  185. }
  186. return
  187. }
  188. //对解析后的表格的kv进行过滤
  189. func (table *Table) KVFilter(isSite bool,codeSite string) {
  190. //1.标准化值查找
  191. //2.对数组的处理
  192. //3.对分包的处理
  193. //4.对KV的处理
  194. //判断表格是否有用,调用abandontable正则数组进行判断
  195. //遍历每一行
  196. table.analyTdKV() //1.遍历每行每列td的sortkv添加到table.SorkVK中;2.td有子表格的处理
  197. as := NewSortMap()
  198. //遍历table.sortkv,进行过滤处理,并放入标准化KV中,如果值是数组跳到下一步处理
  199. for _, k := range table.SortKV.Keys {
  200. //表格描述处理,对成交结果的处理
  201. if regexp.MustCompile("(成交|中标|候选|排名|名次|供应商排序)").MatchString(k) {
  202. table.Desc += "成交结果,"
  203. }
  204. if regexp.MustCompile("^单价").MatchString(k) {
  205. continue
  206. }
  207. v := table.SortKV.Map[k]
  208. if _, ok := v.(string); ok { //table.SortKV.Value为字符串,匹配抽取关键词table.SortKV.Key,匹配到添加k,v到table.StandKV,table.StandKVWeight
  209. k = regSpliteSegment.ReplaceAllString(regReplAllSpace.ReplaceAllString(k, ""), "")
  210. if k == "2、建设规模" {
  211. k = "预算"
  212. }
  213. if k == `中标价(万元)\费率(%)`{
  214. k = "中标价(万元)"
  215. }
  216. kvTags, tag := CommonDataAnaly(k, table.Tag, table.Desc, v,isSite,codeSite) //对key标准化处理,没有找到会走中标
  217. //qutil.Debug(k, v, k1, w1, v1, tag, b)
  218. if tag != "" && table.Tag == "" {
  219. table.Tag = tag
  220. }
  221. MergeKvTags(table.StandKV, kvTags)
  222. } else {
  223. //u.Debug(k, v, "---------")
  224. as.AddKey(k, v)
  225. }
  226. }
  227. //处理值是数组的kv放入标准化kv中//处理table.SortKV.value为数组的情况
  228. table.sortKVArr(as,isSite,codeSite)
  229. //
  230. if len(table.WinnerOrder) > 0 || !table.BPackage {
  231. winnerOrder := []map[string]interface{}{}
  232. maxSort := 0
  233. //调整顺序
  234. for i := 0; i < 2; i++ {
  235. for _, v := range table.WinnerOrder {
  236. sortstr, _ := v["sortstr"].(string)
  237. if (i == 0 && sortstr == "") || (i == 1 && sortstr != "") {
  238. continue
  239. }
  240. sort, _ := v["sort"].(int)
  241. if i == 0 {
  242. if maxSort == 0 || sort > maxSort {
  243. maxSort = sort
  244. }
  245. } else {
  246. maxSort++
  247. v["sort"] = maxSort
  248. }
  249. winnerOrder = append(winnerOrder, v)
  250. }
  251. if len(winnerOrder) == len(table.WinnerOrder) {
  252. break
  253. }
  254. }
  255. table.WinnerOrder = winnerOrder
  256. winnerOrder = []map[string]interface{}{}
  257. L: //遍历每个td,查询中标人
  258. for _, tr := range table.TRs {
  259. for _, td := range tr.TDs {
  260. winnerOrder = winnerOrderEntity.Find(td.Val, true, 3,isSite,codeSite)
  261. if len(winnerOrder) > 0 {
  262. break L
  263. }
  264. }
  265. }
  266. if len(table.WinnerOrder) > 0 {
  267. //中标候选人合并
  268. winnerOrderEntity.Merge(table.WinnerOrder, winnerOrder)
  269. } else if !table.BPackage { //没有table.WinnerOrder也没有分包 将td中的WinnerOrder赋值给table.WinnerOrder
  270. if len(winnerOrder) > 1 {
  271. table.WinnerOrder = winnerOrder
  272. }
  273. }
  274. }
  275. //对中标候选人进行排序
  276. winnerOrderEntity.Order(table.WinnerOrder)
  277. //该表格有一个分包,并且有中标候选人排序的情况下,把中标候选人放到包里面
  278. if table.BlockPackage != nil && table.BlockPackage.Keys != nil && len(table.BlockPackage.Keys) == 1 {
  279. if table.BlockPackage.Map != nil {
  280. onePkgKey := table.BlockPackage.Keys[0]
  281. onePkg, _ := table.BlockPackage.Map[onePkgKey].(*u.BlockPackage)
  282. if onePkg != nil && onePkg.WinnerOrder != nil && len(onePkg.WinnerOrder) == 0 {
  283. onePkg.WinnerOrder = table.WinnerOrder
  284. table.BlockPackage.AddKey(onePkgKey, onePkg)
  285. }
  286. }
  287. }
  288. }
  289. //处理table.SortKV.value为数组的情况
  290. func (table *Table) sortKVArr(as *SortMap,isSite bool,codeSite string) {
  291. winnertag := iswinnertabletag.MatchString(table.Tag) && !nswinnertabletag.MatchString(table.Tag) //table标签
  292. if !winnertag {
  293. winnertag = iswinnertabletag.MatchString(table.TableResult.BlockTag) && !nswinnertabletag.MatchString(table.TableResult.BlockTag) //块标签
  294. }
  295. checkKey := map[int]bool{}
  296. for kn, k := range as.Keys { //遍历table.SortKV.value为数组的key
  297. v := as.Map[k]
  298. if vm, ok := v.([]map[string]interface{}); ok && k == NullTxtBid {
  299. if table.WinnerOrder == nil {
  300. table.WinnerOrder = []map[string]interface{}{}
  301. }
  302. table.WinnerOrder = append(table.WinnerOrder, vm...)
  303. } else {
  304. //增加候选人排序逻辑
  305. if table.WinnerOrder == nil && !checkKey[kn] {
  306. if vs1, ok := v.([]string); ok {
  307. smap := make([]map[string]interface{}, len(vs1))
  308. for n1, _ := range vs1 {
  309. smap[n1] = map[string]interface{}{}
  310. }
  311. //hadSort := false
  312. tmpEntname := make([]string, len(vs1))
  313. tmpPrice := make([]string, len(vs1))
  314. for kn1, k := range as.Keys[kn:] {
  315. v := as.Map[k]
  316. if ContactType["采购单位"].MatchString(k) || ContactType["代理机构"].MatchString(k) {
  317. continue
  318. }
  319. //目前对数组数据的key做判断,但是某些额可以是不满足情况的
  320. //载明内容:[第一中标候选人 第二中标候选人] id:5d00587da5cb26b9b75e367b
  321. if vs, ok := v.([]string); ok && len(vs) == len(vs1) { //数组值的个数相同
  322. res, _, _, _, repl := CheckCommon(k, "bidorder")
  323. kv := ""
  324. if !res {
  325. kt := u.GetTags(filterThText.ReplaceAllString(ClearKey(k, 2), ""),isSite,codeSite)
  326. if kt.Len() > 0 {
  327. kv = kt[0].Value
  328. }
  329. }
  330. //qutil.Debug(k, res, repl, kv, "--", vs)
  331. if !res && kv == "" { //key未验证出,验证数组的val值
  332. checkKey[kn+kn1] = true
  333. if winnertag { //如果是中标信息 在根据val数组信息解析候选人
  334. for vsk, vsv := range vs {
  335. if NullTdReg.MatchString(vsv) { //数据先验证val是否有排序
  336. //hadSort = true
  337. smap[vsk]["sortstr"] = vsv
  338. smap[vsk]["sort"] = GetBidSort(vsv, vsk+1)
  339. } else if findCandidate2.MatchString(vsv) && tmpEntname[vsk] == "" { //数据验证val是否是候选人
  340. entname, _ := winnerOrderEntity.clear("中标单位", vsv).(string)
  341. if entname != "" {
  342. tmpEntname[vsk] = entname
  343. }
  344. } else { //验证val时如果数组中的第一条数据既不满足sort或者entname 判定此数组数据错误
  345. break
  346. }
  347. }
  348. }
  349. }
  350. if res || kv != "" { //连续往下找几个key
  351. checkKey[kn+kn1] = true
  352. SORT:
  353. if repl == "sort" {
  354. //hadSort = true
  355. for vsk, vsv := range vs {
  356. smap[vsk]["sortstr"] = vsv
  357. smap[vsk]["sort"] = GetBidSort(vsv, vsk+1)
  358. }
  359. } else if repl == "entname" || kv == "中标单位" {
  360. for vsk, vsv := range vs {
  361. if winnerReg6.MatchString(vsv) { //k:中标候选人 v:["第一名","第二名"]
  362. repl = "sort"
  363. goto SORT
  364. }
  365. // if entname, _ := smap[vsk]["entname"].(string); entname != "" || len([]rune(vsv)) < 3 {
  366. // break
  367. // }
  368. // entname, _ := winnerOrderEntity.clear("中标单位", vsv).(string)
  369. // if entname != "" {
  370. // smap[vsk]["entname"] = entname
  371. //
  372. if tmpEntname[vsk] != "" || len([]rune(vsv)) < 4 { //排除 单位:["台","个","套"]
  373. break
  374. }
  375. entname, _ := winnerOrderEntity.clear("中标单位", vsv).(string)
  376. if entname != "" {
  377. tmpEntname[vsk] = entname
  378. }
  379. }
  380. } else if kv == "中标金额" {
  381. for vsk, vsv := range vs {
  382. //过滤price 2348273.432元(万元)-->2348273.432
  383. //tmp1, _ := smap[vsk]["price"].(string)
  384. tmp1 := tmpPrice[vsk]
  385. p1num := numberReg2.FindString(tmp1)
  386. p2num := numberReg2.FindString(vsv)
  387. p1 := qutil.Float64All(p1num)
  388. p2 := qutil.Float64All(p2num)
  389. if p2 > p1 {
  390. //smap[vsk]["price"] = winnerOrderEntity.clear("中标金额", vsv+GetMoneyUnit(k, vsv))
  391. price := winnerOrderEntity.clear("中标金额", vsv+GetMoneyUnit(k, vsv))
  392. if pricestr, _ := price.(string); len(pricestr) < 30 && len(pricestr) > 0 {
  393. tmpPrice[vsk] = pricestr
  394. }
  395. }
  396. }
  397. }
  398. }
  399. } else {
  400. //break
  401. }
  402. }
  403. newSmap := []map[string]interface{}{}
  404. //qutil.Debug("smap=======", smap)
  405. //qutil.Debug("tmpEntname--", len(tmpEntname), tmpEntname)
  406. //qutil.Debug("tmpPrice--", len(tmpPrice), tmpPrice)
  407. for n, smap_v := range smap {
  408. //if hadSort { //有排序,再添加entname和price
  409. if len(tmpEntname) > 0 && n < len(tmpEntname) && tmpEntname[n] != "" {
  410. smap_v["entname"] = tmpEntname[n]
  411. if len(tmpPrice) > 0 && n < len(tmpPrice) && tmpPrice[n] != "" {
  412. smap_v["price"] = tmpPrice[n]
  413. }
  414. }
  415. //} else if len(tmpEntname) > 0 {
  416. //fmt.Println("table winnerorder only has entname", tmpEntname)
  417. //}
  418. if len(smap_v) > 2 { //只有排序信息 sort和sortstr
  419. newSmap = append(newSmap, smap_v)
  420. }
  421. }
  422. if len(newSmap) > 0 {
  423. table.WinnerOrder = newSmap
  424. }
  425. }
  426. }
  427. kvTags, tag := CommonDataAnaly(k, table.Tag, table.Desc, v,isSite,codeSite)
  428. if tag != "" && table.Tag == "" {
  429. table.Tag = tag
  430. }
  431. for kk, vv := range kvTags {
  432. table.StandKV[kk] = append(table.StandKV[kk], vv...)
  433. // else if k2 == "中标金额" {
  434. // if qutil.Float64All(v1) > qutil.Float64All(table.StandKV[k2]) {
  435. // table.StandKV[k2] = v1
  436. // }
  437. // }
  438. }
  439. }
  440. }
  441. }
  442. //1.遍历每行每列td的sortkv添加到table.SorkVK中;2.td有子表格的处理
  443. func (table *Table) analyTdKV() {
  444. //遍历每一行
  445. for _, tr := range table.TRs {
  446. for _, td := range tr.TDs {
  447. //fmt.Println(td.BH, td.MustBH, td.Val, td.SortKV.Map)
  448. bc := false
  449. if !td.BH {
  450. //表头是否是无用内容
  451. if td.HeadTd != nil {
  452. bc, _, _, _, _ = CheckCommon(td.HeadTd.Val, "abandontable")
  453. }
  454. }
  455. if !bc {
  456. //td元素有内嵌kv,遍历放入table的Kv中
  457. if len(td.SortKV.Keys) > 0 {
  458. for _, k3 := range td.SortKV.Keys {
  459. _val := td.SortKV.Map[k3]
  460. //thisFlag := false
  461. if td.HeadTd != nil && len([]rune(k3)) < 4 {
  462. k3 = td.HeadTd.Val + k3
  463. }
  464. if table.SortKV.Map[k3] == nil && _val != nil && _val != "" {
  465. //u.Debug(k3, _val)
  466. //if !thisFlag || (thisFlag && table.SortKV.Map[k3] == nil) {
  467. table.SortKV.AddKey(k3, _val)
  468. }
  469. }
  470. }
  471. }
  472. //td有子表格的处理
  473. //u.Debug(td.BH, td.Val, td.SonTableResult)
  474. if td.SonTableResult != nil {
  475. //u.Debug(td.SonTableResult.SortKV.Map, "-------", td.SonTableResult.Tabs)
  476. for k3, v3 := range td.SonTableResult.KvTags {
  477. table.StandKV[k3] = append(table.StandKV[k3], v3...)
  478. }
  479. //中标候选人排序
  480. if table.WinnerOrder == nil || len(table.WinnerOrder) == 0 {
  481. table.WinnerOrder = td.SonTableResult.WinnerOrder
  482. } else {
  483. winnerOrderEntity.Merge(table.WinnerOrder, td.SonTableResult.WinnerOrder)
  484. }
  485. }
  486. }
  487. }
  488. }
  489. //表格结果合并到父表格集中
  490. func (table *Table) MergerToTableresult() {
  491. //对多包表格的多包值的合并处理
  492. if table.BPackage {
  493. table.TableResult.IsMultiPackage = true
  494. for _, v2 := range table.BlockPackage.Keys {
  495. package1 := table.TableResult.PackageMap.Map[v2]
  496. if package1 == nil {
  497. table.TableResult.PackageMap.AddKey(v2, table.BlockPackage.Map[v2])
  498. if vvv, ok := table.BlockPackage.Map[v2].(*u.BlockPackage); ok {
  499. if vvv.TableKV != nil && len(vvv.TableKV.KvTags) > 0 {
  500. MergeKvTags(table.TableResult.KvTags, vvv.TableKV.KvTags)
  501. }
  502. }
  503. } else {
  504. bp := package1.(*u.BlockPackage)
  505. if bp.TableKV == nil {
  506. bp.TableKV = u.NewJobKv()
  507. }
  508. v1 := table.BlockPackage.Map[v2].(*u.BlockPackage)
  509. if v1.TableKV != nil && len(v1.TableKV.KvTags) > 0 {
  510. for k2, v2 := range v1.TableKV.KvTags {
  511. if k2 == "" {
  512. continue
  513. }
  514. isExists := false
  515. for _, v2v := range v2 {
  516. if v2v.Value == "" {
  517. continue
  518. }
  519. for _, v2vv := range bp.TableKV.KvTags[k2] {
  520. if v2v.Value == v2vv.Value {
  521. isExists = true
  522. break
  523. }
  524. }
  525. if !isExists {
  526. bp.TableKV.KvTags[k2] = append(bp.TableKV.KvTags[k2], v2v)
  527. bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
  528. }
  529. }
  530. }
  531. }
  532. if len(v1.WinnerOrder) > 0 && len(bp.WinnerOrder) == 0 {
  533. bp.WinnerOrder = v1.WinnerOrder
  534. }
  535. //table.TableResult.PackageMap.AddKey(k, v)
  536. }
  537. }
  538. // str := ""
  539. // for _, k := range table.TableResult.PackageMap.Keys {
  540. // v := table.TableResult.PackageMap.Map[k].(*u.BlockPackage)
  541. // str += fmt.Sprintf("包号:%s,中标人:%s,中标价:%s,预算:%s,文本:%s,排名:%v ---\t", v.Index, v.TableKV["中标单位"]+v.ColonKV["中标单位"], v.TableKV["中标金额"]+v.ColonKV["中标金额"], v.TableKV["预算"]+v.ColonKV["预算"], v.Text, v.WinnerOrder)
  542. // }
  543. // u.Debug(table, table.TableResult, str)
  544. }
  545. //遍历标准key到tableresult.sortkv中
  546. for _, v := range table.StandKV {
  547. for _, vv := range v {
  548. vv.Value = strings.Replace(vv.Value, "__", "", -1)
  549. }
  550. }
  551. MergeKvTags(table.TableResult.KvTags, table.StandKV)
  552. //表格的块标签
  553. if table.TableResult.BlockTag == "" && table.Tag != "" {
  554. table.TableResult.BlockTag = table.Tag
  555. }
  556. //中标候选人(多个table,现在默认取第一个table的信息,考虑需不需要多个table分析合并数据?)
  557. if table.TableResult.WinnerOrder == nil || len(table.TableResult.WinnerOrder) == 0 {
  558. table.TableResult.WinnerOrder = table.WinnerOrder
  559. }
  560. //增加brand 并列table
  561. if len(table.BrandData) > 0 {
  562. for _, v := range table.BrandData {
  563. if len(v) > 0 {
  564. table.TableResult.BrandData = append(table.TableResult.BrandData, v)
  565. }
  566. }
  567. }
  568. if table.BlockPackage != nil && len(table.BlockPackage.Keys) == 0 {
  569. for _, v := range table.BlockPackage.Keys {
  570. if table.BlockPackage.Map[v] != nil {
  571. if vvv, ok := table.BlockPackage.Map[v].((*u.BlockPackage)); ok {
  572. if vvv.TableKV != nil && len(vvv.TableKV.KvTags) > 0 {
  573. for kk, vv := range vvv.TableKV.KvTags {
  574. if kk == "" {
  575. continue
  576. }
  577. if len(table.TableResult.KvTags[kk]) == 0 {
  578. table.TableResult.KvTags[kk] = vv
  579. }
  580. }
  581. }
  582. }
  583. }
  584. }
  585. }
  586. }
  587. /**
  588. 解析表格入口
  589. 返回:汇总表格对象
  590. **/
  591. func AnalyTableV2(tabs *goquery.Selection, toptype, blockTag, con string, itype int, _id interface{}, ruleBlock *u.RuleBlock,isSite bool,codeSite string) (tabres *TableResult) {
  592. defer qutil.Catch()
  593. //u.Debug(con)
  594. if itype == 1 {
  595. //修复表格
  596. con = RepairCon(con)
  597. }
  598. //生成tableresult对象
  599. tabres = NewTableResult(_id, toptype, blockTag, con, itype, ruleBlock)
  600. //可以有多个table
  601. //for _, table := range tabs {
  602. //隐藏表格跳过
  603. if IsHide(tabs) {
  604. return
  605. }
  606. tabres.GoqueryTabs = tabs
  607. //}
  608. //解析表格集
  609. tabres.Analy(isSite,codeSite)
  610. return
  611. }
  612. //开始解析表格集
  613. func (ts *TableResult) Analy(isSite bool,codeSite string) {
  614. tabs := []*Table{}
  615. contactFormat := &u.ContactFormat{
  616. IndexMap: map[int]string{},
  617. MatchMap: map[string]map[string]bool{},
  618. }
  619. //for _, table := range ts.GoqueryTabs {
  620. tn := NewTable(ts.Html, ts, ts.GoqueryTabs)
  621. //核心模块
  622. tsw := tn.Analy(contactFormat,isSite,codeSite)
  623. for _, tab := range tsw {
  624. if len(tab.TRs) > 0 {
  625. tabs = append(tabs, tab)
  626. }
  627. //fmt.Println("tab.SortKV.Map", tab.SortKV.Keys)
  628. }
  629. //tn.SonTables = append(tn.SonTables, tn)
  630. //}
  631. //统一合并,考虑统一多表格是多包的情况---新增
  632. if len(tabs) > 1 {
  633. pns := map[string]string{}
  634. pnarr := []string{}
  635. for _, table := range tabs {
  636. if len(table.StandKV["项目名称"]) == 0 {
  637. continue
  638. }
  639. pn := table.StandKV["项目名称"][0]
  640. if pn != nil && pn.Value != "" && TitleReg.MatchString(pn.Value) {
  641. pnarr = append(pnarr, pn.Value)
  642. matchres := TitleReg.FindAllStringSubmatch(pn.Value, -1)
  643. if len(matchres) == 1 && len(matchres[0]) > 0 {
  644. v1 := u.PackageNumberConvert(matchres[0][0])
  645. pns[v1] = matchres[0][0]
  646. bp := &u.BlockPackage{}
  647. bp.Index = v1
  648. bp.Origin = matchres[0][0]
  649. if bp.TableKV == nil {
  650. bp.TableKV = u.NewJobKv()
  651. }
  652. for _, k := range []string{"中标金额", "中标单位", "预算", "成交状态", "项目名称", "项目编号", "采购范围"} {
  653. if len(table.StandKV[k]) > 0 {
  654. bp.TableKV.KvTags[k] = append(bp.TableKV.KvTags[k], &u.Tag{Key: k, Value: table.StandKV[k][0].Value})
  655. }
  656. }
  657. bp.WinnerOrder = table.WinnerOrder
  658. if table.BlockPackage.Map[v1] == nil {
  659. table.BPackage = true
  660. table.BlockPackage.AddKey(v1, bp)
  661. }else {
  662. table.BlockPackage.RemoveKey(v1)
  663. table.BlockPackage.AddKey(v1, bp)
  664. }
  665. }
  666. }
  667. }
  668. if len(tabs) == len(pns) {
  669. //多个表格,每个表格都是一个分包 http://www.cxzwfw.gov.cn/info/1009/6963.htm
  670. //项目名称、项目编号、采购单位、招标机构、预算
  671. pname := projectnameReg.ReplaceAllString(pnarr[0], "")
  672. btrue := true
  673. for _, pn := range pnarr[1:] {
  674. pn = projectnameReg.ReplaceAllString(pn, "")
  675. //u.Debug(pn, pname)
  676. if pn != pname {
  677. //项目名称不一致
  678. btrue = false
  679. break
  680. }
  681. }
  682. if btrue {
  683. ts.KvTags["项目名称"] = append(ts.KvTags["项目名称"], &u.Tag{Key: "项目名称", Value: pname, Weight: 100})
  684. for _, table := range tabs {
  685. table.BPackage = true
  686. //预算、中标金额、NullTxtBid成交供应商排名 中标单位 成交状态
  687. if table.BlockPackage != nil && len(table.BlockPackage.Keys) == 1 {
  688. bp := table.BlockPackage.Map[table.BlockPackage.Keys[0]].(*u.BlockPackage)
  689. if table.TableResult.WinnerOrder != nil {
  690. bp.WinnerOrder = table.WinnerOrder
  691. }
  692. if bp != nil && table.StandKV != nil {
  693. if bp.TableKV == nil {
  694. bp.TableKV = u.NewJobKv()
  695. }
  696. for nk, k := range []string{"中标金额", "中标单位", "预算", "成交状态", "项目名称", "项目编号", "采购范围"} {
  697. if len(table.StandKV[k]) > 0 {
  698. bp.TableKV.KvTags[k] = append(bp.TableKV.KvTags[k], &u.Tag{Key: k, Value: table.StandKV[k][0].Value})
  699. }
  700. if nk < 4 {
  701. delete(table.StandKV, k)
  702. }
  703. }
  704. }
  705. }
  706. }
  707. }
  708. }
  709. }
  710. for _, table := range tabs {
  711. table.MergerToTableresult()
  712. // for k, v := range table.TableResult.SortKV.Map {
  713. // qutil.Debug(k, "=====", v)
  714. // }
  715. MergeKvTags(ts.KvTags, table.TableResult.KvTags)
  716. }
  717. }
  718. //解析表格
  719. func (table *Table) Analy(contactFormat *u.ContactFormat,isSite bool,codeSite string) []*Table {
  720. //查找表体中的tr对象
  721. trs := table.Goquery.ChildrenFiltered("tbody,thead,tfoot").ChildrenFiltered("tr")
  722. if trs.Size() == 0 {
  723. trs = table.Goquery.ChildrenFiltered("tr")
  724. }
  725. //遍历节点,初始化table 结构
  726. table.createTabe(trs,isSite,codeSite)
  727. //重置行列
  728. table.ComputeRowColSpan()
  729. //对table结构体进行整体解析处理
  730. ts := table.AnalyTables(contactFormat,isSite,codeSite)
  731. return ts
  732. }
  733. //遍历节点,初始化table 结构体
  734. func (table *Table) createTabe(trs *goquery.Selection,isSite bool,codeSite string) {
  735. trs.Each(func(n int, sel *goquery.Selection) {
  736. //隐藏行不处理
  737. if IsHide(sel) {
  738. return
  739. }
  740. //遍历每行的td
  741. tds := sel.ChildrenFiltered("td,th")
  742. TR := NewTR(table)
  743. tdTextIsNull := false
  744. var empty int
  745. tds.Each(func(m int, selm *goquery.Selection) {
  746. //对隐藏列不处理!!!
  747. if IsHide(selm) {
  748. return
  749. }
  750. //进入每一个单元格
  751. td := NewTD(selm, TR, table,isSite,codeSite) //初始化td,kv处理,td中有table处理,td的方向
  752. //num++
  753. TR.AddTD(td)
  754. if td.Val == "" && td.SonTableResult == nil && len(td.SortKV.Map) == 0 { //删除一个tr,tr中所有td是空值的
  755. empty++
  756. if tds.Size() == empty {
  757. tdTextIsNull = true
  758. }
  759. }
  760. })
  761. //向table添加每行不为空的tr
  762. if !tdTextIsNull {
  763. table.AddTR(TR)
  764. }
  765. })
  766. }
  767. //对table进行整体解析处理
  768. func (tn *Table) AnalyTables(contactFormat *u.ContactFormat,isSite bool,codeSite string) []*Table {
  769. ts := tn.tableSubDemolitionTable() //分包,拆表
  770. for n, table := range ts {
  771. //处理每个table
  772. if len(table.TRs) > 0 {
  773. //删除尾部空白行
  774. table.deleteTrimTr()
  775. //table.Print()
  776. //校对表格
  777. table.Adjust(isSite,codeSite)
  778. //查找表格的标签,table.Tag字段
  779. table.FindTag()
  780. //log.Println(table.TableResult.Id, table.Html)
  781. //分割表格
  782. table.bSplit(n, ts,isSite,codeSite)
  783. table.TdContactFormat(contactFormat,isSite,codeSite) //contactFormat,处理采购单位,代理机构
  784. //开始查找kv,核心模块,table.SortKV
  785. table.FindKV(isSite,codeSite)
  786. //table中抽取品牌,table.BrandData
  787. if u.IsBrandGoods {
  788. table.analyBrand()
  789. }
  790. res, _, _, _, _ := CheckCommon(table.Tag, "abandontable")
  791. if !res {
  792. //过滤、标准化、合并kv,table.StandKV,table.StandKVWeight
  793. table.KVFilter(isSite,codeSite)
  794. }
  795. //对没有表头表格的处理
  796. if table.Tag != "" {
  797. co, m, b := CheckMultiPackage(table.Tag, "")
  798. if b {
  799. table.BPackage = b
  800. if len(table.BlockPackage.Map) == 0 {
  801. for _,av := range m{
  802. kv := u.NewJobKv()
  803. kv.KvTags= table.StandKV
  804. bd:=u.PackageNumberConvert(av[0])
  805. blockPackage := &u.BlockPackage{
  806. Origin:av[0],
  807. Name:av[0],
  808. Text:co,
  809. TableKV:kv,
  810. Index:bd,
  811. }
  812. if bd !=""{
  813. table.BlockPackage.AddKey(bd, blockPackage)
  814. }else {
  815. table.BlockPackage.AddKey(av[0], blockPackage)
  816. }
  817. }
  818. }
  819. table.StandKV["项目名称"] = append(table.StandKV["项目名称"], &u.Tag{Key: "项目名称", Value: table.Tag, Weight: -300})
  820. }
  821. }
  822. //判断是否是多包,并处理分包的//遍历td分块
  823. table.CheckMultiPackageByTable(isSite,codeSite)
  824. //MergeKvTags(table.TableResult.KvTags, table.StandKV)
  825. }
  826. }
  827. return ts
  828. }
  829. //分包,拆表
  830. func (table *Table) tableSubDemolitionTable() []*Table {
  831. tm := []map[string]interface{}{}
  832. tmk := map[string]bool{}
  833. tmn := map[int]map[string]interface{}{}
  834. for rownum, tr := range table.TRs {
  835. if len(tr.TDs) == 1 && table.ColNum > 1 { //tr里面有一列,table里面有多列
  836. td := tr.TDs[0] //取每行第一个td
  837. //td开始列等于0 && td结束列+1等于table列数 && td长度大于1小于50
  838. if td.StartCol == 0 && td.EndCol+1 == table.ColNum && len([]rune(td.Val)) > 1 && len([]rune(td.Val)) < 50 {
  839. con, m1, b := CheckMultiPackage(td.Val, "") //判断分包
  840. if b {
  841. for k, _ := range m1 {
  842. numstr := u.PackageNumberConvert(k)
  843. m2 := map[string]interface{}{
  844. "tag": con,
  845. //"num": numstr,
  846. //"numtxt": v[0],
  847. "startrow": rownum,
  848. }
  849. tmk[numstr] = true
  850. tmn[rownum] = m2
  851. tm = append(tm, m2)
  852. break
  853. }
  854. }
  855. }
  856. }
  857. }
  858. //拆表
  859. ts := []*Table{}
  860. if len(tmk) > 1 && len(tmk) == len(tm) {
  861. var tab1 *Table
  862. for rownum, tr := range table.TRs {
  863. if tab1 == nil {
  864. tab1 = NewTable("", table.TableResult, table.Goquery)
  865. tab1.BSplit = true
  866. if tmn[rownum] != nil {
  867. tab1.StandKV["项目名称"] = append(tab1.StandKV["项目名称"], &u.Tag{Key: "项目名称", Value: tmn[rownum]["tag"].(string), Weight: -100})
  868. }
  869. ts = append(ts, tab1)
  870. }
  871. if tmn[rownum] != nil {
  872. tab1.Tag = tmn[rownum]["tag"].(string)
  873. } else {
  874. tab1.AddTR(tr)
  875. }
  876. if tmn[rownum+1] != nil {
  877. tab1 = nil
  878. }
  879. }
  880. } else {
  881. ts = append(ts, table)
  882. }
  883. return ts
  884. }
  885. //分割表格
  886. func (table *Table) bSplit(n int, ts []*Table,isSite bool,codeSite string) {
  887. if table.BSplit {
  888. if !table.BHeader && n > 0 {
  889. for i := n - 1; i > -1; i-- {
  890. if ts[i].BHeader {
  891. if ts[i].BFirstRow {
  892. //取第一行插入到
  893. table.InsertTR(ts[i].TRs[0])
  894. table.Adjust(isSite,codeSite)
  895. }
  896. break
  897. }
  898. }
  899. }
  900. }
  901. }
  902. //删除尾部空白行
  903. func (table *Table) deleteTrimTr() {
  904. for len(table.TRs) > 0 {
  905. npos := len(table.TRs)
  906. tailTR := table.TRs[npos-1] //最后一个tr,取最后一行
  907. bspace := true
  908. for _, v := range tailTR.TDs {
  909. if v.Val != "" || v.SonTableResult != nil || len(v.SortKV.Keys) > 0 {
  910. bspace = false
  911. break
  912. }
  913. }
  914. //删除尾部空行,是空行的话就删除
  915. if bspace {
  916. table.TRs = table.TRs[:npos-1]
  917. } else {
  918. break
  919. }
  920. }
  921. }
  922. //校对表格
  923. func (table *Table) Adjust(isSite bool,codeSite string) {
  924. //计算行列起止位置,跨行跨列处理
  925. table.ComputeRowColSpan()
  926. // for k1, tr := range table.TRs {
  927. // for k2, td := range tr.TDs {
  928. // qutil.Debug(k1, k2, td.Val, td.StartRow, td.EndRow, td.StartCol, td.EndCol)
  929. // }
  930. // }
  931. //大概计算每个起止行列的概率
  932. table.GetKeyRation()
  933. /*
  934. for k, v := range table.StartAndEndRation {
  935. for k1, v1 := range v.Poss {
  936. bs, _ := json.Marshal(v1)
  937. str := ""
  938. for _, td := range v.Tdmap[v1] {
  939. str += "__" + td.Val + fmt.Sprintf("%d_%d_%d_%d", td.StartRow, td.EndRow, td.StartCol, td.EndCol)
  940. }
  941. qutil.Debug(k, k1, string(bs), v.Rationmap[v1], str)
  942. }
  943. }
  944. */
  945. //u.Debug("tdnum:", num, table.RowNum, table.ColNum)
  946. //是否是规则的表格,单元各个数=行数*列数
  947. table.Brule = table.TDNum == table.RowNum*table.ColNum
  948. count := 0
  949. for _, trs := range table.TRs {
  950. for _, td := range trs.TDs {
  951. if td.BH {
  952. count++
  953. }
  954. }
  955. }
  956. if float32(count)/float32(table.TDNum) < 0.85 {
  957. //精确计算起止行列是表头的概率
  958. table.ComputeRowColIsKeyRation(isSite,codeSite)
  959. bhead := false
  960. L:
  961. for i, tr := range table.TRs {
  962. for _, td := range tr.TDs {
  963. if td.BH {
  964. //qutil.Debug("----=====---", td.Val, len(table.TRs[len(table.TRs)-1].TDs), i, len(table.TRs)-1)
  965. if i == len(table.TRs)-1 && len(table.TRs[len(table.TRs)-1].TDs) == 2 {
  966. res, _, _, _, _ := CheckCommon(td.Val, "abandontable")
  967. if res {
  968. //删除此行
  969. table.TRs = table.TRs[:len(table.TRs)-1]
  970. table.Adjust(isSite,codeSite)
  971. return
  972. }
  973. }
  974. bhead = true
  975. break L
  976. }
  977. }
  978. }
  979. table.BHeader = bhead
  980. }
  981. }
  982. //计算行/列表格的结束位置 StartRow=0 EndRow=0,table.TDNum td个数 table.RowNum 行数
  983. func (table *Table) ComputeRowColSpan() {
  984. n := 0 //td总个数
  985. mapRC := map[int]map[int]int{} //记录第几行pos,起始列对应的合并值
  986. for k, v := range table.TRs {
  987. n += len(v.TDs) //每行的td总数相加
  988. nk := 0 //nk列的起始,k行的起始||如果有合并,起始就不是0
  989. ball := true
  990. rowspans := v.TDs[0].Rowspan //某一行第一个td的rowspan
  991. for _, v1 := range v.TDs {
  992. if v1.Rowspan != rowspans {
  993. ball = false
  994. break
  995. }
  996. }
  997. for _, v1 := range v.TDs {
  998. if ball {
  999. v1.Rowspan = 1
  1000. }
  1001. mc := mapRC[k]
  1002. for {
  1003. if mc != nil && mc[nk] > 0 {
  1004. nk += mc[nk]
  1005. } else {
  1006. break
  1007. }
  1008. }
  1009. v1.StartCol = nk
  1010. nk += v1.Colspan - 1
  1011. v1.EndCol = nk
  1012. if nk >= table.ColNum {
  1013. table.ColNum = nk + 1
  1014. }
  1015. nk++
  1016. v1.StartRow = k
  1017. v1.EndRow = k + v1.Rowspan - 1
  1018. ck := fmtkey("c", v1.StartCol, v1.EndCol)
  1019. tdcs := table.StartAndEndRation[ck]
  1020. if tdcs == nil {
  1021. tdcs = NewTDRationScope(ck)
  1022. table.StartAndEndRation[ck] = tdcs
  1023. table.StartAndEndRationKSort.AddKey(ck, 1)
  1024. }
  1025. tdcs.Addtd(v1)
  1026. rk := fmtkey("r", v1.StartRow, v1.EndRow)
  1027. tdrs := table.StartAndEndRation[rk]
  1028. if tdrs == nil {
  1029. tdrs = NewTDRationScope(rk)
  1030. table.StartAndEndRation[rk] = tdrs
  1031. table.StartAndEndRationKSort.AddKey(rk, 1)
  1032. }
  1033. tdrs.Addtd(v1)
  1034. if v1.Rowspan > 1 {
  1035. for i := 1; i < v1.Rowspan; i++ {
  1036. r := k + i
  1037. if r < len(table.TRs) {
  1038. mc := mapRC[r]
  1039. if mc == nil {
  1040. mc = map[int]int{}
  1041. }
  1042. mc[v1.StartCol] = v1.Colspan
  1043. mapRC[r] = mc
  1044. }
  1045. }
  1046. }
  1047. }
  1048. }
  1049. table.TDNum = n //td总个数
  1050. table.RowNum = len(table.TRs) //tr总行数
  1051. }
  1052. func fmtkey(t string, start, end int) string {
  1053. return fmt.Sprintf("%s_%d_%d", t, start, end)
  1054. }
  1055. //查找每个table的标签,如果有标签可按标签处理,否则根据表格去判断
  1056. func (table *Table) FindTag() {
  1057. //查找每个table的标签,如果有标签可按标签处理,否则根据表格去判断
  1058. if table.Tag != "" {
  1059. return
  1060. }
  1061. if table.Tag == "" {
  1062. table.Tag = table.TableResult.BlockTag
  1063. }
  1064. //u.Debug(table.Tag)
  1065. }
  1066. //计算r/c_start_end的概率
  1067. func (table *Table) GetKeyRation() {
  1068. for _, vn := range table.StartAndEndRationKSort.Keys {
  1069. v := table.StartAndEndRation[vn]
  1070. for _, v1 := range v.Poss {
  1071. count := 0
  1072. n := 0
  1073. for _, td := range v.Tdmap[v1] {
  1074. n++
  1075. if td.BH {
  1076. count++
  1077. }
  1078. }
  1079. v.Rationmap[v1] = float32(count) / float32(n)
  1080. }
  1081. }
  1082. }
  1083. //计算行列是表头的概率调用GetKeyRation
  1084. func (table *Table) ComputeRowColIsKeyRation(isSite bool,codeSite string) {
  1085. //增加对跨行校正限止
  1086. // u.Debug(table.Brule, table.ColNum, table.RowNum, table.TDNum)
  1087. bkeyfirstrow := false
  1088. bkeyfirstcol := false
  1089. if table.Brule { //不存在跨行跨列的情况,规则表格
  1090. checkCompute := map[string]bool{}
  1091. for k, tr := range table.TRs {
  1092. rk := fmtkey("r", tr.TDs[0].StartRow, tr.TDs[0].EndRow)
  1093. if k == 0 { //第1行的概率
  1094. ck := fmtkey("c", tr.TDs[0].StartCol, tr.TDs[0].EndCol)
  1095. //u.Debug(table.BFirstRow, "--", table.StartAndEndRation[rk], table.StartAndEndRation[ck])
  1096. ration1, _ := table.StartAndEndRation[rk].GetTDRation(tr.TDs[0])
  1097. ration2, _ := table.StartAndEndRation[ck].GetTDRation(tr.TDs[0])
  1098. if (len(tr.TDs) == 2 && ration2 < 0.55) && (len(tr.TDs) == 2 && ration1 > 0.5) { //第一行为key
  1099. bkeyfirstrow = true
  1100. ball := true
  1101. for _, td := range tr.TDs {
  1102. if MoneyReg.MatchString(td.Val) {
  1103. bkeyfirstrow = false
  1104. ball = false
  1105. td.BH = false
  1106. break
  1107. }
  1108. }
  1109. for _, td := range tr.TDs {
  1110. if ball {
  1111. //td.BH = true
  1112. td.KeyDirect = 1
  1113. td.KVDirect = 2
  1114. }
  1115. }
  1116. } else if ration2 > 0.55 { //第1列
  1117. bkeyfirstcol = true
  1118. if !checkCompute[ck] {
  1119. checkCompute[ck] = true
  1120. //重置第1列
  1121. for _, tr1 := range table.TRs {
  1122. for _, td1 := range tr1.TDs {
  1123. if td1.StartCol == 0 {
  1124. if !MoneyReg.MatchString(td1.Val) {
  1125. //td1.BH = true
  1126. td1.KeyDirect = 2
  1127. td1.KVDirect = 1
  1128. }
  1129. }
  1130. }
  1131. }
  1132. }
  1133. }
  1134. if !bkeyfirstrow && !bkeyfirstcol {
  1135. if len(tr.TDs) > 1 && ration1 > ration2 && ration1 > 0.5 {
  1136. bkeyfirstrow = true
  1137. for _, td := range tr.TDs {
  1138. if !MoneyReg.MatchString(td.Val) {
  1139. //td.BH = true
  1140. td.KeyDirect = 1
  1141. td.KVDirect = 2
  1142. }
  1143. }
  1144. } else if tr.Table.ColNum > 1 && ration2 > 0.5 {
  1145. bkeyfirstcol = true
  1146. if !checkCompute[ck] {
  1147. checkCompute[ck] = true
  1148. //重置第1列
  1149. for _, tr1 := range table.TRs {
  1150. for _, td1 := range tr1.TDs {
  1151. if td1.StartCol == 0 {
  1152. if !MoneyReg.MatchString(td1.Val) {
  1153. td1.BH = true
  1154. td1.KeyDirect = 2
  1155. td1.KVDirect = 1
  1156. }
  1157. }
  1158. }
  1159. }
  1160. }
  1161. }
  1162. }
  1163. } else {
  1164. if bkeyfirstrow {
  1165. //第一列的概率
  1166. ration1, _ := table.StartAndEndRation[rk].GetTDRation(tr.TDs[0])
  1167. if k == 1 || ration1 < checkval {
  1168. for _, td := range tr.TDs {
  1169. if !td.MustBH {
  1170. td.BH = false
  1171. td.KeyDirect = 0
  1172. td.KVDirect = 0
  1173. }
  1174. }
  1175. } //else {for _, td := range tr.TDs {}}
  1176. } else {
  1177. //列在起作用
  1178. if bkeyfirstcol {
  1179. for _, td := range tr.TDs {
  1180. ck := fmtkey("c", td.StartCol, td.EndCol)
  1181. ration1, _ := table.StartAndEndRation[ck].GetTDRation(td)
  1182. if !checkCompute[ck] {
  1183. checkCompute[ck] = true
  1184. if ration1 >= checkval && td.ColPos != 1 {
  1185. for _, tr1 := range table.TRs {
  1186. for _, td1 := range tr1.TDs {
  1187. if td1.StartCol == td.StartCol {
  1188. if !MoneyReg.MatchString(td1.Val) {
  1189. td1.BH = true
  1190. td1.KeyDirect = 2
  1191. td1.KVDirect = 1
  1192. }
  1193. }
  1194. }
  1195. }
  1196. } else {
  1197. for _, tr1 := range table.TRs[1:] {
  1198. for _, td1 := range tr1.TDs[1:] {
  1199. if td1.StartCol == td.StartCol && !td1.MustBH {
  1200. td1.BH = false
  1201. td1.KeyDirect = 0
  1202. td1.KVDirect = 0
  1203. }
  1204. }
  1205. }
  1206. }
  1207. }
  1208. }
  1209. }
  1210. }
  1211. }
  1212. }
  1213. }
  1214. //qutil.Debug("table.Brule", table.Brule, !bkeyfirstcol && !bkeyfirstrow)
  1215. if !table.Brule || (!bkeyfirstcol && !bkeyfirstrow) {
  1216. //断行问题,虽然同列或同行,但中间被跨行截断,表格方向调整
  1217. for _, k := range table.StartAndEndRationKSort.Keys {
  1218. v := table.StartAndEndRation[k]
  1219. //横向判断,要判断最多的方向,否则会出现不定的情况(map遍历问题)
  1220. k1 := k[:1]
  1221. for _, v2 := range v.Poss {
  1222. lentds := len(v.Tdmap[v2])
  1223. if v.Rationmap[v2] > checkval {
  1224. for _, td := range v.Tdmap[v2] {
  1225. if td.KeyDirect == 0 && !MoneyReg.MatchString(td.Val) {
  1226. if k1 == "r" {
  1227. ck := fmtkey("c", td.StartCol, td.EndCol)
  1228. rt := table.StartAndEndRation[ck]
  1229. //clen := 0
  1230. var fv float32
  1231. var tdn []*TD
  1232. if rt != nil {
  1233. fv, tdn = rt.GetTDRation(td)
  1234. //clen = len(tdn)
  1235. }
  1236. if lentds > 1 {
  1237. if ((tdn != nil && v.Rationmap[v2] > fv) || tdn == nil) && td.Valtype != "BO" {
  1238. td.KeyDirect = 1
  1239. td.KVDirect = 2
  1240. //td.BH = true
  1241. }
  1242. }
  1243. } else {
  1244. ck := fmtkey("r", td.StartRow, td.EndRow)
  1245. rt := table.StartAndEndRation[ck]
  1246. var fv float32
  1247. var tdn []*TD
  1248. //clen := 0
  1249. if rt != nil {
  1250. fv, tdn = rt.GetTDRation(td)
  1251. //clen = len(tdn)
  1252. }
  1253. if lentds > 1 {
  1254. if ((tdn != nil && v.Rationmap[v2] > fv) || tdn == nil) && td.Valtype != "BO" {
  1255. td.KeyDirect = 2
  1256. td.KVDirect = 1
  1257. td.BH = true
  1258. }
  1259. }
  1260. }
  1261. } else {
  1262. break
  1263. }
  1264. }
  1265. } else if v.Rationmap[v2] < 0.5 && len(v.Tdmap[v2]) > 3 {
  1266. for _, td := range v.Tdmap[v2] {
  1267. // u.Debug(td.Val, "-----", td.BH)
  1268. if td.KeyDirect == 0 && td.BH && !td.MustBH {
  1269. if k1 == "r" {
  1270. ck := fmtkey("c", td.StartCol, td.EndCol)
  1271. rt := table.StartAndEndRation[ck]
  1272. clen := 0
  1273. var fv float32
  1274. var tdn []*TD
  1275. if rt != nil {
  1276. fv, tdn = rt.GetTDRation(td)
  1277. clen = len(tdn)
  1278. }
  1279. if lentds >= clen && lentds > 1 {
  1280. if (tdn != nil && v.Rationmap[v2] < fv) || tdn == nil {
  1281. td.BH = false
  1282. }
  1283. }
  1284. } else {
  1285. ck := fmtkey("r", td.StartRow, td.EndRow)
  1286. rt := table.StartAndEndRation[ck]
  1287. var fv float32
  1288. var tdn []*TD
  1289. clen := 0
  1290. if rt != nil {
  1291. fv, tdn = rt.GetTDRation(td)
  1292. clen = len(tdn)
  1293. }
  1294. if lentds >= clen && lentds > 1 {
  1295. if (tdn != nil && v.Rationmap[v2] < fv) || tdn == nil {
  1296. td.BH = false
  1297. }
  1298. }
  1299. }
  1300. } else {
  1301. break
  1302. }
  1303. }
  1304. }
  1305. }
  1306. }
  1307. }
  1308. table.GetKeyRation()
  1309. if len(table.TRs) > 0 && len(table.TRs[0].TDs) > 0 {
  1310. t0 := table.TRs[0].TDs[0]
  1311. key := fmtkey("r", t0.StartRow, t0.EndRow)
  1312. r, t := table.StartAndEndRation[key].GetTDRation(t0)
  1313. if r > 0.9 && len(t) > 1 {
  1314. table.BFirstRow = true
  1315. }
  1316. for k, tr := range table.TRs {
  1317. if len(tr.TDs) == 1 && tr.TDs[0].StartCol == 0 && tr.TDs[0].EndCol+1 == table.ColNum {
  1318. tr.TDs[0].BH = false
  1319. tr.TDs[0].KVDirect = 0
  1320. sv := FindKv(tr.TDs[0].Val, "", 2)
  1321. _, resm := colonkvEntity.entrance(tr.TDs[0].Val, "", nil, 2,isSite,codeSite)
  1322. for k, v := range resm {
  1323. sv.AddKey(k, v)
  1324. }
  1325. if len(sv.Keys) > 0 {
  1326. for _, v1 := range sv.Keys {
  1327. if tr.TDs[0].SortKV.Map[v1] == nil {
  1328. table.SortKV.AddKey(v1, sv.Map[v1])
  1329. }
  1330. }
  1331. } else if table.Tag == "" && k == 0 && len(tr.TDs[0].Val) > 11 {
  1332. table.Tag = tr.TDs[0].Val
  1333. }
  1334. // subVal := tdval_reg.FindAllStringSubmatch(tr.TDs[0].Val, -1)
  1335. // //u.Debug(tr.TDs[0].Val, subVal)
  1336. // if len(subVal) > 0 {
  1337. // for _, subv1 := range subVal {
  1338. // if len(subv1) == 3 {
  1339. // table.SortKV.AddKey(subv1[1], subv1[2])
  1340. // }
  1341. // }
  1342. // } else if k == 0 && len(tr.TDs[0].Val) > 11 {
  1343. // table.Tag = tr.TDs[0].Val
  1344. // }
  1345. }
  1346. // for _, td := range tr.TDs {
  1347. // u.Debug(td.BH, td.Val, "----")
  1348. // }
  1349. }
  1350. }
  1351. }
  1352. //查找表格的kv,调用FindTdVal
  1353. func (table *Table) FindKV(isSite bool,codeSite string) {
  1354. //判断全是key的表格不再查找
  1355. if table.BHeader { //只要一个是key即为true
  1356. direct := If(table.BFirstRow, 2, 1).(int) //kv,2查找方向,向上查找
  1357. vdirect := If(direct == 2, 1, 2).(int)
  1358. //控制跨行表格
  1359. bcon := false
  1360. //增加表格切块判断,只判断切块分包
  1361. //控制中标人排序方向
  1362. bodirect := 0
  1363. //控制中标人排序数值
  1364. sort := 1
  1365. //开始抽取
  1366. for _, tr := range table.TRs {
  1367. bcon = trSingleColumn(tr, bcon, table) //tr单列,是否丢弃内容
  1368. if bcon {
  1369. continue
  1370. }
  1371. if tr.TDs[0].StartRow >= 0 {
  1372. numbh := 0
  1373. for _, td := range tr.TDs {
  1374. if td.BH {
  1375. numbh++
  1376. }
  1377. }
  1378. if numbh > 0 && numbh <= len(tr.TDs)/2 {
  1379. direct, vdirect = 1, 2
  1380. } else {
  1381. direct, vdirect = 2, 1
  1382. }
  1383. }
  1384. for _, td := range tr.TDs {
  1385. /**
  1386. rt := table.StartAndEndRation[fmtkey("r", td.StartCol, td.EndCol)]
  1387. if rt != nil {
  1388. r, t := rt.GetTDRation(td)
  1389. u.Debug(td.BH, td.Val, r, t)
  1390. }
  1391. **/
  1392. // if td.Val == "电视" || td.Val == "电话机" || td.Val == "传真机" || td.Val == "音响" {
  1393. //qutil.Debug("----td.Valtype", td.Valtype, "td.BH:", td.BH, "KVDirect:", td.KVDirect, "Val:", td.Val, "direct:", direct, "vdirect:", vdirect)
  1394. // }
  1395. if !td.BH && td.KVDirect < 3 {
  1396. if !table.FindTdVal(td, direct, vdirect) { //table.FindTdVal()存储了table.SortKV
  1397. if !table.FindTdVal(td, vdirect, direct) {
  1398. //都识别不到时,对第一、二中标候选人的处理
  1399. bo, res := GetBidOrder(td, bodirect, sort)
  1400. if res {
  1401. sort++
  1402. bodirect = bo
  1403. }
  1404. if len(td.SortKV.Map) > 0 {
  1405. for _, tdv := range td.SortKV.Keys {
  1406. if tdv == "" || td.SortKV.Map[tdv] == "" { //value为空或者null不再添加到table.SortKV
  1407. continue
  1408. }
  1409. table.SortKV.AddKey(tdv, td.SortKV.Map[tdv])
  1410. }
  1411. }
  1412. }
  1413. }
  1414. //fmt.Println("td:", td.Val, td.BH, td.HeadTd, td.KVDirect)
  1415. }
  1416. }
  1417. }
  1418. //qutil.Debug("FindKV", table.SortKV.Map)
  1419. } else if len(table.TRs) > 0 { //没有表头的表格处理,默认纵向吧
  1420. res := initLongitudinalData(table) //拼装纵向数组
  1421. //再拆值,类似http://www.ggzy.hi.gov.cn/cgzbgg/16553.jhtml第二列,有多个值
  1422. nmapkeys := []int{}
  1423. nmap := map[int][]*u.Kv{}
  1424. L:
  1425. for _, r1 := range res {
  1426. for n, r := range r1 {
  1427. if len([]rune(r)) < 60 { // 长度小于60才去分
  1428. //res1, _ := GetKVAll(r, "", nil)
  1429. res1, _ := colonkvEntity.entrance(r, "", nil, 2,isSite,codeSite)
  1430. if res1 != nil {
  1431. nmap[n] = res1
  1432. nmapkeys = append(nmapkeys, n)
  1433. /**
  1434. //截取串
  1435. for _k1, _ := range res1 {
  1436. r = regexp.MustCompile(_k1+".*").ReplaceAllString(r, "")
  1437. }
  1438. r1[n] = r
  1439. res[pos] = r1
  1440. **/
  1441. } else if nmap[n] != nil {
  1442. //放空值
  1443. nmap[n] = append(nmap[n], &u.Kv{})
  1444. }
  1445. } else {
  1446. nmap = nil
  1447. nmapkeys = nil
  1448. break L
  1449. }
  1450. }
  1451. }
  1452. //调整
  1453. if len(nmap) > 0 {
  1454. kmapkeys := []string{}
  1455. kmap := map[string][]string{}
  1456. for _, mk := range nmapkeys { //同是第n列
  1457. for pos, m1 := range nmap[mk] {
  1458. k, v := m1.Key, m1.Value
  1459. kv := kmap[k]
  1460. if kv == nil {
  1461. kv = []string{}
  1462. }
  1463. kv = append(kv, v)
  1464. kmap[k] = kv
  1465. kmapkeys = append(kmapkeys, k)
  1466. for _, k := range kmapkeys {
  1467. arr := kmap[k]
  1468. if len(arr) < pos {
  1469. arr = append(arr, "")
  1470. kmap[k] = arr
  1471. kmapkeys = append(kmapkeys, k)
  1472. }
  1473. }
  1474. }
  1475. }
  1476. if len(kmap) > 0 {
  1477. for _, k := range kmapkeys {
  1478. if len(kmap[k]) == 1 {
  1479. table.SortKV.AddKey(k, kmap[k][0])
  1480. } else if len(kmap[k]) > 1 {
  1481. table.SortKV.AddKey(k, kmap[k])
  1482. }
  1483. }
  1484. }
  1485. }
  1486. //=================
  1487. //解析值放到map中
  1488. for _, arr := range res {
  1489. if len(arr) > 0 {
  1490. v1 := arr[0]
  1491. _, _, _, _, repl := CheckCommon(v1, "con")
  1492. if repl == "ENT" {
  1493. table.SortKV.AddKey("中标单位", arr)
  1494. continue
  1495. } else if repl == "BO" {
  1496. table.SortKV.AddKey("排名", arr)
  1497. continue
  1498. }
  1499. }
  1500. }
  1501. }
  1502. //qutil.Debug("FindKV", table.SortKV.Map)
  1503. }
  1504. //初始化组装纵向数据
  1505. func initLongitudinalData(table *Table) [][]string {
  1506. res := make([][]string, len(table.TRs[0].TDs)) //创建table第一行的列数长度
  1507. for n, _ := range res {
  1508. res[n] = []string{}
  1509. }
  1510. for _, tr := range table.TRs {
  1511. for n, td := range table.TRs[0].TDs { //第一行的所有td
  1512. td1 := table.GetTdByRCNo(tr.TDs[0].StartRow, td.StartCol) //根据行号列号获取td对象
  1513. if td1 != nil {
  1514. res[n] = append(res[n], td1.Val)
  1515. } else {
  1516. res[n] = append(res[n], "")
  1517. }
  1518. }
  1519. }
  1520. return res
  1521. }
  1522. //tr单列,是否丢弃内容
  1523. func trSingleColumn(tr *TR, bcon bool, table *Table) bool {
  1524. if len(tr.TDs) == 1 {
  1525. bcon = false
  1526. td := tr.TDs[0]
  1527. if td.StartCol == 0 && td.EndCol+1 == table.ColNum && len([]rune(td.Val)) > 4 && len([]rune(td.Val)) < 50 {
  1528. res, _, _, _, _ := CheckCommon(td.Val, "abandontable")
  1529. if res { //以下内容丢弃
  1530. bcon = true
  1531. }
  1532. }
  1533. }
  1534. return bcon
  1535. }
  1536. //获取中标人顺序
  1537. //direct 0默认 1横向 2纵向
  1538. func GetBidOrder(td *TD, direct, n int) (d int, res bool) {
  1539. if td.Valtype != "BO" {
  1540. return
  1541. }
  1542. if td.Rowspan > 1 {
  1543. for i := 0; i < td.Rowspan; i++ {
  1544. nextcol := 1
  1545. L1:
  1546. for {
  1547. vtd := td.TR.Table.GetTdByRCNo(td.StartRow+i, td.EndCol+nextcol)
  1548. if vtd == nil {
  1549. break L1
  1550. }
  1551. nextcol += vtd.Colspan
  1552. if filter_zbdw_v2.MatchString(vtd.Val) {
  1553. arrbo := td.TR.Table.SortKV.Map[NullTxtBid]
  1554. if arrbo == nil {
  1555. arrbo = []map[string]interface{}{}
  1556. td.TR.Table.SortKV.AddKey(NullTxtBid, arrbo)
  1557. }
  1558. a1 := arrbo.([]map[string]interface{})
  1559. a1 = append(a1, map[string]interface{}{
  1560. "entname": vtd.Val,
  1561. "sortstr": td.Val,
  1562. "sort": GetBidSort(td.Val, n),
  1563. })
  1564. res = true
  1565. td.TR.Table.SortKV.AddKey(NullTxtBid, a1)
  1566. }
  1567. }
  1568. }
  1569. } else if td.Colspan > 1 {
  1570. for i := 1; i < td.Colspan; i++ {
  1571. nextcol := 0
  1572. L2:
  1573. for {
  1574. vtd := td.TR.Table.GetTdByRCNo(td.StartRow+i, td.StartCol+nextcol)
  1575. if vtd == nil || vtd.Colspan >= td.Colspan {
  1576. break L2
  1577. }
  1578. nextcol += vtd.Colspan
  1579. if filter_zbdw_v2.MatchString(vtd.Val) {
  1580. arrbo := td.TR.Table.SortKV.Map[NullTxtBid]
  1581. if arrbo == nil {
  1582. arrbo = []map[string]interface{}{}
  1583. td.TR.Table.SortKV.AddKey(NullTxtBid, arrbo)
  1584. }
  1585. a1 := arrbo.([]map[string]interface{})
  1586. a1 = append(a1, map[string]interface{}{
  1587. "entname": vtd.Val,
  1588. "sortstr": td.Val,
  1589. "sort": GetBidSort(td.Val, n),
  1590. })
  1591. res = true
  1592. td.TR.Table.SortKV.AddKey(NullTxtBid, a1)
  1593. }
  1594. }
  1595. }
  1596. } else {
  1597. rtd := td.TR.Table.GetTdByRCNo(td.StartRow, td.EndCol+1)
  1598. btd := td.TR.Table.GetTdByRCNo(td.EndRow+1, td.StartCol)
  1599. //if ((rtd != nil && !rtd.BH && rtd.Valtype == "BO") || direct == 1) && btd != nil && filter_zbdw_v.MatchString(btd.Val) {
  1600. if ((rtd != nil && !rtd.BH) || direct == 1) && btd != nil && filter_zbdw_v2.MatchString(btd.Val) {
  1601. d = 1
  1602. arrbo := td.TR.Table.SortKV.Map[NullTxtBid]
  1603. if arrbo == nil {
  1604. arrbo = []map[string]interface{}{}
  1605. td.TR.Table.SortKV.AddKey(NullTxtBid, arrbo)
  1606. }
  1607. a1 := arrbo.([]map[string]interface{})
  1608. a1 = append(a1, map[string]interface{}{
  1609. "entname": btd.Val,
  1610. "sortstr": td.Val,
  1611. "sort": GetBidSort(td.Val, n),
  1612. })
  1613. res = true
  1614. td.TR.Table.SortKV.AddKey(NullTxtBid, a1)
  1615. //} else if ((btd != nil && !btd.BH && btd.Valtype == "BO") || direct == 2) && rtd != nil && filter_zbdw_v.MatchString(rtd.Val) {
  1616. } else if ((btd != nil && !btd.BH) || direct == 2) && rtd != nil && filter_zbdw_v2.MatchString(rtd.Val) {
  1617. d = 2
  1618. arrbo := td.TR.Table.SortKV.Map[NullTxtBid]
  1619. if arrbo == nil {
  1620. arrbo = []map[string]interface{}{}
  1621. td.TR.Table.SortKV.AddKey(NullTxtBid, arrbo)
  1622. }
  1623. a1 := arrbo.([]map[string]interface{})
  1624. a1 = append(a1, map[string]interface{}{
  1625. "entname": rtd.Val,
  1626. "sortstr": td.Val,
  1627. "sort": GetBidSort(td.Val, n),
  1628. })
  1629. res = true
  1630. td.TR.Table.SortKV.AddKey(NullTxtBid, a1)
  1631. }
  1632. }
  1633. return
  1634. }
  1635. func GetBidSort(str string, n int) int {
  1636. val := n
  1637. if strings.Index(str, "首选") > -1 {
  1638. val = 1
  1639. } else {
  1640. val = winnerOrderEntity.toNumber(str, n)
  1641. }
  1642. return val
  1643. }
  1644. //查找每一个单元格的表头,调用FindNear
  1645. func (table *Table) FindTdVal(td *TD, direct, vdirect int) (b bool) {
  1646. near := table.FindNear(td, direct)
  1647. // if near != nil {
  1648. // fmt.Println("near----", near.Val, td.Val)
  1649. // }
  1650. // qutil.Debug(near != nil)
  1651. // qutil.Debug(near.BH)
  1652. // qutil.Debug(near.KeyDirect == vdirect, near.KeyDirect == 0)
  1653. // qutil.Debug(near.KVDirect == direct, near.KVDirect == 0)
  1654. // qutil.Debug(near.KVDirect < 3)
  1655. if near != nil && near.BH && (near.KeyDirect == vdirect || near.KeyDirect == 0) && (near.KVDirect == direct || near.KVDirect == 0) && near.KVDirect < 3 {
  1656. near.KVDirect = direct
  1657. near.KeyDirect = vdirect
  1658. td.KVDirect = direct
  1659. key := repSpace.ReplaceAllString(near.Val, "")
  1660. if near.Val == "" {
  1661. key = fmtkey("k", near.TR.RowPos, near.ColPos)
  1662. }
  1663. val := table.SortKV.Map[key]
  1664. //qutil.Debug("====================", "key:", key, "val:", val)
  1665. bthiskey := false
  1666. if val != nil {
  1667. curpos := table.SortKV.Index[key]
  1668. thistr := table.kTD[curpos]
  1669. if thistr != near {
  1670. near.Val += "_"
  1671. for table.SortKV.Map[near.Val] != nil {
  1672. near.Val += "_"
  1673. }
  1674. key = near.Val //之前这个地方没有重置,导致把之前结果覆盖了
  1675. } else {
  1676. bthiskey = true
  1677. }
  1678. }
  1679. bfind := false
  1680. barr := false
  1681. varrpos := -1
  1682. if bthiskey {
  1683. //处理是数组值,且有合并行或合并列的情况 kvscope,对数组值的处理
  1684. pos := table.SortKV.Index[key]
  1685. mval := table.kvscope[pos]
  1686. bvalfind := false
  1687. if direct == 1 { //kv是横向
  1688. L1:
  1689. for k3, v3 := range mval {
  1690. for _, v4 := range v3 {
  1691. if v4.EndRow+1 == td.StartRow && v4.EndCol == td.EndCol {
  1692. varrpos = k3
  1693. bvalfind = true
  1694. break L1
  1695. }
  1696. }
  1697. }
  1698. } else { //kv是纵向
  1699. L2:
  1700. for k3, v3 := range mval {
  1701. for _, v4 := range v3 {
  1702. if v4.EndCol+1 == td.StartCol && v4.EndRow == td.EndRow {
  1703. varrpos = k3
  1704. bvalfind = true
  1705. break L2
  1706. }
  1707. }
  1708. }
  1709. }
  1710. if vals, ok := val.([]string); ok {
  1711. if near.Val == "" {
  1712. bn := false
  1713. for _, vs := range vals {
  1714. if vs != "" && NullTdReg.MatchString(vs) {
  1715. bn = true
  1716. } else {
  1717. bn = false
  1718. break
  1719. }
  1720. }
  1721. if bn {
  1722. near.Val = NullTxtBid
  1723. key = NullTxtBid
  1724. bfind = true
  1725. }
  1726. }
  1727. if bvalfind && varrpos > -1 && len(vals) > varrpos {
  1728. vals[varrpos] = td.Val // += "__" + td.Val
  1729. } else {
  1730. //添加时候去除空值和nil
  1731. newVals := []string{}
  1732. for _, isval := range vals {
  1733. if isval == "" {
  1734. continue
  1735. }
  1736. newVals = append(newVals, isval)
  1737. }
  1738. //vals = append(vals, td.Val)
  1739. if td.Val != "" {
  1740. newVals = append(newVals, td.Val)
  1741. }
  1742. val = newVals
  1743. varrpos = len(vals) - 1
  1744. }
  1745. } else if vals, ok := val.(string); ok && vals != "" && td.Val != "" {
  1746. if bvalfind {
  1747. val = td.Val //vals + "__" + td.Val
  1748. } else {
  1749. tval := []string{vals}
  1750. tval = append(tval, td.Val)
  1751. val = tval
  1752. varrpos = 1
  1753. }
  1754. }
  1755. barr = true
  1756. } else {
  1757. if td.Val != "" {
  1758. val = td.Val
  1759. } else if len(near.SortKV.Map) == 1 && near.SortKV.Map[near.Val] != "" {
  1760. val = near.SortKV.Map[near.Val]
  1761. }
  1762. }
  1763. td.HeadTd = near
  1764. if bfind {
  1765. tkey := fmtkey("k", near.TR.RowPos, near.ColPos)
  1766. table.SortKV.ReplaceKey(key, val, tkey)
  1767. } else {
  1768. if val == nil || val == "" || key == "采购项目预算金额" {
  1769. return
  1770. }
  1771. table.SortKV.AddKey(key, val)
  1772. //if table.SortKV.Map[key] != nil {
  1773. pos := table.SortKV.Index[key]
  1774. //qutil.Debug("=========", "key:", key, "val:", val, "pos:", pos)
  1775. if barr {
  1776. mval := table.kvscope[pos]
  1777. if mval != nil {
  1778. tds := mval[varrpos]
  1779. if tds != nil {
  1780. tds = append(tds, td)
  1781. } else {
  1782. tds = []*TD{td}
  1783. }
  1784. if varrpos > -1 {
  1785. mval[varrpos] = tds
  1786. table.kvscope[pos] = mval
  1787. }
  1788. }
  1789. } else {
  1790. table.kvscope[pos] = map[int][]*TD{
  1791. 0: []*TD{td},
  1792. }
  1793. table.kTD[pos] = near
  1794. }
  1795. //}
  1796. }
  1797. b = true
  1798. }
  1799. return
  1800. }
  1801. //查找单元格的表头时,横向或纵向
  1802. func (table *Table) FindNear(td *TD, direct int) *TD {
  1803. if direct == 1 && td.StartCol > 0 { //左临
  1804. tr := table.TRs[:td.TR.RowPos+1]
  1805. for i := len(tr) - 1; i > -1; i-- {
  1806. tds := tr[i].TDs
  1807. for _, td1 := range tds {
  1808. if td1.StartRow <= td.StartRow && td1.EndRow >= td.EndRow && td1.EndCol+1 == td.StartCol {
  1809. //找到左临节点
  1810. if td1.BH {
  1811. return td1
  1812. } else {
  1813. if td1.HeadTd != nil && td1.HeadTd.KVDirect == direct {
  1814. return td1.HeadTd
  1815. }
  1816. }
  1817. }
  1818. }
  1819. }
  1820. } else if direct == 2 && td.StartRow > 0 { //上临
  1821. tr := table.TRs[:td.TR.RowPos]
  1822. for i := len(tr) - 1; i > -1; i-- {
  1823. tds := tr[i].TDs
  1824. for _, td1 := range tds {
  1825. if td1.StartCol <= td.StartCol && td1.EndCol >= td.EndCol && td1.EndRow+1 == td.StartRow {
  1826. //找到左临节点
  1827. if td1.BH {
  1828. return td1
  1829. } else {
  1830. if td1.HeadTd != nil && td1.HeadTd.KVDirect == direct {
  1831. return td1.HeadTd
  1832. }
  1833. }
  1834. }
  1835. }
  1836. }
  1837. }
  1838. return nil
  1839. }
  1840. //根据行号列号获取td对象
  1841. func (tn *Table) GetTdByRCNo(row, col int) *TD {
  1842. for _, tr := range tn.TRs {
  1843. for _, td := range tr.TDs {
  1844. if td.StartCol <= col && td.EndCol >= col && td.StartRow <= row && td.EndRow >= row {
  1845. return td
  1846. }
  1847. }
  1848. }
  1849. return nil
  1850. }
  1851. //判断表格是否是分包
  1852. func (tn *Table) CheckMultiPackageByTable(isSite bool,codeSite string) (b bool, index []string) {
  1853. pac := 0 //包的数量
  1854. val := 0 //分值
  1855. index = []string{} //存储分包,使用tbale.SortKV的key和value使用正则等处理对值进行判断
  1856. index_pos := []int{} //下标
  1857. //是数组且能找到标段之类的提示
  1858. //arr_count := 0 //计数table.SortKV的value是数组的数量,后面没用
  1859. key_index := -1
  1860. hasPkgTd := map[string]bool{}
  1861. //初始化CheckMultiPackageByTable方法需要的数据
  1862. key_index, index, index_pos, val, pac, hasPkgTd = initCheckMultiPackageByTable(tn, key_index, index, index_pos, val, pac, hasPkgTd)
  1863. //key是分包的情况
  1864. //记录key对应的值
  1865. commonKeyVals := map[string][]string{}
  1866. //记录key出现的次数
  1867. keyExistsCount := map[string]int{}
  1868. if pac > 1 {
  1869. val = 10
  1870. } else {
  1871. //查找标签
  1872. if TableMultiPackageReg_4.MatchString(tn.Tag) {
  1873. val += 4
  1874. } else if TableMultiPackageReg_2.MatchString(tn.Tag) {
  1875. val += 4
  1876. }
  1877. //根据table.SortKV的key判断是否分包,如果没有再根据value判断
  1878. val, index, index_pos = foundPacBySortKV(tn, val, index, index_pos, &keyExistsCount, &commonKeyVals, key_index, hasPkgTd)
  1879. }
  1880. // u.Debug(index)
  1881. //过滤重复及标准化!
  1882. standIndex := []string{}
  1883. standIndex_pos := []int{}
  1884. oldIndex := []string{} //存放包的原始值
  1885. brepeat := map[string]bool{}
  1886. for k, v := range index {
  1887. v = u.PackageNumberConvert(v)
  1888. if !brepeat[v] {
  1889. brepeat[v] = true
  1890. standIndex = append(standIndex, v)
  1891. standIndex_pos = append(standIndex_pos, index_pos[k])
  1892. oldIndex = append(oldIndex, index[k])
  1893. }
  1894. }
  1895. index = standIndex
  1896. //有一个以上的包,并且相同的key出现一次以上,认为这个key是属于包里面的
  1897. if len(commonKeyVals) > 0 {
  1898. for k, v := range commonKeyVals {
  1899. if len(index) > 1 && keyExistsCount[k] < 2 {
  1900. continue
  1901. }
  1902. tn.SortKV.AddKey(k, v)
  1903. }
  1904. }
  1905. //
  1906. isGoonNext := false
  1907. if val > 4 && len(brepeat) > 0 {
  1908. b = true
  1909. //多包解析
  1910. if b {
  1911. tn.BPackage = true
  1912. //根据数组index分包长度添加table.BlockPackage子包数组
  1913. for nk, v := range index {
  1914. if tn.BlockPackage.Map[v] == nil {
  1915. kv := u.NewJobKv()
  1916. kv.KvTags = tn.StandKV
  1917. bp := &u.BlockPackage{}
  1918. bp.Index = v //序号 (转换后编号,只有数字或字母)
  1919. bp.Origin = oldIndex[nk] //包的原始值
  1920. bp.TableKV = kv //table kv (分出的对应的KV值)
  1921. tn.BlockPackage.AddKey(v, bp) //table子包数组
  1922. }
  1923. }
  1924. isGoonNext = tn.manyPackageProcessByIndex(index, standIndex_pos,isSite,codeSite) //多包处理,处理不同情况下的分包
  1925. }
  1926. } else {
  1927. isGoonNext = true
  1928. }
  1929. if isGoonNext { //没有处理成数组的情况下,继续调用正文查找分包的方法
  1930. tn.isGoonNext(isSite,codeSite)
  1931. }
  1932. //查找分包中的中标人排序
  1933. if tn.BlockPackage != nil && tn.BlockPackage.Keys != nil && len(tn.BlockPackage.Keys) > 0 {
  1934. for _, v := range tn.BlockPackage.Keys {
  1935. vv, ok := tn.BlockPackage.Map[v].(*u.BlockPackage)
  1936. if ok && (vv.WinnerOrder == nil || len(vv.WinnerOrder) == 0) {
  1937. vv.WinnerOrder = winnerOrderEntity.Find(vv.Text, true, 2,isSite,codeSite)
  1938. }
  1939. }
  1940. }
  1941. return
  1942. }
  1943. //多包处理,处理不同情况下的分包
  1944. func (tn *Table) manyPackageProcessByIndex(index []string, standIndex_pos []int,isSite bool,codeSite string) (isGoonNext bool) {
  1945. if len(index) == 1 { //是一个的情况
  1946. if len(tn.SortKV.Keys) < 10 && tn.ColNum < 10 && tn.RowNum < 4 { //table带排序的KV值小于10并且小于10列和小于4行
  1947. beq := true
  1948. for _, v2 := range tn.SortKV.Keys {
  1949. if _, ok := tn.SortKV.Map[v2].(string); !ok {
  1950. beq = false
  1951. break
  1952. }
  1953. }
  1954. if beq { //统一处理为数组
  1955. td := tn.GetTdByRCNo(tn.RowNum-1, 0)
  1956. if !td.BH && FindVal2_1.MatchString(td.Val) {
  1957. for _, v2 := range tn.SortKV.Keys {
  1958. tn.SortKV.AddKey(v2, []string{tn.SortKV.Map[v2].(string)})
  1959. }
  1960. } else {
  1961. //没有处理成数组的情况下,继续调用正文查找分包的方法
  1962. isGoonNext = true
  1963. }
  1964. }
  1965. }
  1966. }
  1967. for _, k1 := range tn.SortKV.Keys {
  1968. v1 := tn.SortKV.Map[k1]
  1969. if _, bvs := v1.(string); bvs && len(index) > 1 && !strings.HasSuffix(k1, "_") { //table.SortKV.Map.value为字符串并且index有分包而且table.SortKV.Map.key没有_
  1970. v1_array := []string{v1.(string)}
  1971. underline := ""
  1972. for {
  1973. underline += "_"
  1974. if tn.SortKV.Map[k1+underline] == nil {
  1975. break
  1976. } else if v3, v2_ok := tn.SortKV.Map[k1+underline].(string); v2_ok && v3 != "" {
  1977. v1_array = append(v1_array, v3)
  1978. }
  1979. }
  1980. v1 = v1_array
  1981. }
  1982. if val, bvs := v1.([]string); bvs {
  1983. if len(val) <= len(index) { //table.SortKV.Map.value数组小于等于分包index
  1984. for k, v := range val {
  1985. tn.assemblePackage(k1, v, index[k],isSite,codeSite) //组装解析到的分包
  1986. }
  1987. } else {
  1988. for sk1, sv2 := range index {
  1989. v := val[sk1]
  1990. //处理http://www.hljcg.gov.cn/xwzs!queryOneXwxxqx.action?xwbh=8145b599-a11e-45cb-a76a-12157a715570
  1991. if v == "" && strings.Index(k1, "供应商") > -1 {
  1992. if sk1 != len(index)-1 {
  1993. //u.Debug(val[sk1+1], val[sk1+2])
  1994. if standIndex_pos[sk1+1]-standIndex_pos[sk1] > 1 {
  1995. v = val[standIndex_pos[sk1]+1]
  1996. }
  1997. } else {
  1998. if standIndex_pos[sk1] < len(val)-1 {
  1999. v = val[standIndex_pos[sk1]+1]
  2000. }
  2001. }
  2002. }
  2003. tn.assemblePackage(k1, v, sv2,isSite,codeSite)
  2004. }
  2005. }
  2006. //删除子包的kv
  2007. //u.Debug("----==1==-------", k1)
  2008. k1tags := u.GetTags(k1,isSite,codeSite) //取得匹配
  2009. //if !(len(k1tags) > 0 && k1tags[0].Value == "采购单位") {
  2010. // tn.SortKV.RemoveKey(k1)
  2011. //}
  2012. for _, vcgdw := range k1tags {
  2013. if vcgdw.Value == "采购单位" {
  2014. }
  2015. }
  2016. } else if val, bvs := v1.(string); bvs && len(index) == 1 {
  2017. //删除子包的kv
  2018. kvTags, _ := CommonDataAnaly(k1, "", "", val,isSite,codeSite)
  2019. for kvTag_k, kvTag_v := range kvTags {
  2020. hasValid := false
  2021. for _, kvTag_vv := range kvTag_v {
  2022. if kvTag_vv.IsInvalid {
  2023. continue
  2024. }
  2025. hasValid = true
  2026. }
  2027. if !hasValid {
  2028. continue
  2029. }
  2030. if !(len(kvTags) > 0 && regexp.MustCompile("^(项目|开标|采购单位|招标机构)").MatchString(kvTag_k)) {
  2031. tn.SortKV.RemoveKey(k1)
  2032. tn.assemblePackage(k1, val, index[0],isSite,codeSite)
  2033. //log.Println("remove", k1, val)
  2034. }
  2035. }
  2036. //u.Debug("----==2==-------", k1)
  2037. }
  2038. }
  2039. return isGoonNext
  2040. }
  2041. //没有处理成数组的情况下,继续调用正文查找分包的方法
  2042. func (tn *Table) isGoonNext(isSite bool,codeSite string) {
  2043. blockPackage := map[string]*u.BlockPackage{}
  2044. for _, k := range tn.SortKV.Keys {
  2045. if excludeKey.MatchString(k) || strings.Contains(k, "批复") {
  2046. continue
  2047. }
  2048. str := "" //拼装为冒号kv
  2049. v := tn.SortKV.Map[k]
  2050. nk := regReplAllSpace.ReplaceAllString(k, "")
  2051. if vs, ok := v.([]string); ok {
  2052. str += fmt.Sprintf("%s:%s\n", nk, strings.Join(vs, " "))
  2053. } else {
  2054. str += fmt.Sprintf("%s:%s\n", nk, v)
  2055. }
  2056. b, _ := divisionPackageChild(&blockPackage, str, tn.Tag, false, false,isSite,codeSite) //分块之后分包
  2057. if b && len(blockPackage) > 0 {
  2058. tn.BPackage = true
  2059. for mk, mv := range blockPackage {
  2060. if tn.BlockPackage.Map[mk] == nil {
  2061. tn.BlockPackage.AddKey(mk, mv)
  2062. } else {
  2063. bp := tn.BlockPackage.Map[mk].(*u.BlockPackage)
  2064. if bp.TableKV == nil {
  2065. bp.TableKV = u.NewJobKv()
  2066. }
  2067. if bp.SpaceKV == nil {
  2068. bp.SpaceKV = u.NewJobKv()
  2069. }
  2070. for k2, v2 := range mv.ColonKV.KvTags {
  2071. for _, v2v := range v2 {
  2072. isExists := false
  2073. for _, v2vv := range bp.TableKV.KvTags[k2] {
  2074. if v2v.Value == v2vv.Value {
  2075. isExists = true
  2076. break
  2077. }
  2078. }
  2079. if !isExists {
  2080. bp.TableKV.KvTags[k2] = append(bp.TableKV.KvTags[k2], v2v)
  2081. bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
  2082. }
  2083. }
  2084. }
  2085. for k2, v2 := range mv.SpaceKV.KvTags {
  2086. for _, v2v := range v2 {
  2087. isExists := false
  2088. for _, v2vv := range bp.SpaceKV.KvTags[k2] {
  2089. if v2v.Value == v2vv.Value {
  2090. isExists = true
  2091. break
  2092. }
  2093. }
  2094. if !isExists {
  2095. bp.SpaceKV.KvTags[k2] = append(bp.SpaceKV.KvTags[k2], v2v)
  2096. bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
  2097. }
  2098. }
  2099. }
  2100. }
  2101. }
  2102. tn.BPackage = true
  2103. tn.SortKV.RemoveKey(k)
  2104. }
  2105. }
  2106. }
  2107. //根据table.SortKV的key判断是否分包,如果没有再根据value判断
  2108. func foundPacBySortKV(tn *Table, val int, index []string, index_pos []int, keyExistsCount *map[string]int, commonKeyVals *map[string][]string, key_index int, hasPkgTd map[string]bool) (rval int, rindex []string, rindex_pos []int) {
  2109. keyIsPkg := false
  2110. for in, k := range tn.SortKV.Keys {
  2111. if excludeKey.MatchString(BracketsTextReg.ReplaceAllString(k, "")) { //判断分包前排除
  2112. continue
  2113. }
  2114. v := tn.SortKV.Map[k]
  2115. //key是分包的情况
  2116. if ismatch := FindVal_1.MatchString(k); keyIsPkg || ismatch {
  2117. if ismatch {
  2118. keyIsPkg = true
  2119. val += 4
  2120. pkgFlag := FindVal_1.FindString(k) //对值进行分包判断
  2121. k = strings.Replace(k, pkgFlag, "", -1)
  2122. index = append(index, pkgFlag)
  2123. index_pos = append(index_pos, len(index))
  2124. val += 1
  2125. //pac++
  2126. } else {
  2127. k = strings.TrimRight(k, "_")
  2128. }
  2129. (*keyExistsCount)[k] = (*keyExistsCount)[k] + 1
  2130. (*commonKeyVals)[k] = append((*commonKeyVals)[k], qutil.ObjToString(v))
  2131. } else if k1 := FilterKey_2.ReplaceAllString(k, ""); FindKey_2.MatchString(k1) {
  2132. val += 4
  2133. //value数组分包
  2134. if vs, bvs1 := v.([]string); bvs1 {
  2135. L:
  2136. for in2, v1 := range vs {
  2137. if len([]rune(v1)) < 20 && !moneyNum.MatchString(v1) && FindVal2_1.MatchString(v1) {
  2138. for _, serial := range tn.TableResult.RuleBlock.TitleRegs {
  2139. if serial.MatchString(v1) {
  2140. break L
  2141. }
  2142. }
  2143. if key_index == -1 {
  2144. key_index = in
  2145. } else if key_index != in {
  2146. break
  2147. }
  2148. index = append(index, v1)
  2149. index_pos = append(index_pos, in2)
  2150. val += 1
  2151. //pac++
  2152. }
  2153. }
  2154. } else if v1, ok := v.(string); ok && !hasPkgTd[k] {
  2155. //value字符串分包
  2156. v1 = replPkgConfusion(v1) //替换分包中混淆的词
  2157. for _, v2 := range strings.Split(v1, "/") {
  2158. if len([]rune(v2)) < 20 && !moneyNum.MatchString(v2) && FindVal2_1.MatchString(v2) {
  2159. key_index = in
  2160. index = append(index, v1)
  2161. index_pos = append(index_pos, 0)
  2162. val += 1
  2163. //pac++
  2164. underline := ""
  2165. for {
  2166. underline += "_"
  2167. if tn.SortKV.Map[k+underline] == nil {
  2168. break
  2169. } else if v3, v2_ok := tn.SortKV.Map[k+underline].(string); v2_ok && v3 != "" {
  2170. index = append(index, v3)
  2171. index_pos = append(index_pos, 1)
  2172. } else if v3, v2_ok := tn.SortKV.Map[k+underline].([]string); v2_ok {
  2173. for v2_k, v2_v := range v3 {
  2174. index = append(index, v2_v)
  2175. index_pos = append(index_pos, v2_k+1)
  2176. }
  2177. }
  2178. }
  2179. break
  2180. }
  2181. }
  2182. }
  2183. break
  2184. }
  2185. }
  2186. return val, index, index_pos
  2187. }
  2188. //初始化CheckMultiPackageByTable方法需要的数据
  2189. func initCheckMultiPackageByTable(tn *Table, key_index int, index []string, index_pos []int, val int, pac int, hasPkgTd map[string]bool) (rkey_index int, rindex []string, rindex_pos []int, rval int, rpac int, rhasPkgTd map[string]bool) {
  2190. for in, k := range tn.SortKV.Keys {
  2191. //涉及包号|包件号?|项目标号|规格|型号|招标范围|业绩|废标)|(^编号$)|([^包段标]编号)就跳过
  2192. if excludeKey.MatchString(BracketsTextReg.ReplaceAllString(k, "")) || strings.Contains(k, "批复") {
  2193. continue
  2194. }
  2195. v := tn.SortKV.Map[k]
  2196. if vs, bvs := v.([]string); bvs {
  2197. //arr_count++
  2198. haspkgs := []string{}
  2199. for in2, v1 := range vs {
  2200. v1 = replPkgConfusion(v1) //替换分包中混淆的词
  2201. if len([]rune(v1)) < 8 && !moneyNum.MatchString(v1) && FindVal_1.MatchString(v1) {
  2202. if key_index == -1 {
  2203. key_index = in
  2204. } else if key_index != in {
  2205. break
  2206. }
  2207. index = append(index, FindVal_1.FindString(v1))
  2208. index_pos = append(index_pos, in2)
  2209. val += 1
  2210. pac++
  2211. } else {
  2212. if ok, v1new := isHasOnePkgAndNoKv(v1); ok { //td的值里面有一个包,并且没有冒号kv
  2213. haspkgs = append(haspkgs, v1new)
  2214. }
  2215. }
  2216. }
  2217. /*处理这种情况:
  2218. <tr><td>包一:xxxxxxxxx</td></tr>
  2219. <tr><td>包二:xxxxxxxxx</td></tr>
  2220. */
  2221. if len(index) == 0 && len(haspkgs) > 0 && len(haspkgs) == len(vs) {
  2222. for in2, v1 := range haspkgs {
  2223. if key_index == -1 {
  2224. key_index = in
  2225. } else if key_index != in {
  2226. break
  2227. }
  2228. index = append(index, v1)
  2229. index_pos = append(index_pos, in2)
  2230. val += 1
  2231. pac++
  2232. }
  2233. }
  2234. } else if v1, ok := v.(string); ok {
  2235. v1 = replPkgConfusion(v1) //替换分包中混淆的词
  2236. if len([]rune(v1)) < 8 && !moneyNum.MatchString(v1) && FindVal_1.MatchString(v1) {
  2237. key_index = in
  2238. index = append(index, FindVal_1.FindString(v1))
  2239. index_pos = append(index_pos, 0)
  2240. val += 1
  2241. pac++
  2242. } else if getTd := tn.GetTdByRCNo(0, tn.SortKV.Index[k]); getTd != nil && getTd.KVDirect == 2 { //纵向
  2243. /*处理这种情况:
  2244. <tr><td>包一:xxxxxxxxx</td></tr>
  2245. */
  2246. if ok, v1new := isHasOnePkgAndNoKv(v1); ok {
  2247. hasPkgTd[k] = true
  2248. key_index = in
  2249. index = append(index, v1new)
  2250. index_pos = append(index_pos, 0)
  2251. val += 1
  2252. pac++
  2253. }
  2254. }
  2255. }
  2256. }
  2257. return key_index, index, index_pos, val, pac, hasPkgTd
  2258. }
  2259. //组装解析到的分包,//key如果匹配到抽取关键词就添加到table.SortKV
  2260. func (tn *Table) assemblePackage(k1, v1, key string,isSite bool,codeSite string) {
  2261. bp := tn.BlockPackage.Map[key].(*u.BlockPackage)
  2262. if bp.TableKV == nil {
  2263. bp.TableKV = u.NewJobKv()
  2264. }
  2265. if v1 != "" {
  2266. kvTags, _ := CommonDataAnaly(k1, "中标情况", "", v1,isSite,codeSite) //匹配抽取关键词
  2267. for k3, v3 := range kvTags {
  2268. bp.TableKV.KvTags[k3] = append(bp.TableKV.KvTags[k3], v3...)
  2269. }
  2270. }
  2271. k1 = regReplAllSpace.ReplaceAllString(k1, "")
  2272. //拼接内容
  2273. if !excludeKey.MatchString(k1) {
  2274. bp.Text += fmt.Sprintf("%v:%v\n", k1, v1)
  2275. }
  2276. tn.BlockPackage.AddKey(key, bp)
  2277. }
  2278. /**
  2279. 之前爬虫过来的数据对table表格的抓取异常问题
  2280. 查找并修正不规则表格的字符串,只对全文做处理,块内的表格不需要修正
  2281. **/
  2282. var thbf = regexp.MustCompile("(?i)</?t(head|body|foot)>")
  2283. //需要保留thead
  2284. var saveThead = regexp.MustCompile("(?is)<thead>(.+?)</thead>")
  2285. func RepairCon(con string) string {
  2286. res := saveThead.FindAllStringSubmatch(con, 1)
  2287. th := ""
  2288. if len(res) == 1 && len(res[0]) == 2 {
  2289. th = u.TrimLeftSpace(res[0][1], "")
  2290. }
  2291. con = thbf.ReplaceAllString(con, "")
  2292. con = u.TrimLeftSpace(con, "")
  2293. itbody := strings.Index(con, "<tr")
  2294. iLen := 3
  2295. if itbody == 0 {
  2296. con = findpos(con, iLen, itbody)
  2297. } else {
  2298. itable := strings.Index(con, "<table")
  2299. if itable == -1 || itable > itbody {
  2300. con = findpos(con, iLen, itbody)
  2301. }
  2302. }
  2303. //保留第一个thead
  2304. if th != "" {
  2305. con = strings.Replace(con, th, "<thead>"+th+"</thead>", 1)
  2306. }
  2307. //u.Debug(con)
  2308. return con
  2309. }
  2310. //修复表格
  2311. func findpos(con string, iLen, start int) (newcon string) {
  2312. defer qutil.Catch()
  2313. n := len(con)
  2314. layer := 0
  2315. pos := 0
  2316. if start >= 0 {
  2317. if iLen == 6 {
  2318. for i := iLen + start; i < len(con); i++ {
  2319. if con[i] == '<' && i+6 < n {
  2320. str := con[i : i+6]
  2321. if str == "</tbod" {
  2322. if layer == 0 {
  2323. pos = i
  2324. break
  2325. } else {
  2326. layer--
  2327. }
  2328. i += 6
  2329. } else if str == "<tbody" {
  2330. layer++
  2331. i += 6
  2332. }
  2333. }
  2334. }
  2335. if pos+7 <= n && start+6 < pos {
  2336. newcon = con[:start] + "<table" + con[start+6:pos] + "</table" + con[pos+7:]
  2337. }
  2338. } else {
  2339. layer++
  2340. nq := 0
  2341. lasttr := 0
  2342. for i := iLen + start; i < len(con); i++ {
  2343. if con[i] == '<' && i+4 < n {
  2344. if nq == 0 {
  2345. str := con[i : i+4]
  2346. if str == "</tr" {
  2347. if layer <= 0 {
  2348. pos = i //正常情况不会存在此类情况
  2349. break
  2350. } else {
  2351. layer--
  2352. lasttr = i
  2353. }
  2354. i += 4
  2355. } else if str[:3] == "<tr" {
  2356. layer++
  2357. i += 4
  2358. } else if str == "<tab" && i+6 < n && con[i+4:i+6] == "le" {
  2359. if layer == 0 {
  2360. break
  2361. } else {
  2362. //内嵌的表格
  2363. nq++
  2364. }
  2365. }
  2366. } else {
  2367. if i+6 < n {
  2368. str := con[i : i+6]
  2369. if str == "</tabl" {
  2370. nq--
  2371. } else if str == "<table" {
  2372. nq++
  2373. }
  2374. } else {
  2375. break
  2376. }
  2377. }
  2378. }
  2379. }
  2380. if pos == 0 && lasttr > 3 {
  2381. pos = lasttr + 5
  2382. } else if pos > 0 {
  2383. pos += 5
  2384. }
  2385. if pos <= n && pos < len(con) && start < pos {
  2386. newcon = con[:start] + "<table>" + con[start:pos] + "</table>" + con[pos:]
  2387. }
  2388. }
  2389. }
  2390. if newcon == "" {
  2391. newcon = con
  2392. }
  2393. return
  2394. }
  2395. //td的值里面有一个包,并且没有冒号kv
  2396. func isHasOnePkgAndNoKv(v1 string) (bool, string) {
  2397. v1s := FindVal_1.FindAllString(v1, -1)
  2398. colonCount := len(regDivision.FindAllString(v1, -1))
  2399. if len(v1s) == 1 && colonCount < 2 {
  2400. ispkgcolon := regexp.MustCompile(v1s[0] + "[::]").MatchString(v1)
  2401. if (ispkgcolon && colonCount == 1) || (!ispkgcolon && colonCount == 0) {
  2402. return true, v1s[0]
  2403. }
  2404. }
  2405. return false, v1
  2406. }
  2407. //替换分包中混淆的词
  2408. func replPkgConfusion(v1 string) string {
  2409. v1 = PreReg.ReplaceAllString(v1, "")
  2410. v1 = PreReg1.ReplaceAllString(v1, "")
  2411. v1 = PreCon.ReplaceAllString(v1, "")
  2412. v1 = PreCon2.ReplaceAllString(v1, "")
  2413. return v1
  2414. }
  2415. //对td中的值,进行再处理
  2416. func (tn *Table) TdContactFormat(contactFormat *u.ContactFormat,isSite bool,codeSite string) {
  2417. //处理表格中的联系人信息
  2418. indexMap := contactFormat.IndexMap
  2419. matchMap := contactFormat.MatchMap
  2420. //qutil.Debug("==============================td=======================", indexMap, matchMap)
  2421. weightMap := map[string]map[string]interface{}{} //权重
  2422. mustMatchFirst := len(indexMap) > 0 //第一个必须匹配上
  2423. reCreate := false
  2424. matchCount := 0
  2425. contactTypeTagMap := map[string]map[string][]interface{}{}
  2426. //u.Debug(mustMatchFirst, indexMap, matchMap)
  2427. notMatchTrCount := 0
  2428. allAscFind := true //开启正序查询
  2429. //涉及变量allAscFind,indexMap
  2430. if len(indexMap) == 0 {
  2431. isCanAddToIndexMap := false
  2432. matchPrevFlag := false
  2433. prevCanAddToIndexMap := false
  2434. LS:
  2435. for _, tr := range tn.TRs {
  2436. for td_index, td := range tr.TDs {
  2437. thisTdKvs := tn.tdkv(td) //获取td冒号kv
  2438. if len(thisTdKvs) != 1 {
  2439. continue
  2440. }
  2441. //1.处理带括号的()[]【】采购单位,代理机构;2.识别采购单位联系人、联系电话、代理机构联系人、联系电话
  2442. goOnFunc, isContinue, td_k := tn.tdKV(thisTdKvs[0].Key, &matchPrevFlag, &isCanAddToIndexMap, &indexMap, "LS")
  2443. if !goOnFunc {
  2444. break LS
  2445. }
  2446. if isContinue {
  2447. continue
  2448. }
  2449. //采购单位,代理机构
  2450. for _, k := range HasOrderContactType(td_k) {
  2451. if !ContactType[k].MatchString(td_k) { //不是采购单位,代理机构跳过
  2452. continue
  2453. }
  2454. if len(indexMap) == 0 {
  2455. if isCanAddToIndexMap || (prevCanAddToIndexMap && len(tr.TDs) == 1) {
  2456. myPrevTdVal := ""
  2457. if td_index-2 >= 0 {
  2458. myPrevTdVal = tr.TDs[td_index-2].Val
  2459. if myPrevTdVal != "" && len([]rune(myPrevTdVal)) < 10 && ContactInfoMustReg.MatchString(myPrevTdVal) {
  2460. matchPrevFlag = true
  2461. }
  2462. }
  2463. indexMap[0] = k
  2464. break
  2465. }
  2466. } else {
  2467. indexMap = map[int]string{}
  2468. break LS
  2469. }
  2470. }
  2471. }
  2472. prevCanAddToIndexMap = isCanAddToIndexMap
  2473. isCanAddToIndexMap = false
  2474. }
  2475. if len(indexMap) > 0 {
  2476. allAscFind = false
  2477. }
  2478. }
  2479. //////
  2480. L:
  2481. for tr_index, tr := range tn.TRs {
  2482. thisTrHasMatch := false
  2483. jumpNextTd := false
  2484. for td_index, td := range tr.TDs {
  2485. //和|以?及|与|、多个词和在一起
  2486. jumpNextTd, thisTrHasMatch = tn.tdsMultipleWords(jumpNextTd, td, td_index, tr, thisTrHasMatch, indexMap)
  2487. //分块之后的kv
  2488. thisTdKvs := kvAfterDivideBlock("", td.Text, 3, tn.TableResult.RuleBlock,isSite,codeSite)
  2489. if len(thisTdKvs) == 0 {
  2490. thisTdKvs = tn.tdkv(td) //获取冒号kv
  2491. }
  2492. tdAscFind := true //开启td正序查询
  2493. if len(thisTdKvs) == 0 {
  2494. continue
  2495. } else if allAscFind && len(thisTdKvs) >= 3 && len(indexMap) == 0 {
  2496. //采购人在联系人、电话后面的处理
  2497. tdAscFind = tn.hasIndexMap(thisTdKvs, &indexMap, tdAscFind)
  2498. }
  2499. prevKey := ""
  2500. oldIndexMapLength := len(indexMap)
  2501. thidTdIndex := td_index
  2502. //notmatchCount := 0
  2503. kvTitle := ""
  2504. for _, td_kv := range thisTdKvs {
  2505. //u.Debug(td_kv.Key, td_kv.Value, td_kv.Title)
  2506. iscontinue := false
  2507. td_v := td_kv.Value
  2508. td_k := FilterContactKey(td_kv.Key) //带括号()[]的采购单位,代理机构处理
  2509. td_k_length := len([]rune(td_k))
  2510. if td_k_length < 2 || td_k_length > 15 {
  2511. continue
  2512. }
  2513. //都为正序查询
  2514. if allAscFind && tdAscFind {
  2515. //都为正序查询处理
  2516. matchCount, weightMap, matchMap, thisTrHasMatch, indexMap, iscontinue, reCreate, thidTdIndex = tn.asdFind(td_k, matchCount, weightMap, matchMap, td, thisTrHasMatch, td_kv, indexMap, iscontinue, reCreate, thidTdIndex,isSite,codeSite)
  2517. }
  2518. if iscontinue {
  2519. continue
  2520. }
  2521. //不在同一块中
  2522. if td_kv.Title != "" && kvTitle != td_kv.Title && len(indexMap) > 0 && !ContactInfoMustReg.MatchString(td_kv.Key) {
  2523. thidTdIndex = 0
  2524. matchMap = map[string]map[string]bool{}
  2525. indexMap = map[int]string{}
  2526. }
  2527. kvTitle = td_kv.Title
  2528. //u.Debug(indexMap, td_k, td_v, matchMap)
  2529. if td_k_length < 2 || td_k_length > 10 {
  2530. continue
  2531. }
  2532. if len(indexMap) > 0 {
  2533. //没有识别到采购单位联系人、联系电话、代理机构联系人、联系电话
  2534. if !ContactInfoMustReg.MatchString(td_k) {
  2535. //notmatchCount++
  2536. //if notmatchCount < len(indexMap)*2 && false {//false???????
  2537. // notmatchCount = 0
  2538. // thidTdIndex = 0
  2539. // indexMap = map[int]string{}
  2540. // matchMap = map[string]map[string]bool{}
  2541. //}
  2542. if mustMatchFirst { //indexMap初始值大于0
  2543. break L
  2544. }
  2545. continue
  2546. }
  2547. reCreate = true
  2548. index := td_index
  2549. //oldIndexMapLength原来的indexMap等于0 ,现在的indexMap大于1
  2550. if oldIndexMapLength == 0 && len(indexMap) > 1 {
  2551. if prevKey != td_k {
  2552. prevKey = td_k
  2553. index = td_index
  2554. } else if prevKey == td_k {
  2555. index++
  2556. }
  2557. }
  2558. //kv.value为空
  2559. if filterValue.MatchString(td_v) {
  2560. thisTrHasMatch = true
  2561. continue
  2562. }
  2563. //u.Debug(indexMap, td_k, td_v, matchMap, index, modle)
  2564. //myContactType
  2565. myContactType := indexMap[index]
  2566. if myContactType == "" && len(indexMap) == 1 {
  2567. _, onlyContactType := u.FirstKeyValueInMap(indexMap)
  2568. myContactType, _ = onlyContactType.(string)
  2569. }
  2570. if myContactType == "" {
  2571. continue
  2572. }
  2573. matchCount++
  2574. if matchMap[myContactType] == nil {
  2575. matchMap[myContactType] = map[string]bool{}
  2576. }
  2577. if IsContactKvHandle(ContactInfoMustReg.FindString(td_k), matchMap[myContactType]) {
  2578. continue
  2579. }
  2580. matchMap[myContactType][ContactInfoMustReg.FindString(td_k)] = true
  2581. if ContactType[myContactType].MatchString(td_k) {
  2582. continue
  2583. }
  2584. thisTrHasMatch = true
  2585. //modle
  2586. modle(thisTdKvs, td, myContactType, td_k, td_v, &contactTypeTagMap, tn, &weightMap, tr_index, td_index,isSite,codeSite)
  2587. }
  2588. }
  2589. //u.Debug(td.SortKV.Map)
  2590. }
  2591. if allAscFind && !thisTrHasMatch {
  2592. notMatchTrCount++
  2593. if notMatchTrCount >= 2 {
  2594. notMatchTrCount = 0
  2595. indexMap = map[int]string{}
  2596. }
  2597. }
  2598. }
  2599. //u.Debug("end", matchCount, indexMap, matchMap)
  2600. if matchCount == 0 {
  2601. indexMap = map[int]string{}
  2602. matchMap = map[string]map[string]bool{}
  2603. }
  2604. (*contactFormat).IndexMap = indexMap
  2605. (*contactFormat).MatchMap = matchMap
  2606. // for _, tr := range tn.TRs {
  2607. // for _, td := range tr.TDs {
  2608. // qutil.Debug("td.sort.map---", td.SortKV.Map)
  2609. // }
  2610. // }
  2611. }
  2612. //modle
  2613. func modle(thisTdKvs []*u.Kv, td *TD, myContactType, td_k, td_v string, contactTypeTagMap *map[string]map[string][]interface{}, tn *Table, weightMap *map[string]map[string]interface{}, tr_index, td_index int,isSite bool,codeSite string) {
  2614. modle := 0
  2615. if len(thisTdKvs) == 1 {
  2616. if regReplAllSpace.ReplaceAllString(thisTdKvs[0].Value, "") == "" {
  2617. modle = 1
  2618. } else {
  2619. modle = 2
  2620. }
  2621. }
  2622. if modle == 1 {
  2623. td.Text = myContactType + td_k
  2624. td.Val = td.Text
  2625. } else {
  2626. //
  2627. if !strings.HasSuffix(td_k, "方式") {
  2628. kvTags := GetKvTags([]*u.Kv{&u.Kv{Key: myContactType + td_k, Value: td_v}}, "", BuyerContacts,isSite,codeSite)
  2629. if len(kvTags) == 1 {
  2630. tagVal, _ := u.FirstKeyValueInMap(kvTags)
  2631. if tagVal == "采购单位联系人" && ContactBuyerPersonFilterReg.MatchString(td_v) {
  2632. return
  2633. }
  2634. if (*contactTypeTagMap)[myContactType] == nil {
  2635. (*contactTypeTagMap)[myContactType] = map[string][]interface{}{}
  2636. }
  2637. myOldKeyArray := (*contactTypeTagMap)[myContactType][tagVal]
  2638. if myOldKeyArray != nil {
  2639. tn.TRs[myOldKeyArray[0].(int)].TDs[myOldKeyArray[1].(int)].SortKV.RemoveKey(myContactType + myOldKeyArray[2].(string))
  2640. } else {
  2641. (*contactTypeTagMap)[myContactType][tagVal] = make([]interface{}, 3)
  2642. }
  2643. if (*weightMap)[myContactType] == nil {
  2644. (*weightMap)[myContactType] = map[string]interface{}{}
  2645. }
  2646. (*weightMap)[myContactType][tagVal] = 1
  2647. (*contactTypeTagMap)[myContactType][tagVal] = []interface{}{tr_index, td_index, td_k}
  2648. }
  2649. }
  2650. td.SortKV.AddKey(myContactType+td_k, td_v)
  2651. }
  2652. }
  2653. //都为正序查询
  2654. func (tn *Table) asdFind(td_k string, matchCount int, weightMap map[string]map[string]interface{}, matchMap map[string]map[string]bool, td *TD, thisTrHasMatch bool, td_kv *u.Kv, indexMap map[int]string, iscontinue bool, reCreate bool, thidTdIndex int,isSite bool,codeSite string) (int, map[string]map[string]interface{}, map[string]map[string]bool, bool, map[int]string, bool, bool, int) {
  2655. for _, k := range HasOrderContactType(td_k) { //采购单位,代理机构
  2656. if !ContactType[k].MatchString(td_k) { //没有匹配到采购单位,代理机构
  2657. continue
  2658. }
  2659. matchCount++
  2660. if weightMap[k] == nil {
  2661. weightMap[k] = map[string]interface{}{}
  2662. }
  2663. //匹配到进行处理
  2664. if ContactInfoVagueReg.MatchString(td_k) {
  2665. thisTrHasMatch = tn.matchContactType(&matchMap, k, td_k, td_kv.Value, td, &weightMap, thisTrHasMatch,isSite,codeSite)
  2666. } else if k == "采购单位" { //打标签,权重高的重新覆盖
  2667. kvTags := GetKvTags([]*u.Kv{td_kv}, "", []string{"采购单位"},isSite,codeSite)
  2668. tagVal, weightVal := u.FirstKeyValueInMap(kvTags)
  2669. if tagVal == k {
  2670. if weightMap[k][k] == nil || (weightVal != nil && weightVal.(int) >= weightMap[k][k].(int)) || len(matchMap[k]) == 0 {
  2671. weightMap[k][k] = weightVal.(int)
  2672. matchMap[k] = map[string]bool{}
  2673. indexMap = map[int]string{}
  2674. }
  2675. }
  2676. }
  2677. if u.IsMapHasValue(k, indexMap) { //map中是否存在value
  2678. thisTrHasMatch = true
  2679. iscontinue = true
  2680. continue
  2681. }
  2682. if reCreate {
  2683. indexMap = map[int]string{}
  2684. reCreate = false
  2685. }
  2686. indexMap[thidTdIndex] = k
  2687. iscontinue = true
  2688. thisTrHasMatch = true
  2689. thidTdIndex++
  2690. break
  2691. }
  2692. if len(indexMap) == 0 && td_kv.PrevLine != "" {
  2693. //td_kv.PrevLine
  2694. prevLine := FilterSerial.ReplaceAllString(td_kv.PrevLine, "")
  2695. for k, v := range ContactType { //采购单位,代理机构正则
  2696. if u.IsArrayHasValue(prevLine, v.FindAllString(prevLine, -1)) {
  2697. indexMap[thidTdIndex] = k
  2698. thisTrHasMatch = true
  2699. thidTdIndex++
  2700. }
  2701. }
  2702. }
  2703. if len(indexMap) == 0 && td_kv.Title != "" {
  2704. //td_kv.Title
  2705. if titleMatchType := ContactTypeTitleMatch(td_kv.Title); titleMatchType != "" {
  2706. thidTdIndex = 0
  2707. matchMap = map[string]map[string]bool{}
  2708. indexMap = map[int]string{1: titleMatchType}
  2709. // for i, t := range titleMatchType {
  2710. // indexMap[i+1] = t
  2711. // }
  2712. }
  2713. }
  2714. return matchCount, weightMap, matchMap, thisTrHasMatch, indexMap, iscontinue, reCreate, thidTdIndex
  2715. }
  2716. //匹配到进行处理
  2717. func (tn *Table) matchContactType(matchMap *map[string]map[string]bool, k string, td_k string, td_v string, td *TD, weightMap *map[string]map[string]interface{}, thisTrHasMatch bool,isSite bool,codeSite string) bool {
  2718. if (*matchMap)[k] == nil {
  2719. (*matchMap)[k] = map[string]bool{}
  2720. }
  2721. isAddToMatchMap := true
  2722. if !strings.HasSuffix(td_k, "方式") {
  2723. kvTags := GetKvTags([]*u.Kv{&u.Kv{Key: td_k, Value: td_v}}, "", BuyerContacts,isSite,codeSite)
  2724. if len(kvTags) == 1 {
  2725. tagVal, weightVal := u.FirstKeyValueInMap(kvTags)
  2726. if tagVal == "采购单位联系人" && ContactBuyerPersonFilterReg.MatchString(td_v) {
  2727. isAddToMatchMap = false
  2728. }
  2729. if td.SortKV.Map[tagVal] != nil {
  2730. if (*weightMap)[k][tagVal] == nil || (weightVal != nil && weightVal.(int) >= (*weightMap)[k][tagVal].(int)) {
  2731. (*weightMap)[k][tagVal] = weightVal.(int)
  2732. td.SortKV.AddKey(tagVal, td_v)
  2733. thisTrHasMatch = true
  2734. }
  2735. } else {
  2736. (*weightMap)[k][tagVal] = weightVal.(int)
  2737. }
  2738. }
  2739. }
  2740. if isAddToMatchMap && !filterValue.MatchString(td_v) && td_v != "" {
  2741. (*matchMap)[k][ContactInfoVagueReg.FindString(td_k)] = true
  2742. }
  2743. return thisTrHasMatch
  2744. }
  2745. //采购人在联系人、电话后面的处理
  2746. func (tn *Table) hasIndexMap(thisTdKvs []*u.Kv, indexMap *map[int]string, tdAscFind bool) bool {
  2747. //采购人在联系人、电话后面的处理
  2748. isCanAddToIndexMap := false
  2749. LL:
  2750. for _, td_kv := range thisTdKvs {
  2751. //1.处理带括号的()[]【】采购单位,代理机构;2.识别采购单位联系人、联系电话、代理机构联系人、联系电话
  2752. goOnFunc, isContinue, td_k := tn.tdKV(td_kv.Key, nil, &isCanAddToIndexMap, indexMap, "LL")
  2753. if !goOnFunc {
  2754. break LL
  2755. }
  2756. if isContinue {
  2757. continue
  2758. }
  2759. if len(*indexMap) == 0 {
  2760. for _, k := range HasOrderContactType(td_k) { //采购单位,代理机构
  2761. if !ContactType[k].MatchString(td_k) {
  2762. continue
  2763. }
  2764. if isCanAddToIndexMap && len(*indexMap) == 0 {
  2765. (*indexMap)[0] = k
  2766. break
  2767. }
  2768. }
  2769. }
  2770. }
  2771. if len(*indexMap) > 0 {
  2772. tdAscFind = false
  2773. }
  2774. return tdAscFind
  2775. }
  2776. //和|以?及|与|、多个词和在一起
  2777. func (tn *Table) tdsMultipleWords(jumpNextTd bool, td *TD, td_index int, tr *TR, thisTrHasMatch bool, indexMap map[int]string) (jump, thisTr bool) {
  2778. if !jumpNextTd && len([]rune(td.Text)) >= 5 && len([]rune(td.Text)) <= 15 && regSplit.MatchString(td.Text) && td_index+1 < len(tr.TDs) {
  2779. thisTdVals := regSplit.Split(td.Text, -1)
  2780. nextTdVals := MultipleValueSplitReg.Split(tr.TDs[td_index+1].Val, -1)
  2781. if len(thisTdVals) == len(nextTdVals) { //本次和下个长度相等
  2782. for _, k := range HasOrderContactType(td.Text) { //采购单位,代理机构
  2783. if ContactType[k].MatchString(td.Text) { //采购单位,代理机构
  2784. for thisTdVals_k, thisTdVals_v := range thisTdVals {
  2785. thisTdVals_v = strings.TrimSpace(thisTdVals_v)
  2786. if ContactType[k].MatchString(thisTdVals_v) { //采购单位,代理机构
  2787. thisTrHasMatch = true
  2788. tr.TDs[td_index+1].SortKV.AddKey(thisTdVals_v, nextTdVals[thisTdVals_k])
  2789. continue
  2790. }
  2791. if !ContactInfoMustReg.MatchString(thisTdVals_v) {
  2792. continue
  2793. }
  2794. jumpNextTd = true
  2795. thisTrHasMatch = true
  2796. tr.TDs[td_index+1].SortKV.AddKey(k+thisTdVals_v, nextTdVals[thisTdVals_k])
  2797. }
  2798. break
  2799. }
  2800. }
  2801. if len(indexMap) > 0 {
  2802. _, onlyContactType := u.FirstKeyValueInMap(indexMap)
  2803. if myContactType, _ := onlyContactType.(string); myContactType != "" {
  2804. for thisTdVals_k, thisTdVals_v := range thisTdVals {
  2805. thisTdVals_v = strings.TrimSpace(thisTdVals_v)
  2806. if ContactInfoMustReg.MatchString(thisTdVals_v) {
  2807. jumpNextTd = true
  2808. thisTrHasMatch = true
  2809. tr.TDs[td_index+1].SortKV.AddKey(myContactType+thisTdVals_v, nextTdVals[thisTdVals_k])
  2810. }
  2811. }
  2812. }
  2813. }
  2814. }
  2815. } else {
  2816. jumpNextTd = false
  2817. }
  2818. return jumpNextTd, thisTrHasMatch
  2819. }
  2820. //采购单位,代理机构
  2821. func (tn *Table) tdHasOrderContactType(td_k string, indexMap *map[int]string, tr *TR, prevCanAddToIndexMap, isCanAddToIndexMap, matchPrevFlag *bool, td_index int) (gotoFunc bool) {
  2822. for _, k := range HasOrderContactType(td_k) { //采购单位,代理机构
  2823. if !ContactType[k].MatchString(td_k) {
  2824. continue
  2825. }
  2826. if len(*indexMap) == 0 {
  2827. if (*isCanAddToIndexMap) || (*prevCanAddToIndexMap && len(tr.TDs) == 1) {
  2828. myPrevTdVal := ""
  2829. if td_index-2 >= 0 {
  2830. myPrevTdVal = tr.TDs[td_index-2].Val
  2831. }
  2832. if myPrevTdVal != "" && len([]rune(myPrevTdVal)) < 10 && ContactInfoMustReg.MatchString(myPrevTdVal) {
  2833. (*matchPrevFlag) = true
  2834. }
  2835. (*indexMap)[0] = k
  2836. break
  2837. }
  2838. } else {
  2839. (*indexMap) = map[int]string{}
  2840. return false
  2841. }
  2842. }
  2843. return true
  2844. }
  2845. //1.处理带括号的()[]【】采购单位,代理机构;2.识别采购单位联系人、联系电话、代理机构联系人、联系电话
  2846. func (tn *Table) tdKV(key string, matchPrevFlag, isCanAddToIndexMap *bool, indexMap *map[int]string, gotoName string) (goOnFunc, isContinue bool, td_k string) {
  2847. td_k = FilterContactKey(key) //带括号的()[]【】采购单位,代理机构处理
  2848. td_k_length := len([]rune(td_k))
  2849. if td_k_length < 2 || td_k_length > 15 {
  2850. return true, true, "" //继续执行,跳过当前循环
  2851. }
  2852. isContinue = ContactInfoMustReg.MatchString(td_k) //识别采购单位联系人、联系电话、代理机构联系人、联系电话
  2853. if isContinue || (ContactInfoVagueReg.MatchString(td_k) && u.IsMapHasValue(td_k, ContactType)) {
  2854. if gotoName == "LS" && !(*matchPrevFlag) && len(*indexMap) > 0 {
  2855. (*indexMap) = map[int]string{}
  2856. return false, false, "" //中断外层循环
  2857. }
  2858. if gotoName == "LL" && len(*indexMap) > 0 {
  2859. (*indexMap) = map[int]string{}
  2860. return false, false, ""
  2861. }
  2862. (*isCanAddToIndexMap) = true
  2863. }
  2864. return true, false, td_k //继续执行,不结束当前循环,返回处理后的值
  2865. }
  2866. //获取td冒号kv
  2867. func (tn *Table) tdkv(td *TD) []*u.Kv {
  2868. thisTdKvs := colonkvEntity.GetKvs(td.Text, tn.Desc, 2) //获取冒号kv
  2869. //获取冒号
  2870. if len(thisTdKvs) == 0 {
  2871. tdValue := regReplAllSpace.ReplaceAllString(td.Text, "") //去除空格换行
  2872. if tdValue != "" && len([]rune(tdValue)) < 10 {
  2873. thisTdKvs = append(thisTdKvs, &u.Kv{
  2874. Key: tdValue,
  2875. Value: "",
  2876. })
  2877. }
  2878. }
  2879. return thisTdKvs
  2880. }
  2881. //table中抽取品牌,table.BrandData
  2882. func (table *Table) analyBrand() {
  2883. //5c2d8c05a5cb26b9b782572b
  2884. //产品名称 品牌 规格 单价 单位 数量 小计 质保期
  2885. lineMapArr := make(map[string]*SortMap)
  2886. lineMap := make(map[string]*SortMap)
  2887. brandRule := u.BrandRules
  2888. //初始化lineMapArr,lineMap;
  2889. lineMapArr, lineMap = initLineMapLineMapArr(table) //处理数组数据后,匹配必须title和替换要保存的title
  2890. //qutil.Debug("lineMapArr----", len(lineMapArr))
  2891. if len(lineMapArr) > 0 {
  2892. for _, aMap := range lineMapArr {
  2893. maxNum := 0 //记录最大长度
  2894. arrcount1 := 0 //记录key是否存在必须title(数组数据)
  2895. arrcount2 := 0
  2896. ka := make(map[string][]string) //最终存储数据
  2897. //qutil.Debug("aMap.Keys----", aMap.Keys)
  2898. //匹配商品规则
  2899. arrcount1, arrcount2, ka = table.matchMapArrBrandRule(aMap, brandRule, ka, arrcount1, arrcount2)
  2900. //找最终存储数据的最小len(arr)
  2901. // for _, vf := range ka {
  2902. // //找最短的数组
  2903. // lenVal := len(vf)
  2904. // if minNum == 0 || minNum > lenVal { //maxNum = len(最短数组)
  2905. // minNum = lenVal
  2906. // }
  2907. // }
  2908. //找最终存储数据的最大len(arr),小的补空
  2909. for _, vf1 := range ka {
  2910. lenVal := len(vf1)
  2911. if lenVal > maxNum {
  2912. maxNum = lenVal
  2913. }
  2914. }
  2915. //table.BrandData商品存储
  2916. finishKa := make(map[string][]string)
  2917. for vf2K, vf2 := range ka {
  2918. if len(vf2) < maxNum {
  2919. if vf2K == "unitprice" { //价格的当前总数比最大的总数小就跳过,可能是总价格而不是单个的价格
  2920. continue
  2921. }
  2922. lenMv := maxNum - len(vf2)
  2923. for i := 0; i < lenMv; i++ {
  2924. vf2 = append(vf2, "")
  2925. }
  2926. }
  2927. finishKa[vf2K] = vf2
  2928. }
  2929. hasKey(table, arrcount1) //是否匹配到table中的标题
  2930. //qutil.Debug("finishKa----", finishKa)
  2931. if arrcount1 >= 1 {
  2932. if arrcount1+arrcount2 == 1 { //删除只匹配到一个价钱(总价)
  2933. delete(finishKa, "unitprice")
  2934. }
  2935. finishData := dealArrData(maxNum, finishKa)
  2936. table.BrandData = append(table.BrandData, finishData) //存储了table.BrandData
  2937. }
  2938. }
  2939. }
  2940. //处理string数据后,匹配必须title和替换要保存的title
  2941. //qutil.Debug("lineMap----", len(lineMap))
  2942. if len(lineMap) > 0 {
  2943. for _, sMap := range lineMap {
  2944. strcount1 := 0 //记录key是否存在必须title(字符串数据)
  2945. strcount2 := 0
  2946. endStrMap := make(map[string]string)
  2947. //qutil.Debug(k, "aMap.Keys----", sMap.Keys)
  2948. //匹配商品规则
  2949. strcount1, strcount2, endStrMap = table.matchMapBrandRule(sMap, brandRule, endStrMap, strcount1, strcount2)
  2950. //原始字符串数据处理
  2951. hasKey(table, strcount1) //是否匹配到table中的标题
  2952. //qutil.Debug("endStrMap----", endStrMap)
  2953. if strcount1 >= 1 {
  2954. if strcount1+strcount2 == 1 { //删除只匹配到一个价钱(总价)
  2955. delete(endStrMap, "unitprice")
  2956. }
  2957. finishData := dealStrData(endStrMap) //处理数据
  2958. if len(finishData) > 0 {
  2959. table.BrandData = append(table.BrandData, finishData)
  2960. }
  2961. }
  2962. }
  2963. }
  2964. }
  2965. //字符串匹配商品规则
  2966. func (table *Table) matchMapBrandRule(sMap *SortMap, brandRule map[string]map[string]string, endStrMap map[string]string, strcount1, strcount2 int) (int, int, map[string]string) {
  2967. for _, k1 := range sMap.Keys {
  2968. match := false //记录must是否匹配到
  2969. v1 := qutil.ObjToString(sMap.Map[k1])
  2970. // for k1, v1 := range sMap {
  2971. //qutil.Debug(k1, "++++++++++", v1)
  2972. if v1 == "" {
  2973. continue
  2974. }
  2975. //匹配必须title
  2976. for nameM, r := range brandRule["must"] {
  2977. if convert(k1, r) { //匹配成功
  2978. v1tmp1 := v1
  2979. match = true
  2980. if nameM == "itemname" || nameM == "modal" { //特殊处理itemname
  2981. hasGoods(table, v1)
  2982. if nameM == "itemname" {
  2983. v1tmp1 = filterItem(v1)[0] //过滤itemname
  2984. if v1tmp1 == "" {
  2985. break
  2986. }
  2987. }
  2988. }
  2989. if nameM == "brandname" || nameM == "modal" { //特殊处理brandname
  2990. if endStrMap["brandname"] == "" {
  2991. brand, allNull := hasBrand(table, v1)
  2992. if !allNull {
  2993. endStrMap["brandname"] = brand[0]
  2994. }
  2995. }
  2996. }
  2997. //unitprice
  2998. if nameM == "unitprice" { //处理金额
  2999. v1tmp1 = dealPrice(k1, v1)[0]
  3000. }
  3001. if nameM != "brandname" && endStrMap[nameM] == "" {
  3002. endStrMap[nameM] = v1tmp1
  3003. }
  3004. strcount1++
  3005. }
  3006. }
  3007. //替换其它要保存字段
  3008. if !match {
  3009. for nameR, r := range brandRule["replace"] {
  3010. if convert(k1, r) { //匹配成功
  3011. v1tmp2 := v1
  3012. //totalprice
  3013. if nameR == "totalprice" { //处理金额
  3014. v1tmp2 = dealPrice(k1, v1)[0]
  3015. }
  3016. //number
  3017. if nameR == "number" { //处理数量
  3018. varr1, uname1 := dealNumber(v1)
  3019. v1tmp2 = varr1[0]
  3020. //从number中获取到的单位
  3021. if endStrMap["unitname"] == "" && uname1[0] != "" {
  3022. endStrMap["unitname"] = uname1[0]
  3023. }
  3024. }
  3025. if v1tmp2 != "" {
  3026. endStrMap[nameR] = v1tmp2
  3027. }
  3028. strcount2++
  3029. }
  3030. }
  3031. }
  3032. //}
  3033. }
  3034. return strcount1, strcount2, endStrMap
  3035. }
  3036. //数组匹配商品规则
  3037. func (table *Table) matchMapArrBrandRule(aMap *SortMap, brandRule map[string]map[string]string, ka map[string][]string, arrcount1, arrcount2 int) (int, int, map[string][]string) {
  3038. for _, k0 := range aMap.Keys {
  3039. match := false //记录must是否匹配到
  3040. v0 := aMap.Map[k0].([]string)
  3041. //匹配必须title
  3042. for nameM, r := range brandRule["must"] {
  3043. if convert(k0, r) { //匹配成功
  3044. v0tmp1 := v0
  3045. match = true
  3046. if len(ka[nameM]) != 0 && strings.Contains(k0, "描述") { //防止k0匹配到多次 和特殊情况 物料名称 物料描述同时出现
  3047. continue
  3048. }
  3049. if nameM == "itemname" || nameM == "modal" {
  3050. hasGoods(table, v0...) //判断itemname和modal中有没有商品
  3051. if nameM == "itemname" {
  3052. v0tmp1 = filterItem(v0...) //过滤itemname
  3053. }
  3054. }
  3055. if nameM == "brandname" || nameM == "modal" {
  3056. if len(ka["brandname"]) == 0 {
  3057. brand, allNull := hasBrand(table, v0...)
  3058. if !allNull {
  3059. ka["brandname"] = brand
  3060. }
  3061. }
  3062. }
  3063. //unitprice
  3064. if nameM == "unitprice" { //处理金额
  3065. v0tmp1 = dealPrice(k0, v0...)
  3066. }
  3067. if nameM != "brandname" && len(ka[nameM]) == 0 {
  3068. ka[nameM] = v0tmp1
  3069. }
  3070. arrcount1++
  3071. }
  3072. }
  3073. //替换其它要保存字段
  3074. if !match { //must未匹配,匹配replace
  3075. for nameR, r := range brandRule["replace"] {
  3076. if convert(k0, r) { //匹配成功
  3077. v0tmp2 := v0
  3078. //totalprice
  3079. if nameR == "totalprice" { //处理金额
  3080. v0tmp2 = dealPrice(k0, v0...)
  3081. }
  3082. //number
  3083. if nameR == "number" { //处理数量
  3084. uname0 := []string{}
  3085. v0tmp2, uname0 = dealNumber(v0...)
  3086. if len(ka["unitname"]) == 0 && len(uname0) != 0 {
  3087. ka["unitname"] = uname0
  3088. }
  3089. }
  3090. if len(v0tmp2) > 0 {
  3091. ka[nameR] = v0tmp2
  3092. }
  3093. arrcount2++
  3094. }
  3095. }
  3096. }
  3097. }
  3098. return arrcount1, arrcount2, ka
  3099. }
  3100. //初始化lineMapArr,lineMap
  3101. func initLineMapLineMapArr(table *Table) (lineMapArr map[string]*SortMap, lineMap map[string]*SortMap) {
  3102. lineMapArr = make(map[string]*SortMap)
  3103. lineMap = make(map[string]*SortMap)
  3104. for _, key := range table.SortKV.Keys { //遍历table.SortKV.Keys而不是直接遍历table.SortKV.Map是为了得到table头的顺序
  3105. val := table.SortKV.Map[key]
  3106. key = regReplAllSpace.ReplaceAllString(key, "")
  3107. key = strings.Replace(key, "", "", -1) //处理一个特殊的采购量 经上层处理空格后未处理掉
  3108. if realTypeVal, ok := val.([]string); ok { //val为数组 {"数量":["1","2","3"]}
  3109. /*
  3110. {
  3111. "商品":["",""],
  3112. "商品_"["",""],
  3113. }
  3114. */
  3115. valArr, allempty := filterVal(realTypeVal...) //过滤数据
  3116. if allempty {
  3117. continue
  3118. }
  3119. realTypeVal = valArr
  3120. line := underline.FindString(key)
  3121. lineValMap1 := lineMapArr[line]
  3122. // i := 1
  3123. // L:
  3124. // for { //去除数组空数据
  3125. // last := realTypeVal[len(realTypeVal)-i]
  3126. // if last == "" {
  3127. // i++
  3128. // if i > len(realTypeVal) {
  3129. // break
  3130. // }
  3131. // goto L
  3132. // } else {
  3133. // break
  3134. // }
  3135. // }
  3136. // dislodgeNull := realTypeVal[:(len(realTypeVal) - i + 1)] //去除数组中空数据
  3137. if len(realTypeVal) > 0 {
  3138. if lineValMap1 == nil {
  3139. tmp := NewSortMap()
  3140. tmp.AddKey(key, realTypeVal)
  3141. lineMapArr[line] = tmp
  3142. } else {
  3143. lineValMap1.AddKey(key, realTypeVal)
  3144. }
  3145. }
  3146. //qutil.Debug("lineMapArr---", lineMapArr[line].Keys, lineMapArr[line].Map)
  3147. } else if realTypeVal, b := val.(string); b { //val为字符串 {"数量":"1"}
  3148. /*
  3149. {
  3150. "商品:"",名称:"",
  3151. "商品_:"",名称_:"",
  3152. "商品__:"",名称__:"",
  3153. }
  3154. */
  3155. valArr, allempty := filterVal(realTypeVal) //过滤数据
  3156. if allempty {
  3157. continue
  3158. }
  3159. realTypeVal = valArr[0]
  3160. line := underline.FindString(key)
  3161. lineValMap2 := lineMap[line]
  3162. if lineValMap2 == nil {
  3163. tmp := NewSortMap()
  3164. tmp.AddKey(key, realTypeVal)
  3165. lineMap[line] = tmp
  3166. } else {
  3167. lineValMap2.AddKey(key, realTypeVal)
  3168. }
  3169. //qutil.Debug("lineMap---", lineMap[line].Keys, lineMap[line].Map)
  3170. } else {
  3171. // "_id" : ObjectId("5c2c3802a5cb26b9b78646c4")5c2b0551a5cb26b9b7cb05db否5c2a42e6a5cb26b9b763ba5a采购人:一、采购人5c2b06f5a5cb26b9b7cc4409
  3172. //成交供应商排名 [map[entname:昆明合优科技有限公司 sortstr:第一中标候选人 sort:1] map[sort:2 entname:昆明厚起科技有限公司 sortstr:第二中标候选人] map[entname:云南远安科技发展有限公司 sortstr:第三中标候选人 sort:3]]
  3173. //qutil.Debug("err data:", key, val)
  3174. }
  3175. }
  3176. return lineMapArr, lineMap
  3177. }
  3178. func dealArrData(maxNum int, ka map[string][]string) []map[string]string {
  3179. for k2, v2 := range ka {
  3180. //处理数组长度不相等,使长度一致
  3181. if len(v2) > maxNum {
  3182. ka[k2] = v2[:maxNum]
  3183. }
  3184. }
  3185. finalData := assembleData(ka, 1)
  3186. if len(finalData) > 0 {
  3187. return finalData
  3188. }
  3189. return nil
  3190. }
  3191. func dealStrData(kv map[string]string) []map[string]string {
  3192. finalData := []map[string]string{}
  3193. if len(kv) > 0 {
  3194. finalData = assembleData(kv, 2)
  3195. }
  3196. return finalData
  3197. }
  3198. //组装数据,每一行的数据为一数据集合
  3199. func assembleData(m interface{}, n int) []map[string]string {
  3200. defer qutil.Catch()
  3201. /*
  3202. {
  3203. "itemname":["计算机","打印机","机柜"],
  3204. "number" :["1","12","4"]
  3205. }
  3206. */
  3207. datas := []map[string]string{}
  3208. if n == 1 { //数组数据
  3209. realTypeM := m.(map[string][]string)
  3210. //根据数组数据的顺序 将多个数组中索引相同的数据拼装成一个map,并将这多个map放入一个arr
  3211. /*
  3212. arr1 ["a1","b1","c1"]
  3213. arr2 ["a2","b2","c2"]
  3214. [
  3215. {"a1","a2"},
  3216. {"b1","b2"},
  3217. {"c1","c2"}
  3218. ]
  3219. */
  3220. //start
  3221. for k3, v3 := range realTypeM {
  3222. for _, val := range v3 {
  3223. data := make(map[string]string)
  3224. data[k3] = val
  3225. datas = append(datas, data)
  3226. }
  3227. break
  3228. }
  3229. for i, data := range datas {
  3230. for k4, v4 := range realTypeM {
  3231. if i < len(v4) { //数组数据长度不一致
  3232. if v4[i] != " " {
  3233. data[k4] = v4[i]
  3234. } else {
  3235. delete(data, k4)
  3236. }
  3237. } else {
  3238. fmt.Println("err table")
  3239. }
  3240. }
  3241. datas[i] = data
  3242. }
  3243. //end
  3244. for _, fdv := range datas { //清除空数据和只含特殊符号的数据
  3245. for fmk, fmv := range fdv {
  3246. if tabletdclear.ReplaceAllString(fmv, "") == "" {
  3247. delete(fdv, fmk)
  3248. }
  3249. }
  3250. }
  3251. } else { //字符串数据
  3252. realTypeM := m.(map[string]string)
  3253. datas = append(datas, realTypeM)
  3254. }
  3255. return datas
  3256. }
  3257. ////组装数据,每一行的数据为一数据集合
  3258. //func assembleData(m interface{}, n int) []map[string]string {
  3259. // defer qutil.Catch()
  3260. // /*
  3261. // {
  3262. // "itemname":["计算机","打印机","机柜"],
  3263. // "number" :["1","12","4"]
  3264. // }
  3265. // */
  3266. // datas := []map[string]string{}
  3267. // switch reflect.TypeOf(m).String() {
  3268. // case "map[string][]string": //数组数据
  3269. // realTypeM := m.(map[string][]string)
  3270. // //根据数组数据的顺序 将多个数组中索引相同的数据拼装成一个map,并将这多个map放入一个arr
  3271. // /*
  3272. // arr1 ["a1","b1","c1"]
  3273. // arr2 ["a2","b2","c2"]
  3274. // [
  3275. // {"a1","a2"},
  3276. // {"b1","b2"},
  3277. // {"c1","c2"}
  3278. // ]
  3279. // */
  3280. // //start
  3281. // for k3, v3 := range realTypeM {
  3282. // for _, val := range v3 {
  3283. // data := make(map[string]string)
  3284. // data[k3] = val
  3285. // datas = append(datas, data)
  3286. // }
  3287. // break
  3288. // }
  3289. // for i, data := range datas {
  3290. // for k4, v4 := range realTypeM {
  3291. // if i < len(v4) { //数组数据长度不一致
  3292. // if v4[i] != " " {
  3293. // data[k4] = v4[i]
  3294. // } else {
  3295. // delete(data, k4)
  3296. // //continue
  3297. // }
  3298. // } else {
  3299. // fmt.Println("err table")
  3300. // //continue
  3301. // }
  3302. // }
  3303. // datas[i] = data
  3304. // }
  3305. // //end
  3306. // // for _, fdv := range datas { //清除空数据和只含特殊符号的数据
  3307. // // for fmk, fmv := range fdv {
  3308. // // if tabletdclear.ReplaceAllString(fmv, "") == "" {
  3309. // // delete(fdv, fmk)
  3310. // // }
  3311. // // }
  3312. // // }
  3313. // case "map[string]string": //字符串数据
  3314. // realTypeM := m.(map[string]string)
  3315. // datas = append(datas, realTypeM)
  3316. // default:
  3317. // }
  3318. // return datas
  3319. //}
  3320. func convert(key, r string) bool {
  3321. defer qutil.Catch()
  3322. flag := false
  3323. key = tabletitleclear.ReplaceAllString(key, "")
  3324. reg, err := regexp.Compile(r)
  3325. if err != nil {
  3326. fmt.Println("reg err:", err)
  3327. return false
  3328. }
  3329. flag = reg.MatchString(key)
  3330. return flag
  3331. }
  3332. func hasKey(table *Table, n int) {
  3333. defer qutil.Catch()
  3334. if table.TableResult.HasKey == 1 {
  3335. return
  3336. }
  3337. if n >= 1 {
  3338. table.TableResult.HasKey = 1
  3339. }
  3340. }
  3341. func hasGoods(table *Table, data ...string) {
  3342. defer qutil.Catch()
  3343. goodsArr := make([]string, len(data))
  3344. //fmt.Println("table.TableResult.HasGoods=====", table.TableResult.HasGoods)
  3345. if table.TableResult.HasGoods == 1 {
  3346. return
  3347. }
  3348. for i, d := range data {
  3349. if d != "" {
  3350. goods := u.GoodsGet.CheckSensitiveWord(d)
  3351. //fmt.Println("goods======", goods)
  3352. goodsArr[i] = goods
  3353. if len(goods) > 0 {
  3354. table.TableResult.HasGoods = 1
  3355. break
  3356. }
  3357. }
  3358. }
  3359. }
  3360. //func hasBrand(table *Table, data ...string) {
  3361. // defer qutil.Catch()
  3362. // if table.TableResult.HasBrand == 1 {
  3363. // return
  3364. // }
  3365. // for i, d := range data {
  3366. // if d != "" {
  3367. // brand := u.BrandGet.CheckSensitiveWord(d)
  3368. // qutil.Debug(d, brand)
  3369. // if brand != "" {
  3370. // table.TableResult.HasBrand = 1
  3371. // break
  3372. // }
  3373. // }
  3374. // }
  3375. //}
  3376. func hasBrand(table *Table, data ...string) ([]string, bool) {
  3377. defer qutil.Catch()
  3378. //fmt.Println("table.TableResult.HasBrand---------", table.TableResult.HasBrand)
  3379. brandArr := make([]string, len(data))
  3380. // if table.TableResult.HasBrand == 1 {
  3381. // return brandArr, 1
  3382. // }
  3383. allNull := true
  3384. for i, d := range data {
  3385. //if d != "" {
  3386. brand := u.BrandGet.CheckSensitiveWord(d)
  3387. if brand != "" {
  3388. allNull = false
  3389. }
  3390. //fmt.Println("brand======", brand)
  3391. brandArr[i] = brand
  3392. if len(brand) > 0 {
  3393. table.TableResult.HasBrand = 1
  3394. }
  3395. //}
  3396. }
  3397. return brandArr, allNull
  3398. }
  3399. //过滤td值
  3400. func filterVal(val ...string) ([]string, bool) {
  3401. defer qutil.Catch()
  3402. n := 0 //记录被过滤的个数
  3403. for i, v := range val {
  3404. afterFilter := tabletdclear.ReplaceAllString(v, "")
  3405. afterFilter = NullVal.ReplaceAllString(afterFilter, "")
  3406. if afterFilter == "" {
  3407. n++
  3408. }
  3409. val[i] = afterFilter
  3410. }
  3411. allempty := false
  3412. if n == len(val) { //所有都被过滤掉
  3413. allempty = true
  3414. }
  3415. return val, allempty
  3416. }
  3417. //过滤itemname全是数字
  3418. func filterItem(itemval ...string) []string {
  3419. defer qutil.Catch()
  3420. result := []string{}
  3421. for _, v := range itemval {
  3422. afterFilter := numclear.ReplaceAllString(v, "")
  3423. if afterFilter != "" {
  3424. result = append(result, v)
  3425. } else {
  3426. result = append(result, afterFilter)
  3427. }
  3428. }
  3429. return result
  3430. }
  3431. //处理价格
  3432. func dealPrice(key string, val ...string) []string {
  3433. defer qutil.Catch()
  3434. iswan := strings.Contains(key, "万") //表格title中带有万
  3435. result := []string{}
  3436. for _, v := range val { //1.00万元 1元
  3437. tmparr := strings.Split(v, ".")
  3438. tmparr[0] = moneyNum.ReplaceAllString(tmparr[0], "")
  3439. if iswan {
  3440. result = append(result, tmparr[0]+"0000")
  3441. } else {
  3442. if strings.Contains(v, "万") { //价格中带有万
  3443. result = append(result, tmparr[0]+"0000")
  3444. } else {
  3445. result = append(result, tmparr[0])
  3446. }
  3447. }
  3448. }
  3449. return result
  3450. }
  3451. //处理number
  3452. func dealNumber(val ...string) ([]string, []string) {
  3453. defer qutil.Catch()
  3454. unitnameArr := []string{}
  3455. result := []string{}
  3456. for _, v := range val { //1个 1.00个
  3457. n := numclear.FindString(v)
  3458. unitname := numclear.ReplaceAllString(v, "") //匹配个数后的单位
  3459. unitnameArr = append(unitnameArr, unitname)
  3460. //val[i] = strings.Split(n, ".")[0]
  3461. result = append(result, strings.Split(n, ".")[0])
  3462. }
  3463. return result, unitnameArr
  3464. }
  3465. func (tn *Table) analyProNameAndItemNumber() {
  3466. }