123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086 |
- package pretreated
- import (
- "fmt"
- "jy/clear"
- u "jy/util"
- qutil "qfw/util"
- "regexp"
- "strings"
- "unicode/utf8"
- "github.com/PuerkitoBio/goquery"
- )
- /**
- 全局变量,主要是一堆判断正则
- **/
- var (
- //key 的日期单位
- dateReg *regexp.Regexp = regexp.MustCompile(`[年|月|日|天]`)
- //清理品目中数字
- numclear = regexp.MustCompile("^[\\d一二三四五六七八九十.]+")
- num1 = regexp.MustCompile("(\\d)")
- //清理表格title中的不需要的内容
- tabletitleclear = regexp.MustCompile("[\\s\u3000\u2003\u00a0\\n\u001c、.,.。_/((人民币万元件个公斤户))]")
- tabletitleclear2 = regexp.MustCompile("[\\s\u3000\u2003\u00a0\\n\u001c、,。_??;;~\\-#\\\\()(){}【】\\[\\]<>《》{}〔〕]*")
- //清理表格中是key中包含的空格或数字等
- tablekeyclear = regexp.MustCompile("[\\s\u3000\u2003\u00a0\\n、.,.。_/]+|^[\\d一二三四五六七八九十]+[、.]*|[((【\\[].*?[))】\\]]")
- //清理表格td中的符号
- tabletdclear = regexp.MustCompile("[\\s\u3000\u2003\u00a0\\n\u001c、,。_??;;~\\-#\\\\()(){}【】\\[\\]<>《》{}〔〕¥$]*")
- //判断key是金额,对万元的处理
- moneyreg = regexp.MustCompile("(预算|费|价|额|规模|投资)")
- //key不需要清理-例如折扣 费率
- no_clear_key_reg = regexp.MustCompile(`[((](费率|年|月|日|天|日历天|历天)[))]`)
- //根据表格的内容判断是不是表头,如果含有金额则不是表头
- MoneyReg = regexp.MustCompile("^[\\s ::0-9.万元()()人民币¥$]+$")
- GSReg = regexp.MustCompile(".*公司.*")
- //判断分包时
- moneyNum = regexp.MustCompile("[元整¥万]")
- //对隐藏表格的判断
- display = regexp.MustCompile("(?i).*?display\\s?[:]\\s?none.*")
- //---------------
- //求是分包的概率
- //根据表格的标签对分包进行打分
- TableMultiPackageReg_4 = regexp.MustCompile("(标段|分包|包段|划分|子包|标包|合同段)")
- TableMultiPackageReg_2 = regexp.MustCompile("(概况|范围|情况|内容|详细|结果|信息)")
- //在判断分包打分前过虑表格key
- FilterKey_2 = regexp.MustCompile("招标|投标|项目")
- //根据表格的key进行分包打分
- FindKey_2 = regexp.MustCompile("([分子][包标](号)?|标[号项段包](划分)?|包件?[号段名数]|包[组件])")
- FindKey_3 = regexp.MustCompile("(标段编号|标包)")
- //对值进行分包判断
- FindVal_1 = regexp.MustCompile("[第]?([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)((子|合同|分|施工|监理)?(标段?|包|合同段|标包))|((子|合同|分|施工|监理)?(标|包)(段|号)?)[ \u3000\u2003\u00a0]*([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)")
- FindVal2_1 = regexp.MustCompile("([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+)|^(设计|施工|监理|验收)[分子]?[标包]?[段号]?$")
- //判断分包前排除 包件号?
- excludeKey = regexp.MustCompile("(标识|数量|分包个数|标段代码|涉及包号|分包数量|项目标号|规格|型号|招标范围|业绩|废标|标段选择要求)|(^编号$)|([^包段标]编号)") //编号|划分
- excludeKey2 = regexp.MustCompile("包/[0-9]{0,4}[箱纸张]")
- excludeKey3 = regexp.MustCompile("(分包个数|每包[0-9]*元|标线|国标|享受一包服务)")
- //-------------
- cut = u.NewCut()
- //清理表格标签正则
- ClearTagReg = regexp.MustCompile("<[^>]*?>|[\\s\\n\\r]*$")
- //查找表格标签正则
- ttagreg = regexp.MustCompile("(?s)([^\\n::。,;\\s\u3000\u2003\u00a0]{2,30})[::]?[^::。;!\\n]{0,35}[\\s\\n]*$")
- //判断表格是表头的概率
- checkval = float32(0.6)
- //tdval_reg = regexp.MustCompile(`([\p{Han}][\p{Han}\s、()\\(\\)]{1,9})[::]([^::\\n。]{5,60})(?:[;;,,.。\\n\\t\\s])?`)
- //空格替换
- repSpace = regexp.MustCompile("[\\s\u3000\u2003\u00a0::]+|\\\\t+")
- //对表格kv的处理
- //对不能标准化的key做批识
- filter_tag_zb = regexp.MustCompile("(中标|成交|投标)[\\p{Han}]{0,6}(情况|结果|信息|明细)?")
- //中标金额
- //包含以下字眼做标准化处理
- filter_zbje_k = regexp.MustCompile("(中标|成交|总|拦标|合同|供[应货]商|报)[\\p{Han}、]{0,6}(价|额|[大小]写|[万亿]?元).{0,4}$")
- //简单判断金额
- filter_zbje_jd = regexp.MustCompile("^[^(售|保证)]{0,4}(价|额).{0,4}$")
- //预算金额
- filter_ysje_jd = regexp.MustCompile("(预算|预控价|项目概.|项目信息)")
- //且排队以下字眼的key
- filter_zbje_kn = regexp.MustCompile("得分|打分|时间|业绩|须知|分|电话|要求|需求数量|发布规模$|第[2二3三4四5五]|地址|询价保证金|行号")
- //且值包含以下字眼
- filter_zbje_v = regexp.MustCompile("[¥$$0-9一二三四五六七八九十,,〇零点..壹贰叁肆伍陆柒捌玖拾百佰千仟万亿億元圆角分整正()::大小写]{2,16}")
- //中标单位的处理
- //包含以下字眼的Key标准化
- filter_zbdw_ky = regexp.MustCompile("(中标|成交|拦标|合同|选中|投标|拟|预|最终)[\\p{Han}、]{0,6}(供[应货]商|企业|单位|人|机构)(名称)?.{0,4}$")
- //识别中标单位相关信息
- filter_zbdw_info = regexp.MustCompile("(中标|成交|中选|供(货|应))[^候选]{0,}")
- //简单判断
- filter_zbdw_jd = regexp.MustCompile("(投标|成交|中标|合同)(供应商|单位|人|名称).{0,4}$")
- //且不包含以下字眼
- filter_zbdw_kn = regexp.MustCompile("第[2二3三4四5五]|得分|地址|询价保证金") //且值包含以下字眼
- //且值包含以下字眼
- filter_zbdw_v = regexp.MustCompile("(公司|集团|研究院|设计院|局|厂|部|站|城|店|市|所|处)$|([^购]中心|办公|用品)")
- //且值包含以下字眼
- filter_zbdw_v2 = regexp.MustCompile("(公司|集团|研究院|设计院|局|厂|部|站|城|店|市|所|处)$")
- //Tg = map[string]interface{}{}
- //一些表格没有表头,是空的,对值是排序的做处理对应 NullTxBid
- NullTdReg = regexp.MustCompile("(首选|第[一二三四五1-5])(中标|成交)?(名(称)?|(候选|排序)?(人|单位|供应商))")
- NullTxtBid = "成交供应商排名"
- projectnameReg = regexp.MustCompile("((公开)?招标)*[((第]*[一二三四五六七八九十a-zA-Z0-9]+(标段|包|标|段)[))]*$")
- MhSpilt = regexp.MustCompile("[::]")
- //识别采购单位联系人、联系电话、代理机构联系人、联系电话 -- 名称有异常
- ContactInfoVagueReg = regexp.MustCompile("邮政编码|邮编|名称|(征求意见|报名审核购买)?((联系人?(及|和)?|办公|单位)?(((联系)?(电话|方式|号码)([//及]传真|及手机)?|手机)(号码)?|邮箱(地址)?|(详细)?(地(址|点)))|(联系|收料)(人(姓名)?|方式)|传真|电子邮件|(主要负责|项目(负责|联系)|经办)人)|采购方代表")
- ContactInfoMustReg = regexp.MustCompile("^(" + ContactInfoVagueReg.String() + ")$")
- ContactType = map[string]*regexp.Regexp{
- "采购单位": regexp.MustCompile("(采购(项目.{2}|服务)?|比选|询价|招标(服务)?|甲|建设|委托|发包|业主|使用|谈判|本招标项目经办|征求意见联系|项目实施)(人|单位|部门|机构|机关|(执行)?方$)|(项目|建(库|设))单位|招标人信息|采购中心(地址)?|业主|收料人|采购部"),
- "代理机构": regexp.MustCompile("(代理|受托|集中采购).{0,2}(人|方|单位|公司|机构)|招标机构|采购代理"),
- "中标单位": regexp.MustCompile("^((拟(定)?|预|最终|唯一)?(中标|成交|中选|供(货|应))((成交))?)[^候选]{0,2}(人|方|单位|公司|(服务|供应)?商|企业)"),
- "监督部门": regexp.MustCompile("投诉受理部门"),
- }
- ContactBuyerPersonFilterReg = regexp.MustCompile("(管理局)$")
- MultipleValueSplitReg = regexp.MustCompile("[,,、\\s\u3000\u2003\u00a0]")
- BuyerContacts = []string{"采购单位联系人", "采购单位联系电话", "采购单位联系地址"}
- FilterSerial = regexp.MustCompile(".+[、..::,]")
- underline = regexp.MustCompile("_+$")
- iswinnertabletag = regexp.MustCompile("(中标|候选人|成交|结果|磋商情况)")
- nswinnertabletag = regexp.MustCompile("评得分估|标的信息|班子成员")
- jsonReg = regexp.MustCompile(`\{.+:[^}]*\} `) // \{".*\":\".+\"}
- regHz = regexp.MustCompile("[\u4e00-\u9fa5]")
- winnerOrderAndBidResult = regexp.MustCompile("((中标)?候选人|(中标|评标)结果)")
- WinnerOrderStr = regexp.MustCompile(`(集团|公司|学校|中心|家具城|门诊|[大中小]+学|部|院|局|厂|店|所|队|社|室|厅|段|会|场|行)$`)
- DoubtReg = regexp.MustCompile("(我中心|有(疑问|质疑|异议|意见)|(书面)?提出|不再受理|投诉|质疑|书面形式|监督|公示期(限)?)")
- )
- //在解析时,判断表格元素是否隐藏
- func IsHide(g *goquery.Selection) (b bool) {
- style, exists := g.Attr("style")
- if exists {
- b = display.MatchString(style)
- }
- return
- }
- //59.992664,33.495715,20.001306
- var clearnum *regexp.Regexp = regexp.MustCompile("(([0-9.]{1,6}[,,]+){4,}|(\\d{6}[,,]\\d{2}.){2,})")
- //对表格的key进行标准化处理,多个k相同时,出现覆盖问题
- //待扩展,暂不支持正则标签库 清理key
- func CommonDataAnaly(k, tabletag, tabledesc string, v interface{}, isSite bool, codeSite string) (kvTags map[string][]*u.Tag, returntag string) {
- kvTags = map[string][]*u.Tag{}
- v1 := ""
- if sv, sok := v.(string); sok { //取KV
- v1 = sv
- } else if sv, sok := v.([]string); sok { //是数组先默认取第一个
- if len(sv) >= 1 {
- v1 = sv[0]
- }
- }
- //对值单位的处理 (预算|费|价|额|规模|投资)
- if moneyreg.MatchString(k) {
- v1 += GetMoneyUnit(k, v1)
- }
- //先清理key
- //u.Debug(1, k, v1)
- //指定-key不清理 拦标价(费率或单价等)
- k1:=""
- if !no_clear_key_reg.MatchString(k) {
- k1 = ClearKey(k, 2)
- }
- //u.Debug(2, k)
- //取标准key
- if tabletag == "中标情况" {
- if k1=="价格" {
- k1="中标金额"
- }
- }
- res := u.GetTags(k1, isSite, codeSite)
- if len(res) == 0 && k1 != k {
- res = u.GetTags(k, isSite, codeSite)
- k1 = k
- }
- //log.Println(k, res)
- // if len(res) == 0 {
- // go u.AddtoNoMatchMap(tk)
- // }
- //当取到标准化值时,放入数组
- if len(res) > 0 {
- for _, t1 := range res {
- //降低冒号值的权重
- if MhSpilt.MatchString(v1) {
- t1.Weight -= 50
- }
- if winnerOrderAndBidResult.MatchString(tabletag) && t1.Value == "采购单位联系人" { //处理table中项目负责人
- kvTags[k] = append(kvTags[k], &u.Tag{Key: k, Value: v1, IsInvalid: true})
- } else if regexp.MustCompile("(中标候选人|名单及其排序|排序)").MatchString(tabletag) && t1.Value == "采购单位" {
- kvTags[t1.Value] = append(kvTags[t1.Value], &u.Tag{Key: k1, Value: v1, Weight: t1.Weight - 150})
- } else {
- kvTags[t1.Value] = append(kvTags[t1.Value], &u.Tag{Key: k1, Value: v1, Weight: t1.Weight})
- }
- }
- res[0].IsInvalid = true
- //k1 = res[0].Value
- } /*else {
- kvTags[k] = append(kvTags[k], &u.Tag{Key: k, Value: v1, IsInvalid: true})
- //没有取到标准化key时,对中标金额和中标单位的逻辑处理
- if filter_zbje_k.MatchString(k) && !filter_zbje_kn.MatchString(k) && filter_zbje_v.MatchString(v1) && utf8.RuneCountInString(v1) < 20 {
- if tabletag == "" {
- returntag = "中标情况"
- }
- kvTags["中标金额"] = append(kvTags["中标金额"], &u.Tag{Key: k, Value: v1, Weight: -100, IsInvalid: true})
- } else if filter_zbdw_ky.MatchString(k) && !filter_zbdw_kn.MatchString(k) &&
- filter_zbdw_v.MatchString(v1) {
- kvTags["中标单位"] = append(kvTags["中标单位"], &u.Tag{Key: k, Value: v1, Weight: -100, IsInvalid: true})
- if tabletag == "" {
- returntag = "中标情况"
- }
- } else if !filter_zbje_jd.MatchString(tabletag) && !filter_zbje_jd.MatchString(k) && utf8.RuneCountInString(v1) < 13 {
- //对上一步没有取到标准化key的进一步处理
- if filter_tag_zb.MatchString(tabletag) || filter_tag_zb.MatchString(tabledesc) {
- //u.Debug(v1, k, "-----", filter_zbdw_jd.MatchString(k), filter_zbdw_v.MatchString(v1))
- if filter_zbje_jd.MatchString(k) && !filter_zbje_kn.MatchString(k) && filter_zbje_v.MatchString(v1) {
- if filter_ysje_jd.MatchString(k) {
- kvTags["预算金额"] = append(kvTags["预算金额"], &u.Tag{Key: k, Value: v1, Weight: -100})
- } else if !filter_zbdw_kn.MatchString(k) {
- kvTags["中标金额"] = append(kvTags["中标金额"], &u.Tag{Key: k, Value: v1, Weight: -100})
- }
- }
- }
- }
- }*/
- return
- }
- var glRex *regexp.Regexp = regexp.MustCompile("(成交|中标|候选|排名|名次|供应商排序|中标候选人|名单及其排序|排序)")
- var djReg *regexp.Regexp = regexp.MustCompile("^单价")
- //对解析后的表格的kv进行过滤
- func (table *Table) KVFilter(isSite bool, codeSite string) {
- //1.标准化值查找
- //2.对数组的处理
- //3.对分包的处理
- //4.对KV的处理
- //判断表格是否有用,调用abandontable正则数组进行判断
- //遍历每一行
- table.analyTdKV() //1.遍历每行每列td的sortkv添加到table.SorkVK中;2.td有子表格的处理
- as := NewSortMap()
- //遍历table.sortkv,进行过滤处理,并放入标准化KV中,如果值是数组跳到下一步处理
- for _, k := range table.SortKV.Keys {
- //遍历所有key sort.kv
- //表格描述处理,对成交结果的处理
- if k=="第一询价结果候选人" {
- //fmt.Println("标准化key")
- }
- if glRex.MatchString(k) {
- table.Desc += "成交结果,"
- }
- if djReg.MatchString(k) {
- continue
- }
- v := table.SortKV.Map[k]
- if _, ok := v.(string); ok { //table.SortKV.Value为字符串,匹配抽取关键词table.SortKV.Key,匹配到添加k,v到table.StandKV,table.StandKVWeight
- k = pkgFilter.ReplaceAllString(k, "")
- k = regSpliteSegment.ReplaceAllString(regReplAllSpace.ReplaceAllString(k, ""), "")
- kvTags, tag := CommonDataAnaly(k, table.Tag, table.Desc, v, isSite, codeSite) //对key标准化处理,没有找到会走中标
- if tag != "" && table.Tag == "" {
- table.Tag = tag
- }
- MergeKvTags(table.StandKV, kvTags)
- } else {
- as.AddKey(k, v)
- }
- }
- //核心-候选人-相关
- //处理值是数组的kv 放入标准化kv中 standKV //处理table.SortKV.value为数组的情况
- table.sortKVArr(as, isSite, codeSite)
- //
- if len(table.WinnerOrder) > 0 || !table.BPackage {
- winnerOrder := []map[string]interface{}{}
- maxSort := 0
- //调整顺序
- for i := 0; i < 2; i++ {
- for _, v := range table.WinnerOrder {
- sortstr, _ := v["sortstr"].(string)
- if (i == 0 && sortstr == "") || (i == 1 && sortstr != "") {
- continue
- }
- sort, _ := v["sort"].(int)
- if i == 0 {
- if maxSort == 0 || sort > maxSort {
- maxSort = sort
- }
- } else {
- maxSort++
- v["sort"] = maxSort
- }
- winnerOrder = append(winnerOrder, v)
- }
- if len(winnerOrder) == len(table.WinnerOrder) {
- break
- }
- }
- table.WinnerOrder = winnerOrder
- if len(table.WinnerOrder) == 0 {
- winnerOrder = []map[string]interface{}{}
- //遍历每个td,查询中标人
- for _, tr := range table.TRs {
- for _, td := range tr.TDs {
- winnerOrder = winnerOrderEntity.Find(td.Val, true, 3, isSite, codeSite)
- if len(winnerOrder) > 0 {
- //中标候选人合并
- winnerOrderEntity.Merge(table.WinnerOrder, winnerOrder)
- }
- }
- }
- }
- if !table.BPackage { //没有table.WinnerOrder也没有分包 将td中的WinnerOrder赋值给table.WinnerOrder
- if len(winnerOrder) > 1 {
- table.WinnerOrder = winnerOrder
- }
- }
- }
- //对中标候选人进行排序
- winnerOrderEntity.Order(table.WinnerOrder)
- //该表格有一个分包,并且有中标候选人排序的情况下,把中标候选人放到包里面
- if table.BlockPackage != nil && table.BlockPackage.Keys != nil && len(table.BlockPackage.Keys) == 1 {
- if table.BlockPackage.Map != nil {
- onePkgKey := table.BlockPackage.Keys[0]
- onePkg, _ := table.BlockPackage.Map[onePkgKey].(*u.BlockPackage)
- if onePkg != nil && (onePkg.WinnerOrder != nil || len(onePkg.WinnerOrder) == 0) {
- onePkg.WinnerOrder = table.WinnerOrder
- table.BlockPackage.AddKey(onePkgKey, onePkg)
- }
- }
- }
- }
- var winMoneyReg *regexp.Regexp = regexp.MustCompile("(报价|投标价|投标总价)")
- //处理table.SortKV.value为数组的情况
- func (table *Table) sortKVArr(as *SortMap, isSite bool, codeSite string) {
- winnertag := iswinnertabletag.MatchString(table.Tag) && !nswinnertabletag.MatchString(table.Tag) //table标签
- if !winnertag {
- winnertag = iswinnertabletag.MatchString(table.TableResult.BlockTag) && !nswinnertabletag.MatchString(table.TableResult.BlockTag) //块标签
- }
- if !winnertag {
- winnertag = iswinnertabletag.MatchString(table.Desc)
- }
- if !winnertag {
- winnertag = iswinnertabletag.MatchString(table.Html)
- }
- checkKey := map[int]bool{}
- //tmpBidmout := []string{}
- //log.Println(tmpBidmout)
- for kn, k := range as.Keys { //遍历table.SortKV.value为数组的key
- v := as.Map[k]
- if vm, ok := v.([]map[string]interface{}); ok && k == NullTxtBid {
- if table.WinnerOrder == nil {
- table.WinnerOrder = []map[string]interface{}{}
- }
- table.WinnerOrder = append(table.WinnerOrder, vm...)
- } else {
- //增加候选人排序逻辑
- if (table.WinnerOrder == nil || len(table.WinnerOrder) == 0) && !checkKey[kn] {
- if vs1, ok := v.([]string); ok {
- smap := make([]map[string]interface{}, len(vs1))
- for n1, _ := range vs1 {
- smap[n1] = map[string]interface{}{}
- }
- //hadSort := false
- tmpEntname := make([]string, len(vs1))
- tmpPrice := make([]string, len(vs1))
- for kn1, k := range as.Keys[kn:] {
- v := as.Map[k]
- if ContactType["采购单位"].MatchString(k) || ContactType["代理机构"].MatchString(k) {
- kvTags, _ := CommonDataAnaly(k, table.Tag, table.Desc, v, isSite, codeSite) //对key标准化处理,没有找到会走中标
- for k := range kvTags {
- if table.StandKV[k] == nil {
- MergeKvTags(table.StandKV, kvTags)
- }
- }
- continue
- }
- //目前对数组数据的key做判断,但是某些额可以是不满足情况的
- //载明内容:[第一中标候选人 第二中标候选人] id:5d00587da5cb26b9b75e367b
- if vs, ok := v.([]string); ok && len(vs) == len(vs1) { //数组值的个数相同
- res, _, _, _, repl := CheckCommon(k, "bidorder")
- kv := ""
- if !res {
- kt := u.GetTags(k, isSite, codeSite)
- if kt.Len() > 0 {
- if kt[0].Value == "单品报价" && winnertag {
- kv = "中标金额"
- } else {
- kv = kt[0].Value
- }
- }
- }
- if !res && kv == "" { //key未验证出,验证数组的val值
- checkKey[kn+kn1] = true
- if winnertag { //如果是中标信息 在根据val数组信息解析候选人
- for vsk, vsv := range vs {
- if NullTdReg.MatchString(vsv) { //数据先验证val是否有排序
- //hadSort = true
- smap[vsk]["sortstr"] = vsv
- smap[vsk]["sort"] = GetBidSort(vsv, vsk+1)
- } else if findCandidate2.MatchString(vsv) && tmpEntname[vsk] == "" { //数据验证val是否是候选人
- entname, _ := winnerOrderEntity.clear("中标单位", vsv).(string)
- if entname != "" {
- tmpEntname[vsk] = entname
- }
- } else if winMoneyReg.MatchString(k) && len(tmpPrice[vsk]) == 0 {
- kv = "中标金额"
- }else { //验证val时如果数组中的第一条数据既不满足sort或者entname 判定此数组数据错误
- break
- }
- }
- }
- }
- if res || kv != "" { //连续往下找几个key
- checkKey[kn+kn1] = true
- SORT:
- if repl == "sort" {
- //hadSort = true
- for vsk, vsv := range vs {
- smap[vsk]["sortstr"] = vsv
- smap[vsk]["sort"] = GetBidSort(vsv, vsk+1)
- if findCandidate2.MatchString(vsv) && kv == "中标单位" && tmpEntname[vsk] == "" { //数据验证val是否是候选人
- entname, _ := winnerOrderEntity.clear("中标单位", vsv).(string)
- if entname != "" {
- tmpEntname[vsk] = entname
- }
- }
- }
- } else if repl == "entname" || kv == "中标单位" {
- for vsk, vsv := range vs {
- if winnerReg6.MatchString(vsv) { //k:中标候选人 v:["第一名","第二名"]
- repl = "sort"
- goto SORT
- }
- //if entname, _ := smap[vsk]["entname"].(string); entname != "" || len([]rune(vsv)) < 3 {
- // break
- //}
- //entname, _ := winnerOrderEntity.clear("中标单位", vsv).(string)
- //if entname != "" {
- // smap[vsk]["entname"] = entname
- if tmpEntname[vsk] != "" || len([]rune(vsv)) < 4 { //排除 单位:["台","个","套"]
- break
- }
- entname, _ := winnerOrderEntity.clear("中标单位", vsv).(string)
- if entname != "" {
- tmpEntname[vsk] = entname
- }
- }
- } else if kv == "中标金额" {
- for vsk, vsv := range vs {
- //过滤price 2348273.432元(万元)-->2348273.432
- //tmp1, _ := smap[vsk]["price"].(string)
- tmp1 := tmpPrice[vsk]
- p1num := numberReg2.FindString(tmp1)
- p2num := numberReg2.FindString(vsv)
- p1 := qutil.Float64All(p1num)
- p2 := qutil.Float64All(p2num)
- if p2 > p1 {
- //smap[vsk]["price"] = winnerOrderEntity.clear("中标金额", vsv+GetMoneyUnit(k, vsv))
- price := winnerOrderEntity.clear("中标金额", vsv+GetMoneyUnit(k, vsv))
- if pricestr, _ := price.(string); len(pricestr) < 30 && len(pricestr) > 0 {
- tmpPrice[vsk] = pricestr
- }
- }
- }
- } else if kv == "预算" {
- if strings.Contains(k, "万元") {
- for vsk, vsv := range vs {
- if !strings.Contains(vsv, "万元") {
- vs[vsk] = vsv + "万元"
- }
- }
- }
- }
- }
- } else {
- //break
- }
- }
- newSmap := []map[string]interface{}{}
- //qutil.Debug("smap=======", smap)
- //qutil.Debug("tmpEntname--", len(tmpEntname), tmpEntname)
- //qutil.Debug("tmpPrice--", len(tmpPrice), tmpPrice)
- for n, smap_v := range smap {
- //if hadSort { //有排序,再添加entname和price
- if len(tmpEntname) > 0 && n < len(tmpEntname) && tmpEntname[n] != "" {
- smap_v["entname"] = tmpEntname[n]
- if len(tmpPrice) > 0 && n < len(tmpPrice) && tmpPrice[n] != "" {
- smap_v["price"] = tmpPrice[n]
- }
- }
- //} else if len(tmpEntname) > 0 {
- //fmt.Println("table winnerorder only has entname", tmpEntname)
- //}
- //qutil.Debug("len-smap_v--", len(smap_v))
- if len(smap_v) > 2 { //只有排序信息 sort和sortstr
- newSmap = append(newSmap, smap_v)
- }
- }
- if len(newSmap) > 0 {
- table.WinnerOrder = newSmap
- }
- }
- } else if vsss, ok := v.([]string); ok {
- if (len(table.WinnerOrder) > 0 && table.WinnerOrder[0]["price"] == nil && len(vsss) == len(table.WinnerOrder)) ||
- (len(table.WinnerOrder) > 0 && strings.Contains(k,"总报价") && len(vsss) == len(table.WinnerOrder)){
- kv := ""
- if winMoneyReg.MatchString(k) {
- kv = "中标金额"
- } else {
- kt := u.GetTags(k, isSite, codeSite)
- if kt.Len() > 0 {
- if kt[0].Value == "单品报价" && winnertag {
- kv = "中标金额"
- } else {
- kv = kt[0].Value
- }
- }
- }
- if kv == "中标金额" {
- for i, vx := range vsss {
- p1num := numberReg2.FindString(vx)
- if strings.Contains(p1num, ",") && strings.Contains(p1num, ".") {
- p1num = strings.ReplaceAll(p1num, ",", "")
- }
- p1 := qutil.Float64All(p1num)
- if p1 > 0 {
- //smap[vsk]["price"] = winnerOrderEntity.clear("中标金额", vsv+GetMoneyUnit(k, vsv))
- price := winnerOrderEntity.clear(kv, vx+GetMoneyUnit(k, vx))
- if pricestr, _ := price.(string); len(pricestr) < 30 && len(pricestr) > 0 && !clearnum.MatchString(pricestr) {
- table.WinnerOrder[i]["price"] = pricestr
- }
- }
- }
- }
- } else if table.StandKV[k] == nil {
- kvTags, _ := CommonDataAnaly(k, table.Tag, table.Desc, v, isSite, codeSite) //对key标准化处理,没有找到会走中标
- MergeKvTags(table.StandKV, kvTags)
- }
- }
- }
- }
- //特殊处理--组合候选人
- }
- //1.遍历每行每列td的sortkv添加到table.SorkVK中;2.td有子表格的处理
- func (table *Table) analyTdKV() {
- //遍历每一行
- for _, tr := range table.TRs {
- for _, td := range tr.TDs {
- //fmt.Println(td.BH, td.MustBH, td.Val, td.SortKV.Map)
- bc := false
- if !td.BH {
- //表头是否是无用内容
- if td.HeadTd != nil {
- bc, _, _, _, _ = CheckCommon(td.HeadTd.Val, "abandontable")
- }
- }
- if !bc {
- //td元素有内嵌kv,遍历放入table的Kv中
- if len(td.SortKV.Keys) > 0 {
- for _, k3 := range td.SortKV.Keys {
- _val := td.SortKV.Map[k3]
- //thisFlag := false
- if td.HeadTd != nil && len([]rune(k3)) < 4 {
- k3 = td.HeadTd.Val + k3
- }
- if table.SortKV.Map[k3] == nil && _val != nil && _val != "" {
- //u.Debug(k3, _val)
- //if !thisFlag || (thisFlag && table.SortKV.Map[k3] == nil) {
- table.SortKV.AddKey(k3, _val)
- }
- }
- }
- }
- //td有子表格的处理
- //u.Debug(td.BH, td.Val, td.SonTableResult)
- if td.SonTableResult != nil {
- //u.Debug(td.SonTableResult.SortKV.Map, "-------", td.SonTableResult.Tabs)
- for k3, v3 := range td.SonTableResult.KvTags {
- table.StandKV[k3] = append(table.StandKV[k3], v3...)
- }
- //中标候选人排序
- if table.WinnerOrder == nil || len(table.WinnerOrder) == 0 {
- table.WinnerOrder = td.SonTableResult.WinnerOrder
- } else {
- winnerOrderEntity.Merge(table.WinnerOrder, td.SonTableResult.WinnerOrder)
- }
- }
- }
- }
- }
- //表格结果合并到父表格集中
- func (table *Table) MergerToTableresult() {
- //对多包表格的多包值的合并处理
- if table.BPackage {
- table.TableResult.IsMultiPackage = true
- for _, v2 := range table.BlockPackage.Keys {
- package1 := table.TableResult.PackageMap.Map[v2]
- if package1 == nil {
- table.TableResult.PackageMap.AddKey(v2, table.BlockPackage.Map[v2])
- if vvv, ok := table.BlockPackage.Map[v2].(*u.BlockPackage); ok {
- if vvv.TableKV != nil && len(vvv.TableKV.KvTags) > 0 {
- MergeKvTags(table.TableResult.KvTags, vvv.TableKV.KvTags)
- }
- }
- } else {
- bp := package1.(*u.BlockPackage)
- if bp.TableKV == nil {
- bp.TableKV = u.NewJobKv()
- }
- v1 := table.BlockPackage.Map[v2].(*u.BlockPackage)
- if v1.TableKV != nil && len(v1.TableKV.KvTags) > 0 {
- for k2, v2 := range v1.TableKV.KvTags {
- if k2 == "" {
- continue
- }
- isExists := false
- for _, v2v := range v2 {
- if v2v.Value == "" {
- continue
- }
- for _, v2vv := range bp.TableKV.KvTags[k2] {
- if v2v.Value == v2vv.Value {
- isExists = true
- break
- }
- }
- if !isExists {
- bp.TableKV.KvTags[k2] = append(bp.TableKV.KvTags[k2], v2v)
- bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
- }
- }
- }
- }
- if bp.Bidamount <= 0 && !bp.IsTrueBidamount {
- bp.Bidamount = v1.Bidamount
- bp.IsTrueBidamount = v1.IsTrueBidamount
- }
- if bp.Budget <= 0 && !bp.IsTrueBudget {
- bp.Budget = v1.Budget
- bp.IsTrueBudget = v1.IsTrueBudget
- }
- bp.Text += bp.Text
- if len(v1.WinnerOrder) > 0 && len(bp.WinnerOrder) == 0 {
- bp.WinnerOrder = v1.WinnerOrder
- }
- }
- }
- }
- //遍历标准key到tableresult.sortkv中
- for fieldKey, v := range table.StandKV {
- for _, vv := range v {
- if fieldKey=="项目周期"||fieldKey=="工期单位"||fieldKey=="工期时长" {
- dateStr := dateReg.FindString(vv.Key)
- if dateStr !="" && !strings.Contains(vv.Value,dateStr) {
- vv.Value = vv.Value+dateStr
- }
- }
- vv.Value = strings.Replace(vv.Value, "__", "", -1)
- }
- }
- MergeKvTags(table.TableResult.KvTags, table.StandKV)
- //表格的块标签
- if table.TableResult.BlockTag == "" && table.Tag != "" {
- table.TableResult.BlockTag = table.Tag
- }
- //中标候选人(多个table,现在默认取第一个table的信息,考虑需不需要多个table分析合并数据?)
- if table.TableResult.WinnerOrder == nil || len(table.TableResult.WinnerOrder) == 0 {
- table.TableResult.WinnerOrder = table.WinnerOrder
- }
- //增加brand 并列table
- if len(table.BrandData) > 0 {
- for _, v := range table.BrandData {
- if len(v) > 0 {
- table.TableResult.BrandData = append(table.TableResult.BrandData, v)
- }
- }
- }
- //抽取prince和number 并列table
- if len(table.PriceNumberData) > 0 {
- for _, v := range table.PriceNumberData {
- if len(v) > 0 {
- table.TableResult.PriceNumberData = append(table.TableResult.PriceNumberData, v)
- }
- }
- }
- if table.BlockPackage != nil && len(table.BlockPackage.Keys) == 0 {
- for _, v := range table.BlockPackage.Keys {
- if table.BlockPackage.Map[v] != nil {
- if vvv, ok := table.BlockPackage.Map[v].((*u.BlockPackage)); ok {
- if vvv.TableKV != nil && len(vvv.TableKV.KvTags) > 0 {
- for kk, vv := range vvv.TableKV.KvTags {
- if kk == "" {
- continue
- }
- if len(table.TableResult.KvTags[kk]) == 0 {
- table.TableResult.KvTags[kk] = vv
- }
- }
- }
- }
- }
- }
- }
- }
- /**
- 解析表格入口
- 返回:汇总表格对象
- **/
- func AnalyTableV2(tabs *goquery.Selection, toptype, blockTag, con string, itype int, _id interface{}, ruleBlock *u.RuleBlock, isSite bool, codeSite string) (tabres *TableResult) {
- defer qutil.Catch()
- //u.Debug(con)
- if itype == 1 {
- //修复表格
- con = RepairCon(con)
- }
- //生成tableresult对象
- tabres = NewTableResult(_id, toptype, blockTag, con, itype, ruleBlock)
- if fblbReg.MatchString(blockTag) {
- return
- }
- //可以有多个table
- //for _, table := range tabs {
- //隐藏表格跳过
- if IsHide(tabs) {
- return
- }
- tabres.GoqueryTabs = tabs
- //}
- //解析表格集
- tabres.Analy(isSite, codeSite)
- return
- }
- //开始解析表格集
- func (ts *TableResult) Analy(isSite bool, codeSite string) {
- tabs := []*Table{}
- contactFormat := &u.ContactFormat{
- IndexMap: map[int]string{},
- MatchMap: map[string]map[string]bool{},
- }
- //for _, table := range ts.GoqueryTabs {
- tn := NewTable(ts.Html, ts, ts.GoqueryTabs)
- //核心模块
- tsw := tn.Analy(contactFormat, isSite, codeSite)
- for _, tab := range tsw {
- if len(tab.TRs) > 0 {
- tabs = append(tabs, tab)
- }
- //fmt.Println("tab.SortKV.Map", tab.SortKV.Keys)
- }
- //tn.SonTables = append(tn.SonTables, tn)
- //}
- //统一合并,考虑统一多表格是多包的情况---新增 与子表格合并
- if len(tabs) > 1 {
- pns := map[string]string{}
- pnarr := []string{}
- for _, table := range tabs {
- if len(table.StandKV["项目名称"]) == 0 {
- continue
- }
- pn := table.StandKV["项目名称"][0]
- if pn != nil && pn.Value != "" && TitleReg.MatchString(pn.Value) {
- pnarr = append(pnarr, pn.Value)
- matchres := TitleReg.FindAllStringSubmatch(pn.Value, -1)
- if len(matchres) == 1 && len(matchres[0]) > 0 {
- v1 := u.PackageNumberConvert(matchres[0][0])
- pns[v1] = matchres[0][0]
- bp := &u.BlockPackage{}
- bp.Index = v1
- bp.Origin = matchres[0][0]
- if bp.TableKV == nil {
- bp.TableKV = u.NewJobKv()
- }
- for _, k := range []string{"中标金额", "中标单位", "预算", "成交状态", "项目名称", "项目编号", "采购范围"} {
- if len(table.StandKV[k]) > 0 {
- bp.TableKV.KvTags[k] = append(bp.TableKV.KvTags[k], &u.Tag{Key: k, Value: table.StandKV[k][0].Value})
- }
- }
- bp.WinnerOrder = table.WinnerOrder
- if table.BlockPackage.Map[v1] == nil {
- table.BPackage = true
- table.BlockPackage.AddKey(v1, bp)
- } else {
- table.BlockPackage.RemoveKey(v1)
- table.BlockPackage.AddKey(v1, bp)
- }
- }
- }
- }
- if len(tabs) == len(pns) {
- //多个表格,每个表格都是一个分包 http://www.cxzwfw.gov.cn/info/1009/6963.htm
- //项目名称、项目编号、采购单位、招标机构、预算
- pname := projectnameReg.ReplaceAllString(pnarr[0], "")
- btrue := true
- for _, pn := range pnarr[1:] {
- pn = projectnameReg.ReplaceAllString(pn, "")
- //u.Debug(pn, pname)
- if pn != pname {
- //项目名称不一致
- btrue = false
- break
- }
- }
- if btrue {
- ts.KvTags["项目名称"] = append(ts.KvTags["项目名称"], &u.Tag{Key: "项目名称", Value: pname, Weight: 100})
- for _, table := range tabs {
- table.BPackage = true
- //预算、中标金额、NullTxtBid成交供应商排名 中标单位 成交状态
- if table.BlockPackage != nil && len(table.BlockPackage.Keys) == 1 {
- bp := table.BlockPackage.Map[table.BlockPackage.Keys[0]].(*u.BlockPackage)
- if table.TableResult.WinnerOrder != nil {
- bp.WinnerOrder = table.WinnerOrder
- }
- if bp != nil && table.StandKV != nil {
- if bp.TableKV == nil {
- bp.TableKV = u.NewJobKv()
- }
- for nk, k := range []string{"中标金额", "中标单位", "预算", "成交状态", "项目名称", "项目编号", "采购范围"} {
- if len(table.StandKV[k]) > 0 {
- bp.TableKV.KvTags[k] = append(bp.TableKV.KvTags[k], &u.Tag{Key: k, Value: table.StandKV[k][0].Value})
- }
- if nk < 4 {
- delete(table.StandKV, k)
- }
- }
- }
- }
- }
- }
- }
- }
- for _, table := range tabs {
- table.MergerToTableresult()
- MergeKvTags(ts.KvTags, table.TableResult.KvTags)
- if !table.Brule {
- ts.isUnRulesTab = true
- }
- }
- }
- //解析表格
- func (table *Table) Analy(contactFormat *u.ContactFormat, isSite bool, codeSite string) []*Table {
- //查找表体中的tr对象
- trs := table.Goquery.ChildrenFiltered("tbody,thead,tfoot").ChildrenFiltered("tr")
- if trs.Size() == 0 {
- trs = table.Goquery.ChildrenFiltered("tr")
- }
- ztb := table.Goquery.Find("table").Size()
- if ztb >= 9 {
- return []*Table{}
- }
- //遍历节点,初始化table 结构 TRs Sorts
- table.createTabe(trs, isSite, codeSite)
- if len(table.TRs) == 0 {
- return []*Table{}
- }
- //重置行列
- table.ComputeRowColSpan()
- //对table结构体进行整体解析处理
- ts := table.AnalyTables(contactFormat, isSite, codeSite)
- return ts
- }
- var fblbReg *regexp.Regexp = regexp.MustCompile("(废标|流标|负责人资格|负责人业绩|相关业绩|技术评分明细表|开标记录|附件[:0-9]|越南盾|技术分|填报项目业绩|未通过.*原因)")
- //遍历节点,初始化table 结构体
- func (table *Table) createTabe(trs *goquery.Selection, isSite bool, codeSite string) {
- trs.Each(func(n int, sel *goquery.Selection) {
- //隐藏行不处理
- if IsHide(sel) {
- return
- }
- //遍历每行的td
- tds := sel.ChildrenFiltered("td,th")
- TR := NewTR(table)
- tdTextIsNull := false
- var empty int
- tds.Each(func(m int, selm *goquery.Selection) {
- //对隐藏列不处理!!!
- if IsHide(selm) {
- return
- }
- //进入每一个单元格
- td := NewTD(selm, TR, table, isSite, codeSite) //初始化td,kv处理,td中有table处理,td的方向
- //num++
- TR.AddTD(td)
- if td.Val == "" && td.SonTableResult == nil && len(td.SortKV.Map) == 0 { //删除一个tr,tr中所有td是空值的
- empty++
- if tds.Size() == empty {
- tdTextIsNull = true
- }
- }
- })
- //向table添加每行不为空的tr
- if !tdTextIsNull {
- table.AddTR(TR)
- }
- })
- }
- //对table进行整体解析处理
- func (tn *Table) AnalyTables(contactFormat *u.ContactFormat, isSite bool, codeSite string) []*Table {
- ts := tn.tableSubDemolitionTable() //分包,拆表
- for n, table := range ts {
- //处理每个table
- if len(table.TRs) > 0 {
- //删除尾部空白行
- table.deleteTrimTr()
- //table.Print()
- //校对表格
- table.Adjust(isSite, codeSite)
- //查找表格的标签,table.Tag字段
- table.FindTag()
- //分割表格
- table.bSplit(n, ts, isSite, codeSite)
- table.TdContactFormat(contactFormat, isSite, codeSite) //contactFormat,处理采购单位,代理机构
- //开始查找kv,核心模块,table.SortKV
- table.FindKV(isSite, codeSite)
- //table中抽取品牌,table.BrandData
- if u.IsBrandGoods {
- table.analyBrand()
- }
- //table中抽取单价和个数
- if u.IsPriceNumber {
- //qutil.Debug("======================抽取price和number===========")
- table.extractPriceNumber()
- }
- res, _, _, _, _ := CheckCommon(table.Tag, "abandontable")
- if !res {
- //过滤、标准化、合并kv,table.StandKV,table.StandKVWeight
- table.KVFilter(isSite, codeSite)
- }
- //对有表头表格的处理
- if table.Tag != "" {
- co, m, b := CheckMultiPackage(table.Tag) //分包处理
- if b {
- table.BPackage = b
- if len(table.BlockPackage.Map) == 0 {
- for _, av := range m {
- kv := u.NewJobKv()
- kv.KvTags = table.StandKV
- bd := u.PackageNumberConvert(av[0])
- blockPackage := &u.BlockPackage{
- Origin: av[0],
- Name: av[0],
- Text: co,
- TableKV: kv,
- Index: bd,
- }
- if bd != "" {
- table.BlockPackage.AddKey(bd, blockPackage)
- } else {
- table.BlockPackage.AddKey(av[0], blockPackage)
- }
- }
- }
- table.StandKV["项目名称"] = append(table.StandKV["项目名称"], &u.Tag{Key: "项目名称", Value: table.Tag, Weight: -300})
- }
- }
- //判断是否是多包,并处理分包的//遍历td分块
- table.CheckMultiPackageByTable(isSite, codeSite) //分包处理
- //MergeKvTags(table.TableResult.KvTags, table.StandKV)
- }
- }
- return ts
- }
- //分包,拆表
- func (table *Table) tableSubDemolitionTable() []*Table {
- tm := []map[string]interface{}{}
- tmk := map[string]bool{}
- tmn := map[int]map[string]interface{}{}
- for rownum, tr := range table.TRs {
- if len(tr.TDs) == 1 && table.ColNum > 1 { //tr里面有一列,table里面有多列
- td := tr.TDs[0] //取每行第一个td
- //td开始列等于0 && td结束列+1等于table列数 && td长度大于1小于50
- if td.StartCol == 0 && td.EndCol+1 == table.ColNum && len([]rune(td.Val)) > 1 && len([]rune(td.Val)) < 50 {
- con, m1, b := CheckMultiPackage(td.Val) //判断分包
- if b {
- for k, _ := range m1 {
- numstr := u.PackageNumberConvert(k)
- m2 := map[string]interface{}{
- "tag": con,
- //"num": numstr,
- //"numtxt": v[0],
- "startrow": rownum,
- }
- tmk[numstr] = true
- tmn[rownum] = m2
- tm = append(tm, m2)
- break
- }
- }
- }
- }
- }
- //拆表
- ts := []*Table{}
- if len(tmk) > 1 && len(tmk) == len(tm) {
- var tab1 *Table
- for rownum, tr := range table.TRs {
- if tab1 == nil {
- tab1 = NewTable("", table.TableResult, table.Goquery)
- tab1.BSplit = true
- if tmn[rownum] != nil {
- tab1.StandKV["项目名称"] = append(tab1.StandKV["项目名称"], &u.Tag{Key: "项目名称", Value: tmn[rownum]["tag"].(string), Weight: -100})
- }
- ts = append(ts, tab1)
- }
- if tmn[rownum] != nil {
- tab1.Tag = tmn[rownum]["tag"].(string)
- } else {
- tab1.AddTR(tr)
- }
- if tmn[rownum+1] != nil {
- tab1 = nil
- }
- }
- } else {
- ts = append(ts, table)
- }
- return ts
- }
- //分割表格
- func (table *Table) bSplit(n int, ts []*Table, isSite bool, codeSite string) {
- if table.BSplit {
- if !table.BHeader && n > 0 {
- for i := n - 1; i > -1; i-- {
- if ts[i].BHeader {
- if ts[i].BFirstRow {
- //取第一行插入到
- table.InsertTR(ts[i].TRs[0])
- table.Adjust(isSite, codeSite)
- }
- break
- }
- }
- }
- }
- }
- //删除尾部空白行
- func (table *Table) deleteTrimTr() {
- for len(table.TRs) > 0 {
- npos := len(table.TRs)
- tailTR := table.TRs[npos-1] //最后一个tr,取最后一行
- bspace := true
- for _, v := range tailTR.TDs {
- if v.Val != "" || v.SonTableResult != nil || len(v.SortKV.Keys) > 0 {
- bspace = false
- break
- }
- }
- //删除尾部空行,是空行的话就删除
- if bspace {
- table.TRs = table.TRs[:npos-1]
- } else {
- break
- }
- }
- }
- //校对表格
- func (table *Table) Adjust(isSite bool, codeSite string) {
- //计算行列起止位置,跨行跨列处理
- table.ComputeRowColSpan()
- // for k1, tr := range table.TRs {
- // for k2, td := range tr.TDs {
- // qutil.Debug(k1, k2, td.Val, td.StartRow, td.EndRow, td.StartCol, td.EndCol)
- // }
- // }
- //大概计算每个起止行列的概率
- table.GetKeyRation()
- /*
- for k, v := range table.StartAndEndRation {
- for k1, v1 := range v.Poss {
- bs, _ := json.Marshal(v1)
- str := ""
- for _, td := range v.Tdmap[v1] {
- str += "__" + td.Val + fmt.Sprintf("%d_%d_%d_%d", td.StartRow, td.EndRow, td.StartCol, td.EndCol)
- }
- qutil.Debug(k, k1, string(bs), v.Rationmap[v1], str)
- }
- }
- */
- //u.Debug("tdnum:", num, table.RowNum, table.ColNum)
- //是否是规则的表格,单元各个数=行数*列数
- table.Brule = table.TDNum == table.RowNum*table.ColNum
- count := 0
- for _, trs := range table.TRs {
- for _, td := range trs.TDs {
- if td.BH {
- count++
- }
- }
- }
- if float32(count)/float32(table.TDNum) < 0.85 {
- //精确计算起止行列是表头的概率
- table.ComputeRowColIsKeyRation(isSite, codeSite)
- bhead := false
- L:
- for i, tr := range table.TRs {
- for _, td := range tr.TDs {
- if td.BH {
- //qutil.Debug("----=====---", td.Val, len(table.TRs[len(table.TRs)-1].TDs), i, len(table.TRs)-1)
- if i == len(table.TRs)-1 && len(table.TRs[len(table.TRs)-1].TDs) == 2 {
- res, _, _, _, _ := CheckCommon(td.Val, "abandontable")
- if res {
- //删除此行
- table.TRs = table.TRs[:len(table.TRs)-1]
- table.Adjust(isSite, codeSite)
- return
- }
- }
- bhead = true
- break L
- }
- }
- }
- table.BHeader = bhead
- }
- }
- //计算行/列表格的结束位置 StartRow=0 EndRow=0,table.TDNum td个数 table.RowNum 行数
- func (table *Table) ComputeRowColSpan() {
- n := 0 //td总个数
- mapRC := map[int]map[int]int{} //记录第几行pos,起始列对应的合并值
- for k, v := range table.TRs {
- n += len(v.TDs) //每行的td总数相加
- nk := 0 //nk列的起始,k行的起始||如果有合并,起始就不是0
- ball := true
- rowspans := v.TDs[0].Rowspan //某一行第一个td的rowspan
- for k1, v1 := range v.TDs {
- if k1 == 0 && k == 0 {
- table.TRs[k].TDs[k1].MustBH = true
- table.TRs[k].TDs[k1].BH = true
- }
- if v1.Rowspan != rowspans {
- ball = false
- break
- }
- }
- for _, v1 := range v.TDs {
- if ball {
- v1.Rowspan = 1
- }
- mc := mapRC[k]
- for {
- if mc != nil && mc[nk] > 0 {
- nk += mc[nk]
- } else {
- break
- }
- }
- v1.StartCol = nk
- nk += v1.Colspan - 1
- v1.EndCol = nk
- if nk >= table.ColNum {
- table.ColNum = nk + 1
- }
- nk++
- v1.StartRow = k
- v1.EndRow = k + v1.Rowspan - 1
- ck := fmtkey("c", v1.StartCol, v1.EndCol)
- tdcs := table.StartAndEndRation[ck]
- if tdcs == nil {
- tdcs = NewTDRationScope(ck)
- table.StartAndEndRation[ck] = tdcs
- table.StartAndEndRationKSort.AddKey(ck, 1)
- }
- tdcs.Addtd(v1)
- rk := fmtkey("r", v1.StartRow, v1.EndRow)
- tdrs := table.StartAndEndRation[rk]
- if tdrs == nil {
- tdrs = NewTDRationScope(rk)
- table.StartAndEndRation[rk] = tdrs
- table.StartAndEndRationKSort.AddKey(rk, 1)
- }
- tdrs.Addtd(v1)
- if v1.Rowspan > 1 {
- for i := 1; i < v1.Rowspan; i++ {
- r := k + i
- if r < len(table.TRs) {
- mc := mapRC[r]
- if mc == nil {
- mc = map[int]int{}
- }
- mc[v1.StartCol] = v1.Colspan
- mapRC[r] = mc
- }
- }
- }
- }
- }
- table.TDNum = n //td总个数
- table.RowNum = len(table.TRs) //tr总行数
- }
- func fmtkey(t string, start, end int) string {
- return fmt.Sprintf("%s_%d_%d", t, start, end)
- }
- //查找每个table的标签,如果有标签可按标签处理,否则根据表格去判断
- func (table *Table) FindTag() {
- //查找每个table的标签,如果有标签可按标签处理,否则根据表格去判断
- if table.Tag != "" {
- return
- }
- if table.Tag == "" {
- table.Tag = table.TableResult.BlockTag
- }
- //u.Debug(table.Tag)
- }
- //计算r/c_start_end的概率
- func (table *Table) GetKeyRation() {
- for _, vn := range table.StartAndEndRationKSort.Keys {
- v := table.StartAndEndRation[vn]
- for _, v1 := range v.Poss {
- count := 0
- n := 0
- for _, td := range v.Tdmap[v1] {
- n++
- if td.BH {
- count++
- }
- }
- v.Rationmap[v1] = float32(count) / float32(n)
- }
- }
- }
- //计算行列是表头的概率调用GetKeyRation
- func (table *Table) ComputeRowColIsKeyRation(isSite bool, codeSite string) {
- //增加对跨行校正限止
- // u.Debug(table.Brule, table.ColNum, table.RowNum, table.TDNum)
- bkeyfirstrow := false
- bkeyfirstcol := false
- if table.Brule { //不存在跨行跨列的情况,规则表格
- checkCompute := map[string]bool{}
- for k, tr := range table.TRs {
- rk := fmtkey("r", tr.TDs[0].StartRow, tr.TDs[0].EndRow)
- if k == 0 { //第1行的概率
- ck := fmtkey("c", tr.TDs[0].StartCol, tr.TDs[0].EndCol)
- //u.Debug(table.BFirstRow, "--", table.StartAndEndRation[rk], table.StartAndEndRation[ck])
- ration1, _ := table.StartAndEndRation[rk].GetTDRation(tr.TDs[0])
- ration2, _ := table.StartAndEndRation[ck].GetTDRation(tr.TDs[0])
- if (len(tr.TDs) == 2 && ration2 < 0.55) && (len(tr.TDs) == 2 && ration1 > 0.5) { //第一行为key
- bkeyfirstrow = true
- ball := true
- for _, td := range tr.TDs {
- if MoneyReg.MatchString(td.Val) {
- bkeyfirstrow = false
- ball = false
- td.BH = false
- break
- }
- }
- for _, td := range tr.TDs {
- if ball {
- //td.BH = true
- td.KeyDirect = 1
- td.KVDirect = 2
- }
- }
- } else if ration2 > 0.55 { //第1列
- bkeyfirstcol = true
- if !checkCompute[ck] {
- checkCompute[ck] = true
- //重置第1列
- for _, tr1 := range table.TRs {
- for _, td1 := range tr1.TDs {
- if td1.StartCol == 0 {
- if !MoneyReg.MatchString(td1.Val) {
- //td1.BH = true
- td1.KeyDirect = 2
- td1.KVDirect = 1
- }
- }
- }
- }
- }
- }
- if !bkeyfirstrow && !bkeyfirstcol {
- if len(tr.TDs) > 1 && ration1 > ration2 && ration1 > 0.5 {
- bkeyfirstrow = true
- for _, td := range tr.TDs {
- if !MoneyReg.MatchString(td.Val) {
- //td.BH = true
- td.KeyDirect = 1
- td.KVDirect = 2
- }
- }
- } else if tr.Table.ColNum > 1 && ration2 > 0.5 {
- bkeyfirstcol = true
- if !checkCompute[ck] {
- checkCompute[ck] = true
- //重置第1列
- for _, tr1 := range table.TRs {
- for _, td1 := range tr1.TDs {
- if td1.StartCol == 0 {
- if !MoneyReg.MatchString(td1.Val) {
- td1.BH = true
- td1.KeyDirect = 2
- td1.KVDirect = 1
- }
- }
- }
- }
- }
- }
- }
- } else {
- if bkeyfirstrow {
- //第一列的概率
- ration1, _ := table.StartAndEndRation[rk].GetTDRation(tr.TDs[0])
- if k == 1 || ration1 < checkval {
- for _, td := range tr.TDs {
- if !td.MustBH {
- td.BH = false
- td.KeyDirect = 0
- td.KVDirect = 0
- }
- }
- } //else {for _, td := range tr.TDs {}}
- } else {
- //列在起作用
- if bkeyfirstcol {
- for _, td := range tr.TDs {
- ck := fmtkey("c", td.StartCol, td.EndCol)
- ration1, _ := table.StartAndEndRation[ck].GetTDRation(td)
- if !checkCompute[ck] {
- checkCompute[ck] = true
- if ration1 >= checkval && td.ColPos != 1 {
- for _, tr1 := range table.TRs {
- for _, td1 := range tr1.TDs {
- if td1.StartCol == td.StartCol {
- if !MoneyReg.MatchString(td1.Val) {
- td1.BH = true
- td1.KeyDirect = 2
- td1.KVDirect = 1
- }
- }
- }
- }
- } else {
- for _, tr1 := range table.TRs[1:] {
- for _, td1 := range tr1.TDs[1:] {
- if td1.StartCol == td.StartCol && !td1.MustBH {
- td1.BH = false
- td1.KeyDirect = 0
- td1.KVDirect = 0
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- }
- //qutil.Debug("table.Brule", table.Brule, !bkeyfirstcol && !bkeyfirstrow)
- if !table.Brule || (!bkeyfirstcol && !bkeyfirstrow) {
- //断行问题,虽然同列或同行,但中间被跨行截断,表格方向调整
- for _, k := range table.StartAndEndRationKSort.Keys {
- v := table.StartAndEndRation[k]
- //横向判断,要判断最多的方向,否则会出现不定的情况(map遍历问题)
- k1 := k[:1]
- for _, v2 := range v.Poss {
- lentds := len(v.Tdmap[v2])
- if v.Rationmap[v2] > checkval {
- for _, td := range v.Tdmap[v2] {
- if td.KeyDirect == 0 && !MoneyReg.MatchString(td.Val) && !GSReg.MatchString(td.Val) {
- if k1 == "r" {
- ck := fmtkey("c", td.StartCol, td.EndCol)
- rt := table.StartAndEndRation[ck]
- //clen := 0
- var fv float32
- var tdn []*TD
- if rt != nil {
- fv, tdn = rt.GetTDRation(td)
- //clen = len(tdn)
- }
- if lentds > 1 {
- if ((tdn != nil && v.Rationmap[v2] > fv) || tdn == nil) && td.Valtype != "BO" {
- td.KeyDirect = 1
- td.KVDirect = 2
- //td.BH = true
- }
- }
- } else {
- ck := fmtkey("r", td.StartRow, td.EndRow)
- rt := table.StartAndEndRation[ck]
- var fv float32
- var tdn []*TD
- //clen := 0
- if rt != nil {
- fv, tdn = rt.GetTDRation(td)
- //clen = len(tdn)
- }
- if lentds > 1 {
- if td.Valtype != "NOHEAD" && utf8.RuneCountInString(td.Val) < 15 && ((tdn != nil && v.Rationmap[v2] > fv) || tdn == nil) && td.Valtype != "BO" {
- td.KeyDirect = 2
- td.KVDirect = 1
- td.BH = true
- }
- }
- }
- } else {
- break
- }
- }
- } else if v.Rationmap[v2] < 0.5 && len(v.Tdmap[v2]) > 3 {
- for _, td := range v.Tdmap[v2] {
- // u.Debug(td.Val, "-----", td.BH)
- if td.KeyDirect == 0 && td.BH && !td.MustBH {
- if k1 == "r" {
- ck := fmtkey("c", td.StartCol, td.EndCol)
- rt := table.StartAndEndRation[ck]
- clen := 0
- var fv float32
- var tdn []*TD
- if rt != nil {
- fv, tdn = rt.GetTDRation(td)
- clen = len(tdn)
- }
- if lentds >= clen && lentds > 1 {
- if (tdn != nil && v.Rationmap[v2] < fv) || tdn == nil {
- td.BH = false
- }
- }
- } else {
- ck := fmtkey("r", td.StartRow, td.EndRow)
- rt := table.StartAndEndRation[ck]
- var fv float32
- var tdn []*TD
- clen := 0
- if rt != nil {
- fv, tdn = rt.GetTDRation(td)
- clen = len(tdn)
- }
- if lentds >= clen && lentds > 1 {
- if (tdn != nil && v.Rationmap[v2] < fv) || tdn == nil {
- td.BH = false
- }
- }
- }
- } else {
- break
- }
- }
- }
- }
- }
- }
- table.GetKeyRation()
- if len(table.TRs) > 0 && len(table.TRs[0].TDs) > 0 {
- t0 := table.TRs[0].TDs[0]
- key := fmtkey("r", t0.StartRow, t0.EndRow)
- r, t := table.StartAndEndRation[key].GetTDRation(t0)
- if r > 0.9 && len(t) > 1 {
- table.BFirstRow = true
- }
- for k, tr := range table.TRs {
- if len(tr.TDs) == 1 && tr.TDs[0].StartCol == 0 && tr.TDs[0].EndCol+1 == table.ColNum {
- tr.TDs[0].BH = false
- tr.TDs[0].KVDirect = 0
- sv := FindKv(tr.TDs[0].Val, "", 2)
- _, resm := colonkvEntity.entrance(tr.TDs[0].Val, "", nil, 2, isSite, codeSite)
- for k, v := range resm {
- sv.AddKey(k, v)
- }
- if len(sv.Keys) > 0 {
- for _, v1 := range sv.Keys {
- if tr.TDs[0].SortKV.Map[v1] == nil {
- table.SortKV.AddKey(v1, sv.Map[v1])
- }
- }
- } else if table.Tag == "" && k == 0 && len(tr.TDs[0].Val) > 11 {
- table.Tag = tr.TDs[0].Val
- }
- }
- }
- }
- }
- //查找表格的kv,调用FindTdVal
- func (table *Table) FindKV(isSite bool, codeSite string) {
- //判断全是key的表格不再查找
- if table.BHeader { //只要一个是key即为true
- direct := If(table.BFirstRow, 2, 1).(int) //kv,2查找方向,向上查找
- vdirect := If(direct == 2, 1, 2).(int)
- //控制跨行表格
- bcon := false
- //增加表格切块判断,只判断切块分包
- //控制中标人排序方向
- //bodirect := 0
- //控制中标人排序数值
- //sort := 1
- nextdirect, nextvdirect := 0, 0
- //开始抽取
- //若第一排全为头-临时让第二排-新增 左临 查询,zhengkun
- tb_first_allhead := false
- for tr_index, tr := range table.TRs {
- if tr_index==6 {
- //fmt.Println("调试指定tr")
- }
- bcon = trSingleColumn(tr, bcon, table) //tr单列,是否丢弃内容
- if bcon {
- continue
- }
- if tr.TDs[0].StartRow >= 0 {
- numbh := 0
- for _, td := range tr.TDs {
- //log.Println(tr_index,kkk,td.Val)
- if td.BH {
- numbh++
- }
- }
- if numbh != 0 && numbh == len(tr.TDs) { //5e0d53ef0cf41612e0640495
- if tr_index==0 {
- tb_first_allhead = true
- }
- nextdirect, nextvdirect = 2, 1
- continue
- } else if nextdirect > 0 && nextvdirect > 0 {
- direct, vdirect = 2, 1
- } else if numbh > 0 && numbh <= len(tr.TDs)/2 {
- direct, vdirect = 1, 2
- } else {
- direct, vdirect = 2, 1
- }
- }
- for _, td := range tr.TDs {
- if !td.BH && td.KVDirect < 3 {
- if !table.FindTdVal(td, direct, vdirect) { //table.FindTdVal()存储了table.SortKV
- if !table.FindTdVal(td, vdirect, direct) {
- ////都识别不到时,对第一、二中标候选人的处理
- //bo, res := GetBidOrder(td, bodirect, sort)
- //if res {
- // sort++
- // bodirect = bo
- //}
- //if len(td.SortKV.Map) > 0 {
- // for _, tdv := range td.SortKV.Keys {
- // if tdv == "" || td.SortKV.Map[tdv] == "" { //value为空或者null不再添加到table.SortKV
- // continue
- // }
- // table.SortKV.AddKey(tdv, td.SortKV.Map[tdv])
- // }
- //}
- }
- }
- if tb_first_allhead && tr_index==1 { //临时-让第二排-向左比对
- if !table.FindTdVal(td, 1, 2) { //table.FindTdVal()存储了table.SortKV
- if !table.FindTdVal(td, vdirect, direct) {
- }
- }
- tb_first_allhead = false
- }
- //fmt.Println("td:", td.Val, td.BH, td.HeadTd, td.KVDirect)
- }
- }
- nextdirect, nextvdirect = 0, 0
- }
- //qutil.Debug("FindKV", table.SortKV.Map)
- } else if len(table.TRs) > 0 { //没有表头的表格处理,默认纵向吧
- res := initLongitudinalData(table) //拼装纵向数组
- //再拆值,类似http://www.ggzy.hi.gov.cn/cgzbgg/16553.jhtml第二列,有多个值
- nmapkeys := []int{}
- nmap := map[int][]*u.Kv{}
- L:
- for _, r1 := range res {
- for n, r := range r1 {
- if len([]rune(r)) < 60 { // 长度小于60才去分
- //res1, _ := GetKVAll(r, "", nil)
- res1, _ := colonkvEntity.entrance(r, "", nil, 2, isSite, codeSite)
- if res1 != nil {
- nmap[n] = res1
- nmapkeys = append(nmapkeys, n)
- /**
- //截取串
- for _k1, _ := range res1 {
- r = regexp.MustCompile(_k1+".*").ReplaceAllString(r, "")
- }
- r1[n] = r
- res[pos] = r1
- **/
- } else if nmap[n] != nil {
- //放空值
- nmap[n] = append(nmap[n], &u.Kv{})
- }
- } else {
- nmap = nil
- nmapkeys = nil
- break L
- }
- }
- }
- //调整
- if len(nmap) > 0 {
- kmapkeys := []string{}
- kmap := map[string][]string{}
- for _, mk := range nmapkeys { //同是第n列
- for pos, m1 := range nmap[mk] {
- k, v := m1.Key, m1.Value
- kv := kmap[k]
- if kv == nil {
- kv = []string{}
- }
- kv = append(kv, v)
- kmap[k] = kv
- kmapkeys = append(kmapkeys, k)
- for _, k := range kmapkeys {
- arr := kmap[k]
- if len(arr) < pos {
- arr = append(arr, "")
- kmap[k] = arr
- kmapkeys = append(kmapkeys, k)
- }
- }
- }
- }
- if len(kmap) > 0 {
- for _, k := range kmapkeys {
- if len(kmap[k]) == 1 {
- table.SortKV.AddKey(k, kmap[k][0])
- } else if len(kmap[k]) > 1 {
- table.SortKV.AddKey(k, kmap[k])
- }
- }
- }
- }
- //=================
- //解析值放到map中
- for _, arr := range res {
- if len(arr) > 0 {
- v1 := arr[0]
- _, _, _, _, repl := CheckCommon(v1, "con")
- if repl == "ENT" {
- table.SortKV.AddKey("中标单位", arr)
- continue
- } else if repl == "BO" {
- table.SortKV.AddKey("排名", arr)
- continue
- }
- }
- }
- }
- //qutil.Debug("Table-FindKV", table.SortKV.Map)
- }
- //初始化组装纵向数据
- func initLongitudinalData(table *Table) [][]string {
- res := make([][]string, len(table.TRs[0].TDs)) //创建table第一行的列数长度
- for n, _ := range res {
- res[n] = []string{}
- }
- for _, tr := range table.TRs {
- for n, td := range table.TRs[0].TDs { //第一行的所有td
- td1 := table.GetTdByRCNo(tr.TDs[0].StartRow, td.StartCol) //根据行号列号获取td对象
- if td1 != nil {
- res[n] = append(res[n], td1.Val)
- } else {
- res[n] = append(res[n], "")
- }
- }
- }
- return res
- }
- //tr单列,是否丢弃内容
- func trSingleColumn(tr *TR, bcon bool, table *Table) bool {
- if len(tr.TDs) == 1 {
- bcon = false
- td := tr.TDs[0]
- if td.StartCol == 0 && td.EndCol+1 == table.ColNum && len([]rune(td.Val)) > 4 && len([]rune(td.Val)) < 50 {
- res, _, _, _, _ := CheckCommon(td.Val, "abandontable")
- if res { //以下内容丢弃
- bcon = true
- }
- }
- }
- return bcon
- }
- //获取中标人顺序
- //direct 0默认 1横向 2纵向
- func GetBidOrder(td *TD, direct, n int) (d int, res bool) {
- if td.Valtype != "BO" {
- return
- }
- if td.Rowspan > 1 {
- for i := 0; i < td.Rowspan; i++ {
- nextcol := 1
- L1:
- for {
- vtd := td.TR.Table.GetTdByRCNo(td.StartRow+i, td.EndCol+nextcol)
- if vtd == nil {
- break L1
- }
- nextcol += vtd.Colspan
- if filter_zbdw_v2.MatchString(vtd.Val) {
- arrbo := td.TR.Table.SortKV.Map[NullTxtBid]
- if arrbo == nil {
- arrbo = []map[string]interface{}{}
- td.TR.Table.SortKV.AddKey(NullTxtBid, arrbo)
- }
- a1 := arrbo.([]map[string]interface{})
- a1 = append(a1, map[string]interface{}{
- "entname": vtd.Val,
- "sortstr": td.Val,
- "sort": GetBidSort(td.Val, n),
- })
- res = true
- td.TR.Table.SortKV.AddKey(NullTxtBid, a1)
- }
- }
- }
- } else if td.Colspan > 1 {
- for i := 1; i < td.Colspan; i++ {
- nextcol := 0
- L2:
- for {
- vtd := td.TR.Table.GetTdByRCNo(td.StartRow+i, td.StartCol+nextcol)
- if vtd == nil || vtd.Colspan >= td.Colspan {
- break L2
- }
- nextcol += vtd.Colspan
- if filter_zbdw_v2.MatchString(vtd.Val) {
- arrbo := td.TR.Table.SortKV.Map[NullTxtBid]
- if arrbo == nil {
- arrbo = []map[string]interface{}{}
- td.TR.Table.SortKV.AddKey(NullTxtBid, arrbo)
- }
- a1 := arrbo.([]map[string]interface{})
- a1 = append(a1, map[string]interface{}{
- "entname": vtd.Val,
- "sortstr": td.Val,
- "sort": GetBidSort(td.Val, n),
- })
- res = true
- td.TR.Table.SortKV.AddKey(NullTxtBid, a1)
- }
- }
- }
- } else {
- rtd := td.TR.Table.GetTdByRCNo(td.StartRow, td.EndCol+1)
- btd := td.TR.Table.GetTdByRCNo(td.EndRow+1, td.StartCol)
- //if ((rtd != nil && !rtd.BH && rtd.Valtype == "BO") || direct == 1) && btd != nil && filter_zbdw_v.MatchString(btd.Val) {
- if ((rtd != nil && !rtd.BH) || direct == 1) && btd != nil && filter_zbdw_v2.MatchString(btd.Val) {
- d = 1
- arrbo := td.TR.Table.SortKV.Map[NullTxtBid]
- if arrbo == nil {
- arrbo = []map[string]interface{}{}
- td.TR.Table.SortKV.AddKey(NullTxtBid, arrbo)
- }
- a1 := arrbo.([]map[string]interface{})
- a1 = append(a1, map[string]interface{}{
- "entname": btd.Val,
- "sortstr": td.Val,
- "sort": GetBidSort(td.Val, n),
- })
- res = true
- td.TR.Table.SortKV.AddKey(NullTxtBid, a1)
- //} else if ((btd != nil && !btd.BH && btd.Valtype == "BO") || direct == 2) && rtd != nil && filter_zbdw_v.MatchString(rtd.Val) {
- } else if ((btd != nil && !btd.BH) || direct == 2) && rtd != nil && filter_zbdw_v2.MatchString(rtd.Val) {
- d = 2
- arrbo := td.TR.Table.SortKV.Map[NullTxtBid]
- if arrbo == nil {
- arrbo = []map[string]interface{}{}
- td.TR.Table.SortKV.AddKey(NullTxtBid, arrbo)
- }
- a1 := arrbo.([]map[string]interface{})
- a1 = append(a1, map[string]interface{}{
- "entname": rtd.Val,
- "sortstr": td.Val,
- "sort": GetBidSort(td.Val, n),
- })
- res = true
- td.TR.Table.SortKV.AddKey(NullTxtBid, a1)
- }
- }
- return
- }
- func GetBidSort(str string, n int) int {
- val := n
- if strings.Index(str, "首选") > -1 {
- val = 1
- } else {
- val = winnerOrderEntity.toNumber(str, n)
- }
- return val
- }
- var cleardwReg *regexp.Regexp = regexp.MustCompile("[((]{1}\\d*[人元件个公斤户]/[人元件个公斤户][))]")
- var zbhxrReg *regexp.Regexp = regexp.MustCompile("(中标候选人|投标单位名称)")
- //查找每一个单元格的表头,调用FindNear
- func (table *Table) FindTdVal(td *TD, direct, vdirect int) (b bool) {
- if td.Val == "" || strings.TrimSpace(td.Val) == "" {
- return
- }
- near := table.FindNear(td, direct)
- if near != nil && near.BH && (near.KeyDirect == vdirect || near.KeyDirect == 0) && (near.KVDirect == direct || near.KVDirect == 0) && near.KVDirect < 3 {
- near.KVDirect = direct
- near.KeyDirect = vdirect
- td.KVDirect = direct
- key := repSpace.ReplaceAllString(near.Val, "")
- if key == "名称" && near.StartCol == 0 && near.Rowspan > 0 {
- for _, vn := range table.TRs[near.Rowspan-1].TDs {
- if strings.Contains(vn.Val, "代理") {
- key = "代理机构"
- break
- } else if strings.Contains(vn.Val, "招标") {
- key = "采购单位"
- break
- } else if strings.Contains(vn.Val, "中标") {
- key = "中标单位"
- break
- }
- }
- } else if zbhxrReg.MatchString(key) && findCandidate2.MatchString(td.Val) {
- key = "中标单位"
- } else if key == "单位名称" {
- tmpnewnear := table.FindNear(near, 2)
- if tmpnewnear != nil {
- if tmpnewnear.MustBH || tmpnewnear.BH {
- key = tmpnewnear.Val + near.Val
- }
- } else {
- tmpnewnear = table.FindNear(near, 1)
- if tmpnewnear != nil {
- if tmpnewnear.MustBH || tmpnewnear.BH {
- key = tmpnewnear.Val + near.Val
- }
- }
- }
- }
- if near.Val == "" {
- key = fmtkey("k", near.TR.RowPos, near.ColPos)
- }
- val := table.SortKV.Map[key]
- //qutil.Debug("====================", "key:", key, "val:", val)
- bthiskey := false
- if val != nil {
- curpos := table.SortKV.Index[key]
- thistr := table.kTD[curpos]
- if thistr != near {
- if strings.TrimSpace(near.Val) == "名称" && near.TR != nil && len(near.TR.TDs) > 0 && near.ColPos-1 >= 0 {
- rv := near.TR.TDs[near.ColPos-1].Val
- if near.ColPos > 0 && (strings.Contains(rv, "招标") || strings.Contains(rv, "代理") || strings.Contains(rv, "采购") || strings.Contains(rv, "中标")) {
- near = near.TR.TDs[near.ColPos-1]
- }
- } else {
- bthiskey = true
- }
- } else {
- bthiskey = true
- }
- }
- bfind := false
- barr := false
- varrpos := -1
- if bthiskey {
- //处理是数组值,且有合并行或合并列的情况 kvscope,对数组值的处理
- pos := table.SortKV.Index[key]
- mval := table.kvscope[pos]
- bvalfind := false
- if direct == 1 { //kv是横向
- L1:
- for k3, v3 := range mval {
- for _, v4 := range v3 {
- if v4.EndRow+1 == td.StartRow && v4.EndCol == td.EndCol {
- varrpos = k3
- bvalfind = true
- break L1
- }
- }
- }
- } else { //kv是纵向
- L2:
- for k3, v3 := range mval {
- for _, v4 := range v3 {
- if v4.EndCol+1 == td.StartCol && v4.EndRow == td.EndRow {
- varrpos = k3
- bvalfind = true
- break L2
- }
- }
- }
- }
- if vals, ok := val.([]string); ok {
- if near.Val == "" {
- bn := false
- for _, vs := range vals {
- if vs != "" && NullTdReg.MatchString(vs) {
- bn = true
- } else {
- bn = false
- break
- }
- }
- if bn {
- near.Val = NullTxtBid
- key = NullTxtBid
- bfind = true
- }
- }
- if bvalfind && varrpos > -1 && len(vals) > varrpos {
- tmapval := strings.TrimSpace(cleardwReg.ReplaceAllString(td.Val, ""))
- if tmapval == "" {
- vals = append(vals, td.Val) // 累加
- } else {
- vals = append(vals, tmapval) // 累加
- }
- val = vals
- //vals[varrpos] = td.Val // += "__" + td.Val
- } else {
- //添加时候去除空值和nil
- newVals := []string{}
- for _, isval := range vals {
- if isval == "" {
- continue
- }
- newVals = append(newVals, isval)
- }
- //vals = append(vals, td.Val)
- if td.Val != "" {
- newVals = append(newVals, td.Val)
- }
- val = newVals
- varrpos = len(vals) - 1
- }
- } else if vals, ok := val.(string); ok && vals != "" && td.Val != "" {
- tmapval := strings.TrimSpace(cleardwReg.ReplaceAllString(vals, ""))//已存在的kv
- tmapvaltd := strings.TrimSpace(cleardwReg.ReplaceAllString(td.Val, ""))
- if bvalfind {
- //if tmapvaltd == "" {
- // val = td.Val //vals + "__" + td.Val
- //} else {
- // val = tmapvaltd
- //}
- if key=="中标单位" {
- //不能覆盖---
- }else {
- if tmapvaltd == "" {
- val = td.Val //vals + "__" + td.Val
- } else {
- val = tmapvaltd
- }
- }
- } else{
- if key=="中标单位" {
- //新增不能数组
- }else {
- tval := []string{}
- if tmapval == "" {
- tval = append(tval, vals)
- } else {
- tval = append(tval, tmapval)
- }
- if tmapvaltd == "" {
- tval = append(tval, td.Val)
- } else {
- tval = append(tval, tmapvaltd)
- }
- val = tval
- varrpos = 1
- }
- }
- }
- barr = true
- } else {
- if td.Val != "" {
- tmapval := strings.TrimSpace(cleardwReg.ReplaceAllString(td.Val, ""))
- if tmapval == "" {
- val = td.Val
- } else {
- val = tmapval
- }
- } else if len(near.SortKV.Map) == 1 && near.SortKV.Map[near.Val] != "" {
- val = near.SortKV.Map[near.Val]
- }
- }
- td.HeadTd = near
- if bfind {
- tkey := fmtkey("k", near.TR.RowPos, near.ColPos)
- table.SortKV.ReplaceKey(key, val, tkey)
- } else {
- if key == "单位名称" && len(near.TR.TDs) > 1 {
- if near.TR.TDs[0].Val != "序号" {
- key = near.TR.TDs[0].Val
- }
- }
- table.SortKV.AddKey(key, val)
- pos := table.SortKV.Index[key]
- if barr {
- mval := table.kvscope[pos]
- if mval != nil {
- tds := mval[varrpos]
- if tds != nil {
- tds = append(tds, td)
- } else {
- tds = []*TD{td}
- }
- if varrpos > -1 {
- mval[varrpos] = tds
- table.kvscope[pos] = mval
- }
- }
- } else {
- table.kvscope[pos] = map[int][]*TD{
- 0: []*TD{td},
- }
- table.kTD[pos] = near
- }
- }
- b = true
- }
- return
- }
- //查找单元格的表头时,横向或纵向
- func (table *Table) FindNear(td *TD, direct int) *TD {
- if direct == 1 && td.StartCol > 0 { //左临
- tr := table.TRs[:td.TR.RowPos+1]
- for i := len(tr) - 1; i > -1; i-- {
- tds := tr[i].TDs
- for _, td1 := range tds {
- if td1.StartRow <= td.StartRow && td1.EndRow >= td.EndRow && td1.EndCol+1 == td.StartCol {
- //找到左临节点
- if td1.BH {
- return td1
- } else {
- if td1.HeadTd != nil && td1.HeadTd.KVDirect == direct {
- return td1.HeadTd
- }
- }
- }
- }
- }
- } else if direct == 2 && td.StartRow > 0 { //上临
- tr := table.TRs[:td.TR.RowPos]
- for i := len(tr) - 1; i > -1; i-- {
- tds := tr[i].TDs
- for it, td1 := range tds {
- if td1.StartCol <= td.StartCol && td1.EndCol >= td.EndCol && td1.EndRow+1 == td.StartRow {
- //找到左临节点
- if td1.BH {
- return td1
- } else if len(tr[i].TDs) == len(td.TR.TDs) && td1.HeadTd != nil && td1.HeadTd.KVDirect == direct {
- return td1.HeadTd
- } else if it > 0 && td1.Val == "" && td1.TR.TopTR == nil && len(td.TR.TDs)-(td.StartCol-1) > 0 && strings.Contains(td.TR.TDs[td.StartCol-1].Val, "中标候选人") {
- return tds[it-1]
- } else if td1.HeadTd != nil && td1.HeadTd.KVDirect == direct && td.Colspan == td1.Colspan && td.Rowspan == td.Rowspan {
- return td1.HeadTd
- }
- } else if td1.StartCol <= td.StartCol && td1.EndCol >= td.EndCol && td1.EndRow+1 == td.StartRow {
- }
- }
- }
- }
- return nil
- }
- //根据行号列号获取td对象
- func (tn *Table) GetTdByRCNo(row, col int) *TD {
- for _, tr := range tn.TRs {
- for _, td := range tr.TDs {
- if td.StartCol <= col && td.EndCol >= col && td.StartRow <= row && td.EndRow >= row {
- return td
- }
- }
- }
- return nil
- }
- //判断表格是否是分包
- func (tn *Table) CheckMultiPackageByTable(isSite bool, codeSite string) (b bool, index []string) {
- pac := 0 //包的数量
- val := 0 //分值
- index = []string{} //存储分包,使用tbale.SortKV的key和value使用正则等处理对值进行判断
- index_pos := []int{} //下标
- //是数组且能找到标段之类的提示
- //arr_count := 0 //计数table.SortKV的value是数组的数量,后面没用
- key_index := -1
- hasPkgTd := map[string]bool{}
- //初始化CheckMultiPackageByTable方法需要的数据
- key_index, index, index_pos, val, pac, hasPkgTd = initCheckMultiPackageByTable(tn, key_index, index, index_pos, val, pac, hasPkgTd)
- //key是分包的情况
- //记录key对应的值
- commonKeyVals := map[string][]string{}
- //记录key出现的次数
- keyExistsCount := map[string]int{}
- if pac > 1 {
- val = 10
- } else {
- //查找标签
- if TableMultiPackageReg_4.MatchString(tn.Tag) {
- val += 4
- } else if TableMultiPackageReg_2.MatchString(tn.Tag) {
- val += 4
- }
- //根据table.SortKV的key判断是否分包,如果没有再根据value判断
- val, index, index_pos = foundPacBySortKV(tn, val, index, index_pos, &keyExistsCount, &commonKeyVals, key_index, hasPkgTd)
- }
- // u.Debug(index)
- //过滤重复及标准化!
- standIndex := []string{}
- standIndex_pos := []int{}
- oldIndex := []string{} //存放包的原始值
- brepeat := map[string]bool{}
- for k, v := range index {
- v = u.PackageNumberConvert(v)
- if !brepeat[v] {
- brepeat[v] = true
- standIndex = append(standIndex, v)
- standIndex_pos = append(standIndex_pos, index_pos[k])
- oldIndex = append(oldIndex, index[k])
- }
- }
- index = standIndex
- //有一个以上的包,并且相同的key出现一次以上,认为这个key是属于包里面的
- if len(commonKeyVals) > 0 {
- for k, v := range commonKeyVals {
- if len(index) > 1 && keyExistsCount[k] < 2 {
- continue
- }
- tn.SortKV.AddKey(k, v)
- }
- }
- //
- isGoonNext := false
- if val > 4 && len(brepeat) > 0 {
- b = true
- //多包解析
- if b {
- tn.BPackage = true
- //pnum := len(index)
- //根据数组index分包长度添加table.BlockPackage子包数组
- for nk, v := range index {
- if tn.BlockPackage.Map[v] == nil {
- kv := u.NewJobKv()
- for tnk, tnv := range tn.StandKV {
- if nk >= len(tnv) {
- continue
- } else if len(index) == len(tnv) {
- //特殊处理-
- if tnk=="预算"&& codeSite=="ha_zmdszfcgw_cgxx" && len(tnv)>1{
- isEqErr,budget_v := false,""
- for bk,bv:=range tnv {
- if bk==0 {
- budget_v = bv.Value
- }else {
- if budget_v != bv.Value {
- isEqErr = true
- break
- }
- }
- }
- if isEqErr {
- kv.KvTags[tnk] = append(kv.KvTags[tnk], tnv[nk])
- }
- }else {
- kv.KvTags[tnk] = append(kv.KvTags[tnk], tnv[nk])
- }
- }
- }
- //kv.KvTags = tn.StandKV
- bp := &u.BlockPackage{}
- bp.Index = v //序号 (转换后编号,只有数字或字母)
- bp.Origin = oldIndex[nk] //包的原始值
- bp.TableKV = kv //table kv (分出的对应的KV值)
- bp.Name = v
- if bp.TableKV != nil && bp.TableKV.KvTags != nil && len(bp.TableKV.KvTags) > 0 {
- for kc, cv := range bp.TableKV.KvTags {
- if kc == "预算" && bp.Budget <= 0 {
- moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
- if len(moneys) > 0 {
- if vf, ok := moneys[0].(float64); ok {
- bp.Budget = vf
- bp.IsTrueBudget = moneys[len(moneys)-1].(bool)
- } else if vi, ok := moneys[0].(int); ok {
- bp.Budget = float64(vi)
- bp.IsTrueBudget = moneys[len(moneys)-1].(bool)
- }
- }
- } else if kc == "中标金额" && bp.Bidamount <= 0 {
- moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
- if len(moneys) > 0 {
- if vf, ok := moneys[0].(float64); ok {
- bp.Bidamount = vf
- bp.IsTrueBidamount = moneys[len(moneys)-1].(bool)
- } else if vi, ok := moneys[0].(int); ok {
- bp.Bidamount = float64(vi)
- bp.IsTrueBidamount = moneys[len(moneys)-1].(bool)
- }
- }
- } else if kc == "中标单位" && bp.Winner == "" {
- bp.Winner = cv[0].Value
- }
- //拼接内容
- if !excludeKey.MatchString(kc) {
- bp.Text += fmt.Sprintf("%v:%v\n", kc, cv[0].Value)
- }
- }
- }
- tn.BlockPackage.AddKey(v, bp) //table子包数组
- }
- }
- isGoonNext = tn.manyPackageProcessByIndex(index, standIndex_pos, isSite, codeSite) //多包处理,处理不同情况下的分包
- }
- } else {
- isGoonNext = true
- }
- if isGoonNext { //没有处理成数组的情况下,继续调用正文查找分包的方法
- tn.isGoonNext(isSite, codeSite)
- }
- //查找分包中的中标人排序
- if tn.BlockPackage != nil && tn.BlockPackage.Keys != nil && len(tn.BlockPackage.Keys) > 0 {
- for _, v := range tn.BlockPackage.Keys {
- vv, ok := tn.BlockPackage.Map[v].(*u.BlockPackage)
- if ok && (vv.WinnerOrder == nil || len(vv.WinnerOrder) == 0) {
- vv.WinnerOrder = winnerOrderEntity.Find(vv.Text, true, 2, isSite, codeSite)
- }
- }
- }
- return
- }
- //多包处理,处理不同情况下的分包
- func (tn *Table) manyPackageProcessByIndex(index []string, standIndex_pos []int, isSite bool, codeSite string) (isGoonNext bool) {
- if len(index) == 1 { //是一个的情况
- if len(tn.SortKV.Keys) < 10 && tn.ColNum < 10 && tn.RowNum < 4 { //table带排序的KV值小于10并且小于10列和小于4行
- beq := true
- for _, v2 := range tn.SortKV.Keys {
- if _, ok := tn.SortKV.Map[v2].(string); !ok {
- beq = false
- break
- }
- }
- if beq { //统一处理为数组
- td := tn.GetTdByRCNo(tn.RowNum-1, 0)
- if !td.BH && FindVal2_1.MatchString(td.Val) {
- for _, v2 := range tn.SortKV.Keys {
- tn.SortKV.AddKey(v2, []string{tn.SortKV.Map[v2].(string)})
- }
- } else {
- //没有处理成数组的情况下,继续调用正文查找分包的方法
- isGoonNext = true
- }
- }
- }
- }
- for _, k1 := range tn.SortKV.Keys {
- v1 := tn.SortKV.Map[k1]
- var v1_arr []string
- if vtmpv1, ok := v1.(string); ok {
- v1_arr = PreCon4.FindAllString(qutil.ObjToString(vtmpv1), -1)
- if len(v1_arr) > 0 {
- if dw := Precon4dw.FindString(vtmpv1); dw != "" {
- for i, v := range v1_arr {
- v1_arr[i] = v + dw
- }
- }
- }
- } else if vtmpv1s, ok := v1.([]string); ok {
- v1_arr = vtmpv1s
- }
- if len(v1_arr) > 0 && len(v1_arr) <= len(index) { //table.SortKV.Map.value数组小于等于分包index
- for k, v := range v1_arr {
- tn.assemblePackage(k1, v, index[k], isSite, codeSite) //组装解析到的分包
- }
- }
- }
- return isGoonNext
- }
- //没有处理成数组的情况下,继续调用正文查找分包的方法
- func (tn *Table) isGoonNext(isSite bool, codeSite string) {
- blockPackage := map[string]*u.BlockPackage{}
- for _, k := range tn.SortKV.Keys {
- if excludeKey.MatchString(k) || strings.Contains(k, "批复") || excludeKey3.MatchString(k) {
- continue
- }
- str := "" //拼装为冒号kv
- v := tn.SortKV.Map[k]
- nk := regReplAllSpace.ReplaceAllString(k, "")
- if vs, ok := v.([]string); ok {
- str += fmt.Sprintf("%s:%s\n", nk, strings.Join(vs, " "))
- } else {
- str += fmt.Sprintf("%s:%s\n", nk, v)
- }
- if excludeKey2.MatchString(str) {
- continue
- }
- b, _ := divisionPackageChild(&blockPackage, str, tn.Tag, false, false, isSite, codeSite) //分块之后分包
- if b && len(blockPackage) > 0 {
- tn.BPackage = true
- for mk, mv := range blockPackage {
- if tn.BlockPackage.Map[mk] == nil {
- tn.BlockPackage.AddKey(mk, mv)
- } else {
- bp := tn.BlockPackage.Map[mk].(*u.BlockPackage)
- if bp.TableKV == nil {
- bp.TableKV = u.NewJobKv()
- }
- if bp.SpaceKV == nil {
- bp.SpaceKV = u.NewJobKv()
- }
- for k2, v2 := range mv.ColonKV.KvTags {
- for _, v2v := range v2 {
- isExists := false
- for _, v2vv := range bp.TableKV.KvTags[k2] {
- if v2v.Value == v2vv.Value {
- isExists = true
- break
- }
- }
- if !isExists {
- bp.TableKV.KvTags[k2] = append(bp.TableKV.KvTags[k2], v2v)
- bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
- }
- }
- }
- for k2, v2 := range mv.SpaceKV.KvTags {
- for _, v2v := range v2 {
- isExists := false
- for _, v2vv := range bp.SpaceKV.KvTags[k2] {
- if v2v.Value == v2vv.Value {
- isExists = true
- break
- }
- }
- if !isExists {
- bp.SpaceKV.KvTags[k2] = append(bp.SpaceKV.KvTags[k2], v2v)
- bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
- }
- }
- }
- }
- }
- tn.BPackage = true
- tn.SortKV.RemoveKey(k)
- }
- }
- }
- //根据table.SortKV的key判断是否分包,如果没有再根据value判断
- func foundPacBySortKV(tn *Table, val int, index []string, index_pos []int, keyExistsCount *map[string]int, commonKeyVals *map[string][]string, key_index int, hasPkgTd map[string]bool) (rval int, rindex []string, rindex_pos []int) {
- keyIsPkg := false
- for in, k := range tn.SortKV.Keys {
- if excludeKey.MatchString(BracketsTextReg.ReplaceAllString(k, "")) || excludeKey3.MatchString(k) || regFJWarap.MatchString(k) || regAZWarap.MatchString(k) { //判断分包前排除
- continue
- }
- v := tn.SortKV.Map[k]
- //key是分包的情况
- if ismatch := FindVal_1.MatchString(k); keyIsPkg || ismatch {
- if ismatch {
- keyIsPkg = true
- val += 4
- pkgFlag := FindVal_1.FindString(k) //对值进行分包判断
- k = strings.Replace(k, pkgFlag, "", -1)
- index = append(index, pkgFlag)
- index_pos = append(index_pos, len(index))
- val += 1
- //pac++
- } else {
- k = strings.TrimRight(k, "_")
- }
- (*keyExistsCount)[k] = (*keyExistsCount)[k] + 1
- (*commonKeyVals)[k] = append((*commonKeyVals)[k], qutil.ObjToString(v))
- } else if k1 := FilterKey_2.ReplaceAllString(k, ""); FindKey_2.MatchString(k1) {
- val += 4
- //value数组分包
- if vs, bvs1 := v.([]string); bvs1 {
- L:
- for in2, v1 := range vs {
- if len([]rune(v1)) < 20 && !moneyNum.MatchString(v1) && FindVal2_1.MatchString(v1) {
- for _, serial := range tn.TableResult.RuleBlock.TitleRegs {
- if serial.MatchString(v1) {
- break L
- }
- }
- if key_index == -1 {
- key_index = in
- } else if key_index != in {
- break
- }
- index = append(index, v1)
- index_pos = append(index_pos, in2)
- val += 1
- //pac++
- }
- }
- } else if v1, ok := v.(string); ok && !hasPkgTd[k] {
- //value字符串分包
- v1 = replPkgConfusion(v1) //替换分包中混淆的词
- for _, v2 := range strings.Split(v1, "/") {
- if len([]rune(v2)) < 20 && !moneyNum.MatchString(v2) && FindVal2_1.MatchString(v2) {
- key_index = in
- index = append(index, v1)
- index_pos = append(index_pos, 0)
- val += 1
- //pac++
- underline := ""
- for {
- underline += "_"
- if tn.SortKV.Map[k+underline] == nil {
- break
- } else if v3, v2_ok := tn.SortKV.Map[k+underline].(string); v2_ok && v3 != "" {
- index = append(index, v3)
- index_pos = append(index_pos, 1)
- } else if v3, v2_ok := tn.SortKV.Map[k+underline].([]string); v2_ok {
- for v2_k, v2_v := range v3 {
- index = append(index, v2_v)
- index_pos = append(index_pos, v2_k+1)
- }
- }
- }
- break
- }
- }
- }
- if k1=="标段" && len(index)==0 {
- continue
- }else {
- break
- }
- }
- }
- return val, index, index_pos
- }
- //初始化CheckMultiPackageByTable方法需要的数据
- func initCheckMultiPackageByTable(tn *Table, key_index int, index []string, index_pos []int, val int, pac int, hasPkgTd map[string]bool) (rkey_index int, rindex []string, rindex_pos []int, rval int, rpac int, rhasPkgTd map[string]bool) {
- for in, k := range tn.SortKV.Keys {
- //涉及包号|包件号?|项目标号|规格|型号|招标范围|业绩|废标)|(^编号$)|([^包段标]编号)就跳过
- if excludeKey.MatchString(BracketsTextReg.ReplaceAllString(k, "")) || excludeKey3.MatchString(k) || strings.Contains(k, "批复") {
- continue
- }
- v := tn.SortKV.Map[k]
- if vs, bvs := v.([]string); bvs {
- //arr_count++
- haspkgs := []string{}
- for in2, v1 := range vs {
- v1 = replPkgConfusion(v1) //替换分包中混淆的词
- if len([]rune(v1)) < 8 && !moneyNum.MatchString(v1) && FindVal_1.MatchString(v1) {
- if key_index == -1 {
- key_index = in
- } else if key_index != in {
- break
- }
- index = append(index, FindVal_1.FindString(v1))
- index_pos = append(index_pos, in2)
- val += 1
- pac++
- } else if FindKey_3.MatchString(k) {
- //5db2a101a5cb26b9b73054ac
- index = append(index, v1)
- index_pos = append(index_pos, in2)
- val += 1
- pac++
- } else {
- if ok, v1new := isHasOnePkgAndNoKv(v1); ok { //td的值里面有一个包,并且没有冒号kv
- haspkgs = append(haspkgs, v1new)
- }
- }
- }
- /*处理这种情况:
- <tr><td>包一:xxxxxxxxx</td></tr>
- <tr><td>包二:xxxxxxxxx</td></tr>
- */
- if len(index) == 0 && len(haspkgs) > 0 && len(haspkgs) == len(vs) {
- for in2, v1 := range haspkgs {
- if key_index == -1 {
- key_index = in
- } else if key_index != in {
- break
- }
- index = append(index, v1)
- index_pos = append(index_pos, in2)
- val += 1
- pac++
- }
- }
- } else if v1, ok := v.(string); ok {
- v1 = replPkgConfusion(v1) //替换分包中混淆的词
- if len([]rune(v1)) < 8 && !moneyNum.MatchString(v1) && FindVal_1.MatchString(v1) {
- key_index = in
- index = append(index, FindVal_1.FindString(v1))
- index_pos = append(index_pos, 0)
- val += 1
- pac++
- } else if getTd := tn.GetTdByRCNo(0, tn.SortKV.Index[k]); getTd != nil && getTd.KVDirect == 2 { //纵向
- /*处理这种情况:
- <tr><td>包一:xxxxxxxxx</td></tr>
- */
- if ok, v1new := isHasOnePkgAndNoKv(v1); ok {
- hasPkgTd[k] = true
- key_index = in
- index = append(index, v1new)
- index_pos = append(index_pos, 0)
- val += 1
- pac++
- }
- }
- }
- }
- return key_index, index, index_pos, val, pac, hasPkgTd
- }
- //组装解析到的分包,//key如果匹配到抽取关键词就添加到table.SortKV
- func (tn *Table) assemblePackage(k1, v1, key string, isSite bool, codeSite string) {
- bp := tn.BlockPackage.Map[key].(*u.BlockPackage)
- if bp.TableKV == nil {
- bp.TableKV = u.NewJobKv()
- }
- if v1 != "" {
- kvTags, _ := CommonDataAnaly(k1, "中标情况", "", v1, isSite, codeSite) //匹配抽取关键词
- for k3, v3 := range kvTags {
- if bp.TableKV.KvTags[k3] == nil {
- bp.TableKV.KvTags[k3] = append(bp.TableKV.KvTags[k3], v3...)
- } else if k3 == "预算" && bp.Budget <= 0 {
- moneys := clear.ObjToMoney([]interface{}{v3[0].Value, ""})
- if len(moneys) > 0 {
- if vf, ok := moneys[0].(float64); ok {
- bp.Budget = vf
- bp.IsTrueBudget = moneys[len(moneys)-1].(bool)
- } else if vi, ok := moneys[0].(int); ok {
- bp.Budget = float64(vi)
- bp.IsTrueBudget = moneys[len(moneys)-1].(bool)
- }
- }
- } else if k3 == "中标金额" && bp.Bidamount <= 0 {
- moneys := clear.ObjToMoney([]interface{}{v3[0].Value, ""})
- if len(moneys) > 0 {
- if vf, ok := moneys[0].(float64); ok {
- bp.Bidamount = vf
- bp.IsTrueBidamount = moneys[len(moneys)-1].(bool)
- } else if vi, ok := moneys[0].(int); ok {
- bp.Bidamount = float64(vi)
- bp.IsTrueBidamount = moneys[len(moneys)-1].(bool)
- }
- }
- }
- }
- }
- k1 = regReplAllSpace.ReplaceAllString(k1, "")
- //拼接内容
- if !excludeKey.MatchString(k1) {
- bp.Text += fmt.Sprintf("%v:%v\n", k1, v1)
- }
- tn.BlockPackage.AddKey(key, bp)
- }
- /**
- 之前爬虫过来的数据对table表格的抓取异常问题
- 查找并修正不规则表格的字符串,只对全文做处理,块内的表格不需要修正
- **/
- var thbf = regexp.MustCompile("(?i)</?t(head|body|foot)>")
- //需要保留thead
- var saveThead = regexp.MustCompile("(?is)<thead>(.+?)</thead>")
- var clearpkg = regexp.MustCompile("(标示|标识)")
- func RepairCon(con string) string {
- con = clearpkg.ReplaceAllString(con, "")
- res := saveThead.FindAllStringSubmatch(con, 1)
- th := ""
- if len(res) == 1 && len(res[0]) == 2 {
- th = u.TrimLeftSpace(res[0][1], "")
- }
- con = thbf.ReplaceAllString(con, "")
- con = u.TrimLeftSpace(con, "")
- itbody := strings.Index(con, "<tr")
- iLen := 3
- if itbody == 0 {
- con = findpos(con, iLen, itbody)
- } else {
- itable := strings.Index(con, "<table")
- if itable == -1 || itable > itbody {
- con = findpos(con, iLen, itbody)
- }
- }
- //保留第一个thead
- if th != "" {
- con = strings.Replace(con, th, "<thead>"+th+"</thead>", 1)
- }
- //u.Debug(con)
- return con
- }
- //修复表格
- func findpos(con string, iLen, start int) (newcon string) {
- defer qutil.Catch()
- n := len(con)
- layer := 0
- pos := 0
- if start >= 0 {
- if iLen == 6 {
- for i := iLen + start; i < len(con); i++ {
- if con[i] == '<' && i+6 < n {
- str := con[i : i+6]
- if str == "</tbod" {
- if layer == 0 {
- pos = i
- break
- } else {
- layer--
- }
- i += 6
- } else if str == "<tbody" {
- layer++
- i += 6
- }
- }
- }
- if pos+7 <= n && start+6 < pos {
- newcon = con[:start] + "<table" + con[start+6:pos] + "</table" + con[pos+7:]
- }
- } else {
- layer++
- nq := 0
- lasttr := 0
- for i := iLen + start; i < len(con); i++ {
- if con[i] == '<' && i+4 < n {
- if nq == 0 {
- str := con[i : i+4]
- if str == "</tr" {
- if layer <= 0 {
- pos = i //正常情况不会存在此类情况
- break
- } else {
- layer--
- lasttr = i
- }
- i += 4
- } else if str[:3] == "<tr" {
- layer++
- i += 4
- } else if str == "<tab" && i+6 < n && con[i+4:i+6] == "le" {
- if layer == 0 {
- break
- } else {
- //内嵌的表格
- nq++
- }
- }
- } else {
- if i+6 < n {
- str := con[i : i+6]
- if str == "</tabl" {
- nq--
- } else if str == "<table" {
- nq++
- }
- } else {
- break
- }
- }
- }
- }
- if pos == 0 && lasttr > 3 {
- pos = lasttr + 5
- } else if pos > 0 {
- pos += 5
- }
- if pos <= n && pos < len(con) && start < pos {
- newcon = con[:start] + "<table>" + con[start:pos] + "</table>" + con[pos:]
- }
- }
- }
- if newcon == "" {
- newcon = con
- }
- return
- }
- //td的值里面有一个包,并且没有冒号kv
- func isHasOnePkgAndNoKv(v1 string) (bool, string) {
- v1s := FindVal_1.FindAllString(v1, -1)
- colonCount := len(regDivision.FindAllString(v1, -1))
- if len(v1s) == 1 && colonCount < 2 {
- ispkgcolon := regexp.MustCompile(v1s[0] + "[::]").MatchString(v1)
- if (ispkgcolon && colonCount == 1) || (!ispkgcolon && colonCount == 0) {
- return true, v1s[0]
- }
- }
- return false, v1
- }
- //替换分包中混淆的词
- func replPkgConfusion(v1 string) string {
- v1 = PreReg.ReplaceAllString(v1, "")
- v1 = PreReg1.ReplaceAllString(v1, "")
- v1 = PreCon.ReplaceAllString(v1, "")
- v1 = PreCon2.ReplaceAllString(v1, "")
- return v1
- }
- //对td中的值,进行再处理
- func (tn *Table) TdContactFormat(contactFormat *u.ContactFormat, isSite bool, codeSite string) {
- //处理表格中的联系人信息
- indexMap := contactFormat.IndexMap
- matchMap := contactFormat.MatchMap
- //qutil.Debug("==============================td=======================", indexMap, matchMap)
- weightMap := map[string]map[string]interface{}{} //权重
- mustMatchFirst := len(indexMap) > 0 //第一个必须匹配上
- reCreate := false
- matchCount := 0
- contactTypeTagMap := map[string]map[string][]interface{}{}
- //qutil.Debug("============================", mustMatchFirst, indexMap, matchMap)
- notMatchTrCount := 0
- allAscFind := true //开启正序查询
- //涉及变量allAscFind,indexMap
- if len(indexMap) == 0 {
- isCanAddToIndexMap := false
- matchPrevFlag := false
- prevCanAddToIndexMap := false
- LS:
- for _, tr := range tn.TRs {
- for td_index, td := range tr.TDs {
- thisTdKvs := tn.tdkv(td) //获取td冒号kv
- //qutil.Debug(td.Val, len(thisTdKvs))
- // for _, v := range thisTdKvs {
- // qutil.Debug(v.Key, v.Value)
- // }
- if len(thisTdKvs) != 1 {
- preTdIndex := td_index - 1
- if preTdIndex >= 0 {
- preTdVal := tr.TDs[td_index-1].Val
- tdType := "" //前一个td中是否是采购、代理、中标
- for k, v := range ContactType {
- if v.MatchString(preTdVal) {
- tdType = k
- break
- }
- }
- if tdType != "" {
- for _, this := range thisTdKvs {
- if str := ContactInfoVagueReg.FindString(this.Key); str != "" {
- td.SortKV.AddKey(tdType+str, this.Value)
- }
- }
- }
- }
- continue
- }
- //1.处理带括号的()[]【】采购单位,代理机构;2.识别采购单位联系人、联系电话、代理机构联系人、联系电话
- goOnFunc, isContinue, td_k := tn.tdKV(thisTdKvs[0].Key, &matchPrevFlag, &isCanAddToIndexMap, &indexMap, "LS")
- //qutil.Debug("goOnFunc---", goOnFunc, "isContinue---", isContinue, "indexMap---", indexMap, "isCanAddToIndexMap---", isCanAddToIndexMap)
- if !goOnFunc {
- break LS
- }
- if isContinue {
- continue
- }
- //采购单位,代理机构,中标单位
- //qutil.Debug("td_k---", td_k, HasOrderContactType(td_k))
- for _, k := range HasOrderContactType(td_k) {
- if !ContactType[k].MatchString(td_k) { //不是采购单位,代理机构,中标单位跳过
- continue
- }
- if len(indexMap) == 0 {
- //qutil.Debug("isCanAddToIndexMap---", isCanAddToIndexMap, "prevCanAddToIndexMap---", prevCanAddToIndexMap, len(tr.TDs))
- if isCanAddToIndexMap || (prevCanAddToIndexMap && len(tr.TDs) == 1) {
- myPrevTdVal := ""
- if td_index-2 >= 0 {
- myPrevTdVal = tr.TDs[td_index-2].Val
- if myPrevTdVal != "" && len([]rune(myPrevTdVal)) < 10 && ContactInfoMustReg.MatchString(myPrevTdVal) {
- matchPrevFlag = true
- }
- }
- indexMap[0] = k
- break
- }
- } else {
- indexMap = map[int]string{}
- break LS
- }
- }
- }
- prevCanAddToIndexMap = isCanAddToIndexMap
- isCanAddToIndexMap = false
- }
- if len(indexMap) > 0 {
- allAscFind = false
- }
- }
- //////
- L:
- for tr_index, tr := range tn.TRs {
- thisTrHasMatch := false
- jumpNextTd := false
- for td_index, td := range tr.TDs {
- //和|以?及|与|、多个词和在一起
- jumpNextTd, thisTrHasMatch = tn.tdsMultipleWords(jumpNextTd, td, td_index, tr, thisTrHasMatch, indexMap)
- //分块之后的kv
- thisTdKvs := kvAfterDivideBlock("", td.Text, 3, tn.TableResult.RuleBlock, isSite, codeSite)
- if len(thisTdKvs) == 0 {
- thisTdKvs = tn.tdkv(td) //获取冒号kv
- }
- tdAscFind := true //开启td正序查询
- //qutil.Debug("---", td.Val, len(thisTdKvs), len(indexMap))
- if len(thisTdKvs) == 0 {
- continue
- } else if allAscFind && len(thisTdKvs) >= 3 && len(indexMap) == 0 {
- //采购人在联系人、电话后面的处理
- tdAscFind = tn.hasIndexMap(thisTdKvs, &indexMap, tdAscFind)
- }
- //qutil.Debug(len(thisTdKvs), len(tr.TDs))
- // if len(thisTdKvs) >= 2 && len(tr.TDs) == 1 { //td中包含多个kv值 5d6b2aa2a5cb26b9b73e79d2
- // tmpIndexMap := map[int]string{}
- // start := 0
- // for _, td_kv := range thisTdKvs {
- // qutil.Debug(td_kv.Key)
- // for _, k := range HasOrderContactType(td_kv.Key) {
- // tmpIndexMap[start] = k
- // start++
- // }
- // }
- // indexMap = tmpIndexMap
- // }
- prevKey := ""
- oldIndexMapLength := len(indexMap)
- thidTdIndex := td_index
- //notmatchCount := 0
- kvTitle := ""
- //qutil.Debug("indexMap++++++++++++++++++", indexMap, "len(thisTdKvs)", len(thisTdKvs), oldIndexMapLength)
- if len(thisTdKvs) >= 2 { //td中有多个kv重置indexMap
- indexMap = map[int]string{}
- allAscFind = true
- }
- for _, td_kv := range thisTdKvs {
- iscontinue := false
- td_v := td_kv.Value
- td_k := FilterContactKey(td_kv.Key) //带括号()[]的采购单位,代理机构处理
- td_k_length := len([]rune(td_k))
- if td_k_length < 2 || td_k_length > 15 {
- continue
- }
- //都为正序查询
- //qutil.Debug("td_k+++", td_k, "td_v+++", td_v, "allAscFind+++", allAscFind, "tdAscFind+++", tdAscFind)
- if allAscFind && tdAscFind {
- //都为正序查询处理
- matchCount, weightMap, matchMap, thisTrHasMatch, indexMap, iscontinue, reCreate, thidTdIndex = tn.asdFind(td_k, matchCount, weightMap, matchMap, td, thisTrHasMatch, td_kv, indexMap, iscontinue, reCreate, thidTdIndex, isSite, codeSite)
- }
- //qutil.Debug("indexMap++++++", indexMap, len(indexMap), "iscontinue+++", iscontinue)
- if iscontinue {
- continue
- }
- //不在同一块中
- //qutil.Debug(td_kv.Title, kvTitle, !ContactInfoMustReg.MatchString(td_kv.Key))
- if td_kv.Title != "" && kvTitle != td_kv.Title && len(indexMap) > 0 && !ContactInfoMustReg.MatchString(td_kv.Key) {
- thidTdIndex = 0
- matchMap = map[string]map[string]bool{}
- indexMap = map[int]string{}
- }
- kvTitle = td_kv.Title
- //qutil.Debug(td_k_length, td_k_length, len(indexMap))
- if td_k_length < 2 || td_k_length > 10 {
- continue
- }
- if len(indexMap) > 0 {
- //没有识别到采购单位联系人、联系电话、代理机构联系人、联系电话
- if !ContactInfoMustReg.MatchString(td_k) {
- //notmatchCount++
- //if notmatchCount < len(indexMap)*2 && false {//false???????
- // notmatchCount = 0
- // thidTdIndex = 0
- // indexMap = map[int]string{}
- // matchMap = map[string]map[string]bool{}
- //}
- if mustMatchFirst { //indexMap初始值大于0
- break L
- }
- continue
- }
- reCreate = true
- index := td_index
- //oldIndexMapLength原来的indexMap等于0 ,现在的indexMap大于1
- if oldIndexMapLength == 0 && len(indexMap) > 1 {
- if prevKey != td_k {
- prevKey = td_k
- index = td_index
- } else if prevKey == td_k {
- index++
- }
- }
- //kv.value为空
- if filterValue.MatchString(td_v) {
- thisTrHasMatch = true
- continue
- }
- //u.Debug(indexMap, td_k, td_v, matchMap, index, modle)
- //myContactType
- myContactType := indexMap[index]
- //qutil.Debug(indexMap, index, myContactType)
- if myContactType == "" && len(indexMap) == 1 {
- _, onlyContactType := u.FirstKeyValueInMap(indexMap)
- myContactType, _ = onlyContactType.(string)
- }
- //qutil.Debug("indexMap+++", indexMap, "index+++", index, "myContactType+++", myContactType)
- if myContactType == "" {
- continue
- }
- matchCount++
- if matchMap[myContactType] == nil {
- matchMap[myContactType] = map[string]bool{}
- }
- if IsContactKvHandle(ContactInfoMustReg.FindString(td_k), matchMap[myContactType]) {
- continue
- }
- matchMap[myContactType][ContactInfoMustReg.FindString(td_k)] = true
- if ContactType[myContactType].MatchString(td_k) {
- continue
- }
- thisTrHasMatch = true
- //modle
- modle(thisTdKvs, td, myContactType, td_k, td_v, &contactTypeTagMap, tn, &weightMap, tr_index, td_index, isSite, codeSite)
- }
- }
- //qutil.Debug("map===", td.SortKV.Map)
- }
- if allAscFind && !thisTrHasMatch {
- notMatchTrCount++
- if notMatchTrCount >= 2 {
- notMatchTrCount = 0
- indexMap = map[int]string{}
- }
- }
- }
- //u.Debug("end", matchCount, indexMap, matchMap)
- if matchCount == 0 {
- indexMap = map[int]string{}
- matchMap = map[string]map[string]bool{}
- }
- (*contactFormat).IndexMap = indexMap
- (*contactFormat).MatchMap = matchMap
- // for _, tr := range tn.TRs {
- // for _, td := range tr.TDs {
- // qutil.Debug("td.sort.map---", td.SortKV.Map)
- // }
- // }
- }
- //modle
- func modle(thisTdKvs []*u.Kv, td *TD, myContactType, td_k, td_v string, contactTypeTagMap *map[string]map[string][]interface{}, tn *Table, weightMap *map[string]map[string]interface{}, tr_index, td_index int, isSite bool, codeSite string) {
- modle := 0
- if len(thisTdKvs) == 1 {
- if regReplAllSpace.ReplaceAllString(thisTdKvs[0].Value, "") == "" {
- modle = 1
- } else {
- modle = 2
- }
- }
- if modle == 1 {
- td.Text = myContactType + td_k
- td.Val = td.Text
- } else {
- //
- if !strings.HasSuffix(td_k, "方式") {
- kvTags := GetKvTags([]*u.Kv{&u.Kv{Key: myContactType + td_k, Value: td_v}}, "", BuyerContacts, isSite, codeSite)
- if len(kvTags) == 1 {
- tagVal, _ := u.FirstKeyValueInMap(kvTags)
- if tagVal == "采购单位联系人" && ContactBuyerPersonFilterReg.MatchString(td_v) {
- return
- }
- if (*contactTypeTagMap)[myContactType] == nil {
- (*contactTypeTagMap)[myContactType] = map[string][]interface{}{}
- }
- myOldKeyArray := (*contactTypeTagMap)[myContactType][tagVal]
- if myOldKeyArray != nil {
- tn.TRs[myOldKeyArray[0].(int)].TDs[myOldKeyArray[1].(int)].SortKV.RemoveKey(myContactType + myOldKeyArray[2].(string))
- } else {
- (*contactTypeTagMap)[myContactType][tagVal] = make([]interface{}, 3)
- }
- if (*weightMap)[myContactType] == nil {
- (*weightMap)[myContactType] = map[string]interface{}{}
- }
- (*weightMap)[myContactType][tagVal] = 1
- (*contactTypeTagMap)[myContactType][tagVal] = []interface{}{tr_index, td_index, td_k}
- }
- }
- td.SortKV.AddKey(myContactType+td_k, td_v)
- }
- }
- //都为正序查询
- func (tn *Table) asdFind(td_k string, matchCount int, weightMap map[string]map[string]interface{}, matchMap map[string]map[string]bool, td *TD, thisTrHasMatch bool, td_kv *u.Kv, indexMap map[int]string, iscontinue bool, reCreate bool, thidTdIndex int, isSite bool, codeSite string) (int, map[string]map[string]interface{}, map[string]map[string]bool, bool, map[int]string, bool, bool, int) {
- for _, k := range HasOrderContactType(td_k) { //采购单位,代理机构,中标单位
- if !ContactType[k].MatchString(td_k) { //没有匹配到采购单位,代理机构,中标单位
- continue
- }
- matchCount++
- if weightMap[k] == nil {
- weightMap[k] = map[string]interface{}{}
- }
- //匹配到进行处理
- if ContactInfoVagueReg.MatchString(td_k) {
- thisTrHasMatch = tn.matchContactType(&matchMap, k, td_k, td_kv.Value, td, &weightMap, thisTrHasMatch, isSite, codeSite)
- } else if k == "采购单位" { //打标签,权重高的重新覆盖
- kvTags := GetKvTags([]*u.Kv{td_kv}, "", []string{"采购单位"}, isSite, codeSite)
- tagVal, weightVal := u.FirstKeyValueInMap(kvTags)
- if tagVal == k {
- if weightMap[k][k] == nil || (weightVal != nil && weightVal.(int) >= weightMap[k][k].(int)) || len(matchMap[k]) == 0 {
- weightMap[k][k] = weightVal.(int)
- matchMap[k] = map[string]bool{}
- indexMap = map[int]string{}
- }
- }
- }
- if u.IsMapHasValue(k, indexMap) { //map中是否存在value
- thisTrHasMatch = true
- iscontinue = true
- continue
- }
- if reCreate {
- indexMap = map[int]string{}
- reCreate = false
- }
- indexMap[thidTdIndex] = k
- iscontinue = true
- thisTrHasMatch = true
- thidTdIndex++
- break
- }
- if len(indexMap) == 0 && td_kv.PrevLine != "" {
- //td_kv.PrevLine
- prevLine := FilterSerial.ReplaceAllString(td_kv.PrevLine, "")
- for k, v := range ContactType { //采购单位,代理机构正则
- if u.IsArrayHasValue(prevLine, v.FindAllString(prevLine, -1)) {
- indexMap[thidTdIndex] = k
- thisTrHasMatch = true
- thidTdIndex++
- }
- }
- }
- if len(indexMap) == 0 && td_kv.Title != "" {
- //td_kv.Title
- if titleMatchType := ContactTypeTitleMatch(td_kv.Title); titleMatchType != "" {
- thidTdIndex = 0
- matchMap = map[string]map[string]bool{}
- indexMap = map[int]string{1: titleMatchType}
- // for i, t := range titleMatchType {
- // indexMap[i+1] = t
- // }
- }
- }
- return matchCount, weightMap, matchMap, thisTrHasMatch, indexMap, iscontinue, reCreate, thidTdIndex
- }
- //匹配到进行处理
- func (tn *Table) matchContactType(matchMap *map[string]map[string]bool, k string, td_k string, td_v string, td *TD, weightMap *map[string]map[string]interface{}, thisTrHasMatch bool, isSite bool, codeSite string) bool {
- if (*matchMap)[k] == nil {
- (*matchMap)[k] = map[string]bool{}
- }
- isAddToMatchMap := true
- if !strings.HasSuffix(td_k, "方式") {
- kvTags := GetKvTags([]*u.Kv{&u.Kv{Key: td_k, Value: td_v}}, "", BuyerContacts, isSite, codeSite)
- if len(kvTags) == 1 {
- tagVal, weightVal := u.FirstKeyValueInMap(kvTags)
- if tagVal == "采购单位联系人" && ContactBuyerPersonFilterReg.MatchString(td_v) {
- isAddToMatchMap = false
- }
- if td.SortKV.Map[tagVal] != nil {
- if (*weightMap)[k][tagVal] == nil || (weightVal != nil && weightVal.(int) >= (*weightMap)[k][tagVal].(int)) {
- (*weightMap)[k][tagVal] = weightVal.(int)
- td.SortKV.AddKey(tagVal, td_v)
- thisTrHasMatch = true
- }
- } else {
- (*weightMap)[k][tagVal] = weightVal.(int)
- }
- }
- }
- if isAddToMatchMap && !filterValue.MatchString(td_v) && td_v != "" {
- (*matchMap)[k][ContactInfoVagueReg.FindString(td_k)] = true
- }
- return thisTrHasMatch
- }
- //采购人在联系人、电话后面的处理
- func (tn *Table) hasIndexMap(thisTdKvs []*u.Kv, indexMap *map[int]string, tdAscFind bool) bool {
- //采购人在联系人、电话后面的处理
- isCanAddToIndexMap := false
- LL:
- for _, td_kv := range thisTdKvs {
- //1.处理带括号的()[]【】采购单位,代理机构;2.识别采购单位联系人、联系电话、代理机构联系人、联系电话
- goOnFunc, isContinue, td_k := tn.tdKV(td_kv.Key, nil, &isCanAddToIndexMap, indexMap, "LL")
- if !goOnFunc {
- break LL
- }
- if isContinue {
- continue
- }
- if len(*indexMap) == 0 {
- for _, k := range HasOrderContactType(td_k) { //采购单位,代理机构
- if !ContactType[k].MatchString(td_k) {
- continue
- }
- if isCanAddToIndexMap && len(*indexMap) == 0 {
- (*indexMap)[0] = k
- break
- }
- }
- }
- }
- if len(*indexMap) > 0 {
- tdAscFind = false
- }
- return tdAscFind
- }
- //和|以?及|与|、多个词和在一起
- func (tn *Table) tdsMultipleWords(jumpNextTd bool, td *TD, td_index int, tr *TR, thisTrHasMatch bool, indexMap map[int]string) (jump, thisTr bool) {
- if !jumpNextTd && len([]rune(td.Text)) >= 5 && len([]rune(td.Text)) <= 15 && regSplit.MatchString(td.Text) && td_index+1 < len(tr.TDs) {
- thisTdVals := regSplit.Split(td.Text, -1)
- nextTdVals := MultipleValueSplitReg.Split(tr.TDs[td_index+1].Val, -1)
- if len(thisTdVals) == len(nextTdVals) { //本次和下个长度相等
- for _, k := range HasOrderContactType(td.Text) { //采购单位,代理机构
- if ContactType[k].MatchString(td.Text) { //采购单位,代理机构
- for thisTdVals_k, thisTdVals_v := range thisTdVals {
- thisTdVals_v = strings.TrimSpace(thisTdVals_v)
- if ContactType[k].MatchString(thisTdVals_v) { //采购单位,代理机构
- thisTrHasMatch = true
- tr.TDs[td_index+1].SortKV.AddKey(thisTdVals_v, nextTdVals[thisTdVals_k])
- continue
- }
- if !ContactInfoMustReg.MatchString(thisTdVals_v) {
- continue
- }
- jumpNextTd = true
- thisTrHasMatch = true
- tr.TDs[td_index+1].SortKV.AddKey(k+thisTdVals_v, nextTdVals[thisTdVals_k])
- }
- break
- }
- }
- if len(indexMap) > 0 {
- _, onlyContactType := u.FirstKeyValueInMap(indexMap)
- if myContactType, _ := onlyContactType.(string); myContactType != "" {
- for thisTdVals_k, thisTdVals_v := range thisTdVals {
- thisTdVals_v = strings.TrimSpace(thisTdVals_v)
- if ContactInfoMustReg.MatchString(thisTdVals_v) {
- jumpNextTd = true
- thisTrHasMatch = true
- tr.TDs[td_index+1].SortKV.AddKey(myContactType+thisTdVals_v, nextTdVals[thisTdVals_k])
- }
- }
- }
- }
- }
- } else {
- jumpNextTd = false
- }
- return jumpNextTd, thisTrHasMatch
- }
- //采购单位,代理机构
- func (tn *Table) tdHasOrderContactType(td_k string, indexMap *map[int]string, tr *TR, prevCanAddToIndexMap, isCanAddToIndexMap, matchPrevFlag *bool, td_index int) (gotoFunc bool) {
- for _, k := range HasOrderContactType(td_k) { //采购单位,代理机构
- if !ContactType[k].MatchString(td_k) {
- continue
- }
- if len(*indexMap) == 0 {
- if (*isCanAddToIndexMap) || (*prevCanAddToIndexMap && len(tr.TDs) == 1) {
- myPrevTdVal := ""
- if td_index-2 >= 0 {
- myPrevTdVal = tr.TDs[td_index-2].Val
- }
- if myPrevTdVal != "" && len([]rune(myPrevTdVal)) < 10 && ContactInfoMustReg.MatchString(myPrevTdVal) {
- (*matchPrevFlag) = true
- }
- (*indexMap)[0] = k
- break
- }
- } else {
- (*indexMap) = map[int]string{}
- return false
- }
- }
- return true
- }
- //1.处理带括号的()[]【】采购单位,代理机构;2.识别采购单位联系人、联系电话、代理机构联系人、联系电话
- func (tn *Table) tdKV(key string, matchPrevFlag, isCanAddToIndexMap *bool, indexMap *map[int]string, gotoName string) (goOnFunc, isContinue bool, td_k string) {
- td_k = FilterContactKey(key) //带括号的()[]【】采购单位,代理机构处理
- td_k_length := len([]rune(td_k))
- if td_k_length < 2 || td_k_length > 15 {
- return true, true, "" //继续执行,跳过当前循环
- }
- isContinue = ContactInfoMustReg.MatchString(td_k) //识别采购单位联系人、联系电话、代理机构联系人、联系电话
- if isContinue || (ContactInfoVagueReg.MatchString(td_k) && u.IsMapHasValue(td_k, ContactType)) {
- if gotoName == "LS" && !(*matchPrevFlag) && len(*indexMap) > 0 {
- (*indexMap) = map[int]string{}
- return false, false, "" //中断外层循环
- }
- if gotoName == "LL" && len(*indexMap) > 0 {
- (*indexMap) = map[int]string{}
- return false, false, ""
- }
- (*isCanAddToIndexMap) = true
- }
- return true, false, td_k //继续执行,不结束当前循环,返回处理后的值
- }
- //获取td冒号kv
- func (tn *Table) tdkv(td *TD) []*u.Kv {
- thisTdKvs := colonkvEntity.GetKvs(td.Text, tn.Desc, 2) //获取冒号kv
- //获取冒号
- if len(thisTdKvs) == 0 {
- tdValue := regReplAllSpace.ReplaceAllString(td.Text, "") //去除空格换行
- if tdValue != "" && len([]rune(tdValue)) < 10 {
- thisTdKvs = append(thisTdKvs, &u.Kv{
- Key: tdValue,
- Value: "",
- })
- }
- }
- return thisTdKvs
- }
- func (table *Table) extractPriceNumber() {
- lineMapArr := make(map[string]*SortMap)
- lineMap := make(map[string]*SortMap)
- lineMapArr, lineMap = initLineMapLineMapArr(table) //不同数据类型的数据组合
- //qutil.Debug("lineMapArr----", lineMapArr)
- if len(lineMapArr) > 0 {
- for _, arrMap := range lineMapArr {
- resultArrMap := table.matchMapArrPrinceNumber(arrMap) //最终数据
- //qutil.Debug("resultArrMap-------------------", resultArrMap)
- //处理数组长度不一致情况
- if len(resultArrMap) > 0 {
- numLen := len(resultArrMap["number"])
- priceLen := len(resultArrMap["price"])
- itemLen := len(resultArrMap["item"])
- maxNum := numLen //获取最大长度
- if numLen == 0 { //没有
- maxNum = priceLen
- }
- //取个数数据的长度为基准(数据长度可能不一致)
- if numLen != priceLen && numLen > 0 && priceLen > 0 { //有number和price数据且长度不同,进行数据增减补齐
- if priceLen > numLen { //price多,删
- tmpArr := resultArrMap["price"]
- resultArrMap["price"] = tmpArr[:numLen]
- } else if priceLen < numLen { //price少,补空
- for {
- resultArrMap["price"] = append(resultArrMap["price"], "")
- //qutil.Debug("=============price==============")
- if len(resultArrMap["price"]) == numLen {
- break
- }
- }
- }
- }
- if maxNum > 0 && itemLen > 0 && maxNum != itemLen { //有price或者number,item长度保持一致
- if itemLen > maxNum {
- tmpArr := resultArrMap["item"]
- resultArrMap["item"] = tmpArr[:maxNum]
- } else if itemLen < maxNum {
- for {
- resultArrMap["item"] = append(resultArrMap["item"], "")
- //qutil.Debug("=============item==============")
- if len(resultArrMap["item"]) == maxNum {
- break
- }
- }
- }
- }
- //封装成一一对应数据
- /*
- {
- "price" :["123","125"],
- "number" :["1","12"]
- }
- 转换为:
- [
- {"price":"123","number":"1"},
- {"price":"125","number":"12"}
- ]
- */
- finishData := []map[string]interface{}{}
- //qutil.Debug("maxNum--------------------", maxNum)
- for t := 0; t < maxNum; t++ {
- tmpMap := make(map[string]interface{})
- if len(resultArrMap["number"]) > 0 {
- number := resultArrMap["number"][t]
- tmpMap["number"] = number
- }
- if len(resultArrMap["price"]) > 0 {
- price := resultArrMap["price"][t]
- tmpMap["price"] = price
- }
- if len(resultArrMap["item"]) > 0 {
- item := resultArrMap["item"][t]
- runeItem := []rune(qutil.ObjToString(item))
- if len(runeItem) > 50 {
- tmpMap["item"] = string(runeItem[:50])
- } else {
- tmpMap["item"] = item
- }
- }
- finishData = append(finishData, tmpMap)
- }
- //qutil.Debug(finishData)
- if len(finishData) > 0 {
- table.PriceNumberData = append(table.PriceNumberData, finishData)
- }
- }
- }
- }
- if len(lineMap) > 0 {
- for _, strMap := range lineMap {
- resultStrMap := table.matchMapPrinceNumber(strMap)
- //qutil.Debug("resultStrMap---", resultStrMap)
- if len(resultStrMap) > 0 {
- if resultStrMap["price"] != nil || resultStrMap["number"] != nil { //有price或者number在保存
- if item := qutil.ObjToString(resultStrMap["item"]); item != "" { //item过长截取
- runeItem := []rune(item)
- if len(runeItem) > 50 {
- resultStrMap["item"] = string(runeItem[:50])
- }
- }
- finishData := []map[string]interface{}{}
- finishData = append(finishData, resultStrMap)
- //qutil.Debug("finishData---", finishData)
- if len(finishData) > 0 {
- table.PriceNumberData = append(table.PriceNumberData, finishData)
- }
- }
- }
- }
- }
- //qutil.Debug("table.PriceNumberData---------", table.PriceNumberData)
- }
- //数组数据抽取price和number
- func (table *Table) matchMapArrPrinceNumber(arrMap *SortMap) map[string][]interface{} {
- result := make(map[string][]interface{}) //最终存储数据
- for _, key := range arrMap.Keys {
- val := arrMap.Map[key].([]string)
- for f, reg := range u.PriceNumberReg {
- key = tabletitleclear2.ReplaceAllString(key, "") //过滤
- if reg.MatchString(key) { //匹配成功
- //qutil.Debug("arr----key", key, val, f)
- tmp := []interface{}{}
- if f == "price" {
- tmp = dealPriceInterface(key, val...) //处理金额,单位转换
- } else if f == "number" {
- tmp = dealNumberInterface(val...) //处理数量
- } else {
- for _, v := range val {
- tmp = append(tmp, v)
- }
- }
- if len(tmp) > 0 {
- result[f] = tmp
- }
- }
- }
- }
- return result
- }
- //字符串数据抽取price和number
- func (table *Table) matchMapPrinceNumber(strMap *SortMap) map[string]interface{} {
- result := make(map[string]interface{})
- for _, key := range strMap.Keys {
- val := qutil.ObjToString(strMap.Map[key])
- for f, reg := range u.PriceNumberReg {
- key = tabletitleclear2.ReplaceAllString(key, "") //过滤
- if reg.MatchString(key) { //匹配成功
- //qutil.Debug("str----key", key, val)
- if f == "price" {
- if len(regHz.FindAllString(val, -1)) > 5 { //price中汉字过多视为内容错误
- continue
- }
- tmp := dealPriceInterface(key, val)[0] //处理金额,单位转换
- result[f] = tmp
- } else if f == "number" {
- tmp := dealNumberInterface(val)[0]
- result[f] = tmp
- } else {
- result[f] = val
- }
- }
- }
- }
- return result
- }
- //table中抽取品牌,table.BrandData
- func (table *Table) analyBrand() {
- //5c2d8c05a5cb26b9b782572b
- //产品名称 品牌 规格 单价 单位 数量 小计 质保期
- lineMapArr := make(map[string]*SortMap)
- lineMap := make(map[string]*SortMap)
- brandRule := u.BrandRules
- //初始化lineMapArr,lineMap;
- lineMapArr, lineMap = initLineMapLineMapArr(table) //处理数组数据后,匹配必须title和替换要保存的title
- //qutil.Debug("lineMapArr----", len(lineMapArr))
- if len(lineMapArr) > 0 {
- for _, aMap := range lineMapArr {
- maxNum := 0 //记录最大长度
- arrcount1 := 0 //记录key是否存在必须title(数组数据)
- arrcount2 := 0
- ka := make(map[string][]string) //最终存储数据
- //qutil.Debug("aMap.Keys----", aMap.Keys)
- //匹配商品规则
- arrcount1, arrcount2, ka = table.matchMapArrBrandRule(aMap, brandRule, ka, arrcount1, arrcount2)
- //找最终存储数据的最小len(arr)
- // for _, vf := range ka {
- // //找最短的数组
- // lenVal := len(vf)
- // if minNum == 0 || minNum > lenVal { //maxNum = len(最短数组)
- // minNum = lenVal
- // }
- // }
- //找最终存储数据的最大len(arr),小的补空
- for _, vf1 := range ka {
- lenVal := len(vf1)
- if lenVal > maxNum {
- maxNum = lenVal
- }
- }
- //table.BrandData商品存储
- finishKa := make(map[string][]string)
- for vf2K, vf2 := range ka {
- if len(vf2) < maxNum {
- if vf2K == "unitprice" { //价格的当前总数比最大的总数小就跳过,可能是总价格而不是单个的价格
- continue
- }
- lenMv := maxNum - len(vf2)
- for i := 0; i < lenMv; i++ {
- vf2 = append(vf2, "")
- }
- }
- finishKa[vf2K] = vf2
- }
- hasKey(table, arrcount1) //是否匹配到table中的标题
- //qutil.Debug("finishKa----", finishKa)
- if arrcount1 >= 1 {
- if arrcount1+arrcount2 == 1 { //删除只匹配到一个价钱(总价)
- delete(finishKa, "unitprice")
- }
- finishData := dealArrData(maxNum, finishKa)
- table.BrandData = append(table.BrandData, finishData) //存储table.BrandData
- }
- }
- }
- //处理string数据后,匹配必须title和替换要保存的title
- //qutil.Debug("lineMap----", len(lineMap))
- if len(lineMap) > 0 {
- for _, sMap := range lineMap {
- strcount1 := 0 //记录key是否存在必须title(字符串数据)
- strcount2 := 0
- endStrMap := make(map[string]string)
- //qutil.Debug(k, "aMap.Keys----", sMap.Keys)
- //匹配商品规则
- strcount1, strcount2, endStrMap = table.matchMapBrandRule(sMap, brandRule, endStrMap, strcount1, strcount2)
- //原始字符串数据处理
- hasKey(table, strcount1) //是否匹配到table中的标题
- //qutil.Debug("endStrMap----", endStrMap)
- if strcount1 >= 1 {
- if strcount1+strcount2 == 1 { //删除只匹配到一个价钱(总价)
- delete(endStrMap, "unitprice")
- }
- finishData := dealStrData(endStrMap) //处理数据
- if len(finishData) > 0 {
- table.BrandData = append(table.BrandData, finishData)
- }
- }
- }
- }
- }
- //字符串匹配商品规则
- func (table *Table) matchMapBrandRule(sMap *SortMap, brandRule map[string]map[string]string, endStrMap map[string]string, strcount1, strcount2 int) (int, int, map[string]string) {
- for _, k1 := range sMap.Keys {
- match := false //记录must是否匹配到
- v1 := qutil.ObjToString(sMap.Map[k1])
- // for k1, v1 := range sMap {
- //qutil.Debug(k1, "++++++++++", v1)
- if v1 == "" {
- continue
- }
- //匹配必须title
- for nameM, r := range brandRule["must"] {
- if convert(k1, r) { //匹配成功
- v1tmp1 := v1
- match = true
- if nameM == "itemname" || nameM == "modal" { //特殊处理itemname
- hasGoods(table, v1)
- if nameM == "itemname" {
- v1tmp1 = filterItem(v1)[0] //过滤itemname
- if v1tmp1 == "" {
- break
- }
- }
- }
- if nameM == "brandname" || nameM == "modal" { //特殊处理brandname
- if endStrMap["brandname"] == "" {
- brand, allNull := hasBrand(table, v1)
- if !allNull {
- endStrMap["brandname"] = brand[0]
- }
- }
- }
- //unitprice
- if nameM == "unitprice" { //处理金额
- v1tmp1 = dealPrice(k1, v1)[0]
- }
- if nameM != "brandname" && endStrMap[nameM] == "" {
- endStrMap[nameM] = v1tmp1
- }
- strcount1++
- }
- }
- //替换其它要保存字段
- if !match {
- for nameR, r := range brandRule["replace"] {
- if convert(k1, r) { //匹配成功
- v1tmp2 := v1
- //totalprice
- if nameR == "totalprice" { //处理金额
- v1tmp2 = dealPrice(k1, v1)[0]
- }
- //number
- if nameR == "number" { //处理数量
- varr1, uname1 := dealNumber(v1)
- v1tmp2 = varr1[0]
- //从number中获取到的单位
- if endStrMap["unitname"] == "" && uname1[0] != "" {
- endStrMap["unitname"] = uname1[0]
- }
- }
- if v1tmp2 != "" {
- endStrMap[nameR] = v1tmp2
- }
- strcount2++
- }
- }
- }
- //}
- }
- return strcount1, strcount2, endStrMap
- }
- //数组匹配商品规则
- func (table *Table) matchMapArrBrandRule(aMap *SortMap, brandRule map[string]map[string]string, ka map[string][]string, arrcount1, arrcount2 int) (int, int, map[string][]string) {
- for _, k0 := range aMap.Keys {
- match := false //记录must是否匹配到
- v0 := aMap.Map[k0].([]string)
- //匹配必须title
- for nameM, r := range brandRule["must"] {
- if convert(k0, r) { //匹配成功
- v0tmp1 := v0
- match = true
- if len(ka[nameM]) != 0 && strings.Contains(k0, "描述") { //防止k0匹配到多次 和特殊情况 物料名称 物料描述同时出现
- continue
- }
- if nameM == "itemname" || nameM == "modal" {
- hasGoods(table, v0...) //判断itemname和modal中有没有商品
- if nameM == "itemname" {
- v0tmp1 = filterItem(v0...) //过滤itemname
- }
- }
- if nameM == "brandname" || nameM == "modal" {
- if len(ka["brandname"]) == 0 {
- brand, allNull := hasBrand(table, v0...)
- if !allNull {
- ka["brandname"] = brand
- }
- }
- }
- //unitprice
- if nameM == "unitprice" { //处理金额
- v0tmp1 = dealPrice(k0, v0...)
- }
- if nameM != "brandname" && len(ka[nameM]) == 0 {
- ka[nameM] = v0tmp1
- }
- arrcount1++
- }
- }
- //替换其它要保存字段
- if !match { //must未匹配,匹配replace
- for nameR, r := range brandRule["replace"] {
- if convert(k0, r) { //匹配成功
- v0tmp2 := v0
- //totalprice
- if nameR == "totalprice" { //处理金额
- v0tmp2 = dealPrice(k0, v0...)
- }
- //number
- if nameR == "number" { //处理数量
- uname0 := []string{}
- v0tmp2, uname0 = dealNumber(v0...)
- if len(ka["unitname"]) == 0 && len(uname0) != 0 {
- ka["unitname"] = uname0
- }
- }
- if len(v0tmp2) > 0 {
- ka[nameR] = v0tmp2
- }
- arrcount2++
- }
- }
- }
- }
- return arrcount1, arrcount2, ka
- }
- //初始化lineMapArr,lineMap
- func initLineMapLineMapArr(table *Table) (lineMapArr map[string]*SortMap, lineMap map[string]*SortMap) {
- lineMapArr = make(map[string]*SortMap)
- lineMap = make(map[string]*SortMap)
- for _, key := range table.SortKV.Keys { //遍历table.SortKV.Keys而不是直接遍历table.SortKV.Map是为了得到table头的顺序
- val := table.SortKV.Map[key]
- key = regReplAllSpace.ReplaceAllString(key, "")
- key = strings.Replace(key, "", "", -1) //处理一个特殊的采购量 经上层处理空格后未处理掉
- //qutil.Debug(key, "---------------------------", val)
- if realTypeVal, ok := val.([]string); ok { //val为数组 {"数量":["1","2","3"]}
- /*
- {
- "商品":["",""],
- "商品_"["",""],
- }
- */
- valArr, allempty := filterVal(realTypeVal...) //过滤数据
- if allempty {
- continue
- }
- realTypeVal = valArr
- line := underline.FindString(key)
- lineValMap1 := lineMapArr[line]
- // i := 1
- // L:
- // for { //去除数组空数据
- // last := realTypeVal[len(realTypeVal)-i]
- // if last == "" {
- // i++
- // if i > len(realTypeVal) {
- // break
- // }
- // goto L
- // } else {
- // break
- // }
- // }
- // dislodgeNull := realTypeVal[:(len(realTypeVal) - i + 1)] //去除数组中空数据
- if len(realTypeVal) > 0 {
- if lineValMap1 == nil {
- tmp := NewSortMap()
- tmp.AddKey(key, realTypeVal)
- lineMapArr[line] = tmp
- } else {
- lineValMap1.AddKey(key, realTypeVal)
- }
- }
- //qutil.Debug("lineMapArr---", lineMapArr[line].Keys, lineMapArr[line].Map)
- } else if realTypeVal, b := val.(string); b { //val为字符串 {"数量":"1"}
- /*
- {
- "商品:"",名称:"",
- "商品_:"",名称_:"",
- "商品__:"",名称__:"",
- }
- */
- valArr, allempty := filterVal(realTypeVal) //过滤数据
- if allempty {
- continue
- }
- realTypeVal = valArr[0]
- line := underline.FindString(key)
- lineValMap2 := lineMap[line]
- if lineValMap2 == nil {
- tmp := NewSortMap()
- tmp.AddKey(key, realTypeVal)
- lineMap[line] = tmp
- } else {
- lineValMap2.AddKey(key, realTypeVal)
- }
- //qutil.Debug("lineMap---", lineMap[line].Keys, lineMap[line].Map)
- } else {
- // "_id" : ObjectId("5c2c3802a5cb26b9b78646c4")5c2b0551a5cb26b9b7cb05db否5c2a42e6a5cb26b9b763ba5a采购人:一、采购人5c2b06f5a5cb26b9b7cc4409
- //成交供应商排名 [map[entname:昆明合优科技有限公司 sortstr:第一中标候选人 sort:1] map[sort:2 entname:昆明厚起科技有限公司 sortstr:第二中标候选人] map[entname:云南远安科技发展有限公司 sortstr:第三中标候选人 sort:3]]
- //qutil.Debug("err data:", key, val)
- }
- }
- return lineMapArr, lineMap
- }
- func dealArrData(maxNum int, ka map[string][]string) []map[string]string {
- for k2, v2 := range ka {
- //处理数组长度不相等,使长度一致
- if len(v2) > maxNum {
- ka[k2] = v2[:maxNum]
- }
- }
- finalData := assembleData(ka, 1)
- if len(finalData) > 0 {
- return finalData
- }
- return nil
- }
- func dealStrData(kv map[string]string) []map[string]string {
- finalData := []map[string]string{}
- if len(kv) > 0 {
- finalData = assembleData(kv, 2)
- }
- return finalData
- }
- //组装数据,每一行的数据为一数据集合
- func assembleData(m interface{}, n int) []map[string]string {
- defer qutil.Catch()
- /*
- {
- "itemname":["计算机","打印机","机柜"],
- "number" :["1","12","4"]
- }
- */
- datas := []map[string]string{}
- if n == 1 { //数组数据
- realTypeM := m.(map[string][]string)
- //根据数组数据的顺序 将多个数组中索引相同的数据拼装成一个map,并将这多个map放入一个arr
- /*
- arr1 ["a1","b1","c1"]
- arr2 ["a2","b2","c2"]
- [
- {"a1","a2"},
- {"b1","b2"},
- {"c1","c2"}
- ]
- */
- //start
- for k3, v3 := range realTypeM {
- for _, val := range v3 {
- data := make(map[string]string)
- data[k3] = val
- datas = append(datas, data)
- }
- break
- }
- for i, data := range datas {
- for k4, v4 := range realTypeM {
- if i < len(v4) { //数组数据长度不一致
- if v4[i] != " " {
- data[k4] = v4[i]
- } else {
- delete(data, k4)
- }
- } else {
- fmt.Println("err table")
- }
- }
- datas[i] = data
- }
- //end
- for _, fdv := range datas { //清除空数据和只含特殊符号的数据
- for fmk, fmv := range fdv {
- if tabletdclear.ReplaceAllString(fmv, "") == "" {
- delete(fdv, fmk)
- }
- }
- }
- } else { //字符串数据
- realTypeM := m.(map[string]string)
- datas = append(datas, realTypeM)
- }
- return datas
- }
- ////组装数据,每一行的数据为一数据集合
- //func assembleData(m interface{}, n int) []map[string]string {
- // defer qutil.Catch()
- // /*
- // {
- // "itemname":["计算机","打印机","机柜"],
- // "number" :["1","12","4"]
- // }
- // */
- // datas := []map[string]string{}
- // switch reflect.TypeOf(m).String() {
- // case "map[string][]string": //数组数据
- // realTypeM := m.(map[string][]string)
- // //根据数组数据的顺序 将多个数组中索引相同的数据拼装成一个map,并将这多个map放入一个arr
- // /*
- // arr1 ["a1","b1","c1"]
- // arr2 ["a2","b2","c2"]
- // [
- // {"a1","a2"},
- // {"b1","b2"},
- // {"c1","c2"}
- // ]
- // */
- // //start
- // for k3, v3 := range realTypeM {
- // for _, val := range v3 {
- // data := make(map[string]string)
- // data[k3] = val
- // datas = append(datas, data)
- // }
- // break
- // }
- // for i, data := range datas {
- // for k4, v4 := range realTypeM {
- // if i < len(v4) { //数组数据长度不一致
- // if v4[i] != " " {
- // data[k4] = v4[i]
- // } else {
- // delete(data, k4)
- // //continue
- // }
- // } else {
- // fmt.Println("err table")
- // //continue
- // }
- // }
- // datas[i] = data
- // }
- // //end
- // // for _, fdv := range datas { //清除空数据和只含特殊符号的数据
- // // for fmk, fmv := range fdv {
- // // if tabletdclear.ReplaceAllString(fmv, "") == "" {
- // // delete(fdv, fmk)
- // // }
- // // }
- // // }
- // case "map[string]string": //字符串数据
- // realTypeM := m.(map[string]string)
- // datas = append(datas, realTypeM)
- // default:
- // }
- // return datas
- //}
- func convert(key, r string) bool {
- defer qutil.Catch()
- flag := false
- key = tabletitleclear.ReplaceAllString(key, "")
- reg, err := regexp.Compile(r)
- if err != nil {
- fmt.Println("reg err:", err)
- return false
- }
- flag = reg.MatchString(key)
- return flag
- }
- func hasKey(table *Table, n int) {
- defer qutil.Catch()
- if table.TableResult.HasKey == 1 {
- return
- }
- if n >= 1 {
- table.TableResult.HasKey = 1
- }
- }
- func hasGoods(table *Table, data ...string) {
- defer qutil.Catch()
- goodsArr := make([]string, len(data))
- //fmt.Println("table.TableResult.HasGoods=====", table.TableResult.HasGoods)
- if table.TableResult.HasGoods == 1 {
- return
- }
- for i, d := range data {
- if d != "" {
- goods := u.GoodsGet.CheckSensitiveWord(d)
- //fmt.Println("goods======", goods)
- goodsArr[i] = goods
- if len(goods) > 0 {
- table.TableResult.HasGoods = 1
- break
- }
- }
- }
- }
- //func hasBrand(table *Table, data ...string) {
- // defer qutil.Catch()
- // if table.TableResult.HasBrand == 1 {
- // return
- // }
- // for i, d := range data {
- // if d != "" {
- // brand := u.BrandGet.CheckSensitiveWord(d)
- // qutil.Debug(d, brand)
- // if brand != "" {
- // table.TableResult.HasBrand = 1
- // break
- // }
- // }
- // }
- //}
- func hasBrand(table *Table, data ...string) ([]string, bool) {
- defer qutil.Catch()
- //fmt.Println("table.TableResult.HasBrand---------", table.TableResult.HasBrand)
- brandArr := make([]string, len(data))
- // if table.TableResult.HasBrand == 1 {
- // return brandArr, 1
- // }
- allNull := true
- for i, d := range data {
- //if d != "" {
- brand := u.BrandGet.CheckSensitiveWord(d)
- if brand != "" {
- allNull = false
- }
- //fmt.Println("brand======", brand)
- brandArr[i] = brand
- if len(brand) > 0 {
- table.TableResult.HasBrand = 1
- }
- //}
- }
- return brandArr, allNull
- }
- var clearnn *regexp.Regexp = regexp.MustCompile("([\\d.]*)[\\n\\s]*[\\((][\\d.]+[)\\)]")
- //过滤td值
- func filterVal(val ...string) ([]string, bool) {
- defer qutil.Catch()
- n := 0 //记录被过滤的个数
- for i, v := range val {
- if len(clearnn.FindStringSubmatch(v)) > 0 {
- tmpv := clearnn.FindStringSubmatch(v)[1]
- if tmpv != "" {
- v = tmpv
- }
- }
- afterFilter := tabletdclear.ReplaceAllString(v, "")
- afterFilter = NullVal.ReplaceAllString(afterFilter, "")
- if afterFilter == "" {
- n++
- }
- val[i] = afterFilter
- }
- allempty := false
- if n == len(val) { //所有都被过滤掉
- allempty = true
- }
- return val, allempty
- }
- //过滤itemname全是数字
- func filterItem(itemval ...string) []string {
- defer qutil.Catch()
- result := []string{}
- for _, v := range itemval {
- afterFilter := numclear.ReplaceAllString(v, "")
- if afterFilter != "" {
- result = append(result, v)
- } else {
- result = append(result, afterFilter)
- }
- }
- return result
- }
- //处理价格
- func dealPriceInterface(key string, val ...string) (result []interface{}) {
- defer qutil.Catch()
- for _, v := range val {
- if num1.MatchString(v) { //含数字
- tdIsWan := strings.Contains(v, "万")
- if !tdIsWan {
- if strings.Contains(key, "万") {
- v = v + "万"
- }
- }
- data := []interface{}{v, ""}
- money := clear.ObjToMoney(data)[0]
- result = append(result, money)
- } else {
- result = append(result, "")
- }
- }
- return
- }
- //处理number
- func dealNumberInterface(val ...string) (result []interface{}) {
- defer qutil.Catch()
- for _, v := range val { //1个 1.00个
- n := numclear.FindString(v)
- if n == "" {
- result = append(result, "")
- } else if tmp := clear.NumChar[n]; tmp != nil { //一二三...
- result = append(result, tmp)
- } else { //数字
- result = append(result, qutil.IntAll(strings.Split(n, ".")[0]))
- }
- }
- return
- }
- //处理价格
- func dealPrice(key string, val ...string) []string {
- defer qutil.Catch()
- result := []string{}
- for _, v := range val {
- data := []interface{}{v, key}
- money := clear.ObjToMoney(data)[0]
- result = append(result, fmt.Sprintf("%v", money))
- }
- // result := []string{}
- // for _, v := range val { //1.00万元 1元 2.25元/斤
- // tmparr := strings.Split(v, ".")
- // tmparr[0] = moneyNum.ReplaceAllString(tmparr[0], "")
- // if iswan {
- // result = append(result, tmparr[0]+"0000")
- // } else { //td val值带万
- // if strings.Contains(v, "万") { //价格中带有万
- // result = append(result, tmparr[0]+"0000")
- // } else {
- // result = append(result, tmparr[0])
- // }
- // }
- // }
- return result
- }
- //处理number
- func dealNumber(val ...string) ([]string, []string) {
- defer qutil.Catch()
- unitnameArr := []string{}
- result := []string{}
- for _, v := range val { //1个 1.00个
- n := numclear.FindString(v)
- unitname := numclear.ReplaceAllString(v, "") //匹配个数后的单位
- unitnameArr = append(unitnameArr, unitname)
- //val[i] = strings.Split(n, ".")[0]
- result = append(result, strings.Split(n, ".")[0])
- }
- return result, unitnameArr
- }
- func (tn *Table) analyProNameAndItemNumber() {
- }
|